Skip to content

armgit / camel

Public Imported from GitHub
0

🐫 CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org

refactor BaseEnvironment into SingleStep and MultiStep environments (#1810)

halleritedac4d318bb75
diff --git a/camel/datasets/base.py b/camel/datasets/base.py
index ec96cad4c..f65fb2582 100644
--- a/camel/datasets/base.py
+++ b/camel/datasets/base.py
@@ -432,8 +432,7 @@ class GenerativeDataset(Dataset):
self.verifier = verifier
self.agent = agent
- self.seed = seed
- random.seed(self.seed)
+ self._rng = random.Random(seed)
self._data: List[DataPoint] = []
@@ -606,6 +605,12 @@ class GenerativeDataset(Dataset):
)
return self._data[idx]
+ def sample(self) -> DataPoint:
+ if len(self._data) == 0:
+ raise RuntimeError("Dataset is empty, cannot sample.")
+ idx = self._rng.randint(0, len(self._data) - 1)
+ return self[idx]
+
def save_to_jsonl(self, file_path: Union[str, Path]) -> None:
r"""Saves the dataset to a JSONL (JSON Lines) file.
diff --git a/camel/environments/__init__.py b/camel/environments/__init__.py
index b0bbfb9a3..c580dd577 100644
--- a/camel/environments/__init__.py
+++ b/camel/environments/__init__.py
@@ -11,6 +11,15 @@
# See the License for the specific language governing permissions and
# limitations under the License.
# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
-from .base import BaseEnvironment
+from .models import Action, Environment, Observation, StepResult
+from .multi_step import MultiStepEnv
+from .single_step import SingleStepEnv
-__all__ = ["BaseEnvironment"]
+__all__ = [
+ "Environment",
+ "SingleStepEnv",
+ "MultiStepEnv",
+ "Action",
+ "Observation",
+ "StepResult",
+]
diff --git a/camel/environments/base.py b/camel/environments/base.py
deleted file mode 100644
index edd0ce021..000000000
--- a/camel/environments/base.py
+++ /dev/null
@@ -1,509 +0,0 @@
-# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
-
-import asyncio
-from abc import ABC, abstractmethod
-from datetime import datetime, timezone
-from typing import Any, Dict, List, Optional, Tuple
-
-from pydantic import BaseModel, Field
-
-from camel.agents import ChatAgent
-from camel.datasets.base import GenerativeDataset, StaticDataset
-from camel.extractors.base import BaseExtractor
-from camel.logger import get_logger
-from camel.verifiers.base import (
- BaseVerifier,
- VerificationResult,
-)
-from camel.verifiers.models import (
- VerificationOutcome,
- VerifierInput,
-)
-
-logger = get_logger(__name__)
-
-# TODO: Add MachineInfo into this file
-# TODO: TeacherAgent should be renamed into neural_reward_model.
-# This is where PRMs or such could be useful.
-# Should probably be its own class and not just raw ChatAgent
-
-
-class Action(BaseModel):
- r"""Represents an action taken in an environment.
-
- This class defines the input context, the LLM-generated output, and
- metadata required for verification and tracking within an RL
- framework.
-
- Attributes:
- problem_statement (str): The task or query given to the LLM as
- input.
- llm_response (str): The response generated by the LLM.
- final_answer (Optional[str]): The reference solution, if
- available, used for supervised learning or evaluation.
- metadata (Dict[str, Any]): Additional metadata such as model
- parameters, prompt details, or response confidence scores.
- timestamp (datetime): The timestamp when the action was
- generated (UTC).
- """
-
- problem_statement: str = Field(description="Problem statement for the LLM")
- llm_response: str = Field(description="Generated response from the LLM")
- final_answer: Optional[str] = Field(
- None, description="Reference solution if available"
- )
- metadata: Dict[str, Any] = Field(
- default_factory=dict,
- description="Additional metadata about the generation",
- )
- timestamp: datetime = Field(
- default_factory=lambda: datetime.now(timezone.utc),
- description="When the response was generated (UTC)",
- )
-
-
-class Observation(BaseModel):
- r"""Environment observation.
-
- Attributes:
- question: The question posed to the LLM.
- context: Additional context for the question.
- metadata: Optional metadata about the observation.
- """
-
- question: str = Field(..., description="The question posed to the LLM")
- context: Dict[str, Any] = Field(
- default_factory=dict, description="Additional context for the question"
- )
- metadata: Optional[Dict[str, Any]] = Field(
- default=None, description="Optional metadata about the observation"
- )
-
-
-class StepResult(BaseModel):
- r"""Result of an environment step.
-
- Attributes:
- observation: The next observation.
- reward: Dictionary of reward scores for different aspects.
- done: Whether the episode is complete.
- info: Additional information about the step.
- """
-
- observation: Observation = Field(..., description="The next observation")
- reward: float = Field(..., description="Total reward of the action")
- rewards_dict: Dict[str, float] = Field(
- default_factory=dict,
- description="Dictionary of reward scores for different aspects",
- )
- done: bool = Field(..., description="Whether the episode is complete")
- info: Dict[str, Any] = Field(
- default_factory=dict,
- description="Additional information about the step",
- )
-
-
-class BaseEnvironment(ABC):
- r"""Base class for all RLVR training environments.
-
- An environment ties everything together. It:
- 1. Holds state and manages curriculum progression
- 2. Defines reward functions and hint generation
- 3. Manages dataset and task selection
- 4. Provides reset and step functions
- 5. Handles verifier setup and teardown
- 6. Enables proactive agent behavior
- 7. Supports practice environment creation
- 8. Facilitates chain-of-thought verification
-
- Key Features:
- - Curriculum learning with adaptive difficulty
- - Reward shaping based on solution quality
- - Hint generation from verified solutions
- - Task selection based on agent progress
- - Practice environment generation
- - Chain-of-thought validation
- """
-
- def __init__(
- self,
- dataset: StaticDataset,
- verifier: BaseVerifier,
- extractor: BaseExtractor,
- max_steps: Optional[int] = None,
- teacher_agent: Optional[ChatAgent] = None,
- curriculum_config: Optional[Dict[str, Any]] = None,
- practice_env_config: Optional[Dict[str, Any]] = None,
- **kwargs,
- ) -> None:
- r"""Initialize the environment.
-
- Args:
- dataset (StaticDataset): Dataset to sample questions from.
- verifier (BaseVerifier): Verifier to check responses.
- extractor (BaseExtractor): Extractor to process LLM responses.
- max_steps (Optional[int]): Maximum steps per episode. (default:
- :obj:`None`)
- teacher_agent (Optional[ChatAgent]): Optional agent for reward
- shaping and hints. (default: :obj:`None`)
- curriculum_config (Optional[Dict[str, Any]]): Configuration for
- curriculum learning including:
- - difficulty_levels: List of available difficulty levels
- - promotion_threshold: Score needed to advance
- - demotion_threshold: Score triggering level decrease
- - min_questions_per_level: Questions before promotion
- (default: :obj:`None`)
- practice_env_config (Optional[Dict[str, Any]]): Configuration for
- practice environments:
- - max_practice_envs: Maximum concurrent environments
- - difficulty_range: Allowed difficulty variation
- - focus_areas: Specific skills to practice
- (default: :obj:`None`)
- **kwargs: Additional environment parameters.
- """
- self.dataset = dataset
- self.verifier = verifier
- self.extractor = extractor
- self.max_steps = max_steps
- self.teacher_agent = teacher_agent
- self._metadata = kwargs
-
- # State tracking
- self._is_setup: bool = False
- self._current_step: int = 0
- self._episode_ended: bool = False
- self._state: Dict[str, Any] = self._get_initial_state()
- self._last_observation: Optional[Observation] = None
- self._episode_history: List[Tuple[Observation, Action]] = []
-
- @abstractmethod
- async def setup(self) -> None:
- r"""Set up the environment, including verifier initialization."""
- if self._is_setup:
- return
-
- try:
- # Initialize core components
- if hasattr(self.verifier, 'setup'):
- await self.verifier.setup()
- if hasattr(self.dataset, 'setup'):
- await self.dataset.setup()
- if hasattr(self.extractor, 'setup'):
- await self.extractor.setup()
-
- # initialize agents if present
- if self.teacher_agent:
- await self.teacher_agent.reset()
-
- self._is_setup = True
- logger.info('Environment setup completed successfully')
- except Exception as e:
- logger.error(f'Failed to setup environment: {e}')
- raise
-
- @abstractmethod
- async def teardown(self) -> None:
- r"""Clean up resources, including verifier teardown."""
- if not self._is_setup:
- return
-
- try:
- # Cleanup components
- if hasattr(self.verifier, 'cleanup'):
- await self.verifier.cleanup()
- if hasattr(self.dataset, 'cleanup'):
- await self.dataset.cleanup()
- if hasattr(self.extractor, 'cleanup'):
- await self.extractor.cleanup()
-
- self._is_setup = False
- logger.info('Environment teardown completed successfully')
- except Exception as e:
- logger.error(f'Failed to teardown environment: {e}')
- raise
-
- @abstractmethod
- async def reset(self) -> Observation:
- r"""Reset the environment to initial state.
-
- Returns:
- Initial observation for the episode
- """
-
- if not self._is_setup:
- await self.setup()
-
- # Reset state
- self._current_step = 0
- self._episode_ended = False
- self._episode_history = []
- self._state = self._get_initial_state()
-
- # Get initial observation
- observation = self._get_next_observation()
- if observation is None:
- raise RuntimeError("Failed to get initial observation")
-
- self._last_observation = observation
-
- return observation
-
- @abstractmethod
- async def step(self, action: Action) -> StepResult:
- r"""Take a step in the environment.
-
- Args:
- action: Action containing everything that is needed
- to progress in the environment
-
- Returns:
- StepResult containing next observation, reward, done flag, and info
- """
- if self.max_steps and self._current_step >= self.max_steps:
- return StepResult(
- observation=self._get_terminal_observation(),
- reward=0,
- rewards_dict={},
- done=True,
- info={"reason": "max_steps_reached"},
- )
-
- if not self._is_setup:
- raise RuntimeError("Environment not set up. Call setup() first.")
- if self._episode_ended:
- raise RuntimeError("Episode has ended. Call reset() first.")
- if self._last_observation is None:
- raise RuntimeError("No current observation. Call reset() first.")
-
- self._current_step += 1
-
- current_obs: Observation = self._last_observation
- self._episode_history.append((current_obs, action))
-
- # extract verifiable part from llm response
- extraction_result = await self.extractor.extract(action.llm_response)
-
- # Ensure extraction_result is a string
- if extraction_result is None:
- extraction_result = ""
-
- # verify the extracted
- verification_result = await self.verifier.verify(
- VerifierInput(
- llm_response=extraction_result,
- ground_truth=action.final_answer,
- )
- )
-
- # compute rewards
- total_reward, rewards_dict = await self.compute_reward(
- action, extraction_result, verification_result
- )
-
- # check termination
- done = self._is_done()
-
- next_obs = (
- self._get_terminal_observation()
- if done
- else self._get_next_observation()
- )
-
- self._last_observation = next_obs
- self._episode_ended = done
-
- return StepResult(
- observation=next_obs,
- reward=total_reward,
- rewards_dict=rewards_dict,
- done=done,
- info={
- "extraction_result": extraction_result,
- "verification_result": verification_result,
- "step": self._current_step,
- "state": self._state,
- },
- )
-
- @abstractmethod
- def _get_initial_state(self) -> Dict[str, Any]:
- r"""Get initial environment state."""
-
- return {
- "current_datapoint": None,
- "attempts": 0,
- "success_rate": 0.0,
- "rewards": [],
- "termination_reason": None,
- }
-
- @abstractmethod
- def _get_next_observation(self) -> Observation:
- r"""Get the next observation for the environment.
-
- Returns:
- Observation for the next step
- """
- if not self.dataset or len(self.dataset) == 0:
- logger.warning(
- "Dataset is empty. Attempting to generate new data..."
- )
- if isinstance(self.dataset, GenerativeDataset):
- try:
- asyncio.run(
- self.dataset.generate_new(1)
- ) # Generate at least one datapoint
- logger.info("Generated new datapoint successfully.")
- except Exception as e:
- logger.error(f"Failed to generate new data: {e}")
- return self._get_terminal_observation()
- else:
- logger.error("Dataset is empty and not a GenerativeDataset.")
- return self._get_terminal_observation()
-
- try:
- # Ensure dataset is not empty after generation attempt
- if len(self.dataset) == 0:
- logger.error("Dataset is still empty after generation.")
- return self._get_terminal_observation()
-
- # Sample the next datapoint
- datapoint_idx = self._current_step % len(self.dataset)
- datapoint = self.dataset[datapoint_idx]
-
- if not datapoint:
- logger.error(f"Invalid datapoint at index {datapoint_idx}")
- return self._get_terminal_observation()
-
- self._state["current_datapoint"] = datapoint
-
- # Extract necessary attributes safely
- question = getattr(datapoint, "question", None)
- final_answer = getattr(datapoint, "final_answer", None)
- rationale = getattr(datapoint, "rationale", None)
- metadata = getattr(datapoint, "metadata", {})
-
- if not question or not final_answer:
- logger.error(
- f"Datapoint at index {datapoint_idx} "
- "is missing required fields."
- )
- return self._get_terminal_observation()
-
- observation = Observation(
- question=question,
- context={
- "final_answer": final_answer,
- "rationale": rationale,
- },
- metadata={
- "step": self._current_step,
- "datapoint_id": str(datapoint_idx),
- "verified": metadata.get("verified", False),
- **metadata,
- },
- )
-
- logger.debug(
- f"Generated observation for step {self._current_step}"
- )
- return observation
-
- except (IndexError, AttributeError) as e:
- logger.error(f"Error getting next observation: {e}")
- return self._get_terminal_observation()
- except Exception as e:
- logger.error(f"Unexpected error getting next observation: {e}")
- return self._get_terminal_observation()
-
- @abstractmethod
- def _get_terminal_observation(self) -> Observation:
- r"""Get the terminal observation when episode ends.
-
- Returns:
- Terminal observation
- """
- return Observation(
- question="Episode completed",
- context={},
- metadata={"terminal": True, "final_step": self._current_step},
- )
-
- @abstractmethod
- async def compute_reward(
- self,
- action: Action,
- extraction_result: str,
- verification_result: VerificationResult,
- ) -> Tuple[float, Dict[str, float]]:
- r"""Compute reward scores for different aspects of the response.
-
- Args:
- response: The response.
- extraction_result: Extracted information from response
- verification_result: Result from the verifier.
-
- Returns:
- - Total reward
- - Dictionary of reward scores for different aspects.
- """
- rewards: Dict[str, float] = {}
-
- # Get success from verification result status
- verification_success = float(
- verification_result.status == VerificationOutcome.SUCCESS
- )
- rewards["correctness"] = 1.0 if verification_success > 0.5 else 0.0
-
- # Update state
- self._state["rewards"].append(rewards)
- total_attempts = self._state["attempts"] + 1
- self._state["success_rate"] = (
- self._state["success_rate"] * (total_attempts - 1)
- + verification_success
- ) / total_attempts
-
- further_rewards = await self._compute_reward(
- action, extraction_result, verification_result
- )
-
- rewards = rewards | further_rewards
-
- return sum(rewards.values()), rewards
-
- @abstractmethod
- async def _compute_reward(
- self,
- action: Action,
- extraction_result: str,
- verification_result: VerificationResult,
- ) -> Dict[str, float]:
- pass
-
- def _is_done(self) -> bool:
- r"""Check if episode should terminate."""
- if self.max_steps and self._current_step >= self.max_steps:
- return True
- return False
-
- @property
- def metadata(self) -> Dict[str, Any]:
- r"""Get environment metadata."""
- return self._metadata.copy()
-
- @property
- def current_step(self) -> int:
- r"""Get current step number."""
- return self._current_step
diff --git a/camel/environments/models.py b/camel/environments/models.py
new file mode 100644
index 000000000..3de9bc9b1
--- /dev/null
+++ b/camel/environments/models.py
@@ -0,0 +1,111 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+
+from datetime import datetime, timezone
+from typing import Any, Dict, Optional, Protocol
+
+from pydantic import BaseModel, Field
+
+
+class Action(BaseModel):
+ r"""Represents an action taken in an environment.
+
+ This class defines the input context, the LLM-generated output, and
+ metadata required for verification and tracking within an RL
+ framework.
+
+ Attributes:
+ llm_response (str): The response generated by the LLM.
+ metadata (Dict[str, Any]): Additional metadata such as model
+ parameters, prompt details, or response confidence scores.
+ timestamp (datetime): The timestamp when the action was
+ generated (UTC).
+ """
+
+ llm_response: str = Field(description="Generated response from the LLM")
+ metadata: Dict[str, Any] = Field(
+ default_factory=dict,
+ description="Additional metadata about the generation",
+ )
+ timestamp: datetime = Field(
+ default_factory=lambda: datetime.now(timezone.utc),
+ description="When the response was generated (UTC)",
+ )
+
+
+class Observation(BaseModel):
+ r"""Environment observation.
+
+ Attributes:
+ question: The question posed to the LLM.
+ context: Additional context for the question.
+ metadata: Optional metadata about the observation.
+ """
+
+ question: str = Field(..., description="The question posed to the LLM")
+ context: Dict[str, Any] = Field(
+ default_factory=dict, description="Additional context for the question"
+ )
+ metadata: Optional[Dict[str, Any]] = Field(
+ default=None, description="Optional metadata about the observation"
+ )
+
+
+class StepResult(BaseModel):
+ r"""Result of an environment step.
+
+ Attributes:
+ observation: The next observation.
+ reward: Dictionary of reward scores for different aspects.
+ done: Whether the episode is complete.
+ info: Additional information about the step.
+ """
+
+ observation: Observation = Field(..., description="The next observation")
+ reward: float = Field(..., description="Total reward of the action")
+ rewards_dict: Dict[str, float] = Field(
+ default_factory=dict,
+ description="Dictionary of reward scores for different aspects",
+ )
+ done: bool = Field(..., description="Whether the episode is complete")
+ info: Dict[str, Any] = Field(
+ default_factory=dict,
+ description="Additional information about the step",
+ )
+
+
+class Environment(Protocol):
+ async def reset(self) -> Observation:
+ r"""Reset the environment to an initial state.
+
+ Returns:
+ Initial observation for the episode
+ """
+ ...
+
+ async def step(self, action: Action) -> StepResult:
+ r"""Take a step in the environment.
+
+ Args:
+ action: Action containing everything that is needed
+ to progress in the environment
+
+ Returns:
+ StepResult containing next observation, reward, done flag, and info
+ """
+ ...
+
+ async def close(self) -> None:
+ r"""Perform a full cleanup of all environment resources."""
+ ...
diff --git a/camel/environments/multi_step.py b/camel/environments/multi_step.py
new file mode 100644
index 000000000..df0013954
--- /dev/null
+++ b/camel/environments/multi_step.py
@@ -0,0 +1,271 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+
+from abc import ABC, abstractmethod
+from typing import Any, Dict, List, Optional, Tuple
+
+from camel.extractors.base import BaseExtractor
+from camel.logger import get_logger
+
+from .models import Action, Observation, StepResult
+
+logger = get_logger(__name__)
+
+
+class MultiStepEnv(ABC):
+ r"""A multi-step environment for reinforcement learning with LLMs."""
+
+ def __init__(
+ self,
+ extractor: BaseExtractor,
+ max_steps: Optional[int] = None,
+ **kwargs,
+ ) -> None:
+ r"""Initialize the environment.
+
+ Args:
+ extractor: Extractor to process LLM responses.
+ max_steps: Maximum steps per episode.
+ **kwargs: Additional environment parameters.
+ """
+ self.extractor = extractor
+ self.max_steps = max_steps
+ self._metadata = kwargs
+
+ # State tracking
+ self._is_setup: bool = False
+ self._current_step: int = 0
+ self._episode_ended: bool = False
+ self._state: Dict[str, Any] = self._get_initial_state()
+ self._last_observation: Optional[Observation] = None
+ self._episode_history: List[Tuple[Observation, Action]] = []
+
+ async def setup(self) -> None:
+ r"""Set up the environment by initializing the verifier and extractor.
+
+ This method ensures that the environment is ready for interaction.
+ It sets up necessary components, including the verifier and extractor.
+
+ Raises:
+ Exception: If setup fails due to an internal error.
+ """
+
+ if self._is_setup:
+ return
+
+ try:
+ await self.extractor.setup()
+ await self._setup()
+ self._is_setup = True
+ logger.info('Environment setup completed successfully')
+ except Exception as e:
+ logger.error(f'Failed to setup environment: {e}')
+ raise
+
+ @abstractmethod
+ async def _setup(self) -> None:
+ pass
+
+ async def close(self) -> None:
+ r"""Clean up and close all resources used by the environment.
+ This method shuts down the verifier, calls the internal
+ close function that is implemented in any MultiStepEnv,
+ and ensures that the environment is properly closed.
+
+ Raises:
+ Exception: If an error occurs while closing the environment.
+ """
+ if not self._is_setup:
+ return
+
+ try:
+ await self.extractor.cleanup()
+
+ await self._close()
+
+ self._is_setup = False
+ logger.info('Environment teardown completed successfully')
+ except Exception as e:
+ logger.error(f'Failed to teardown environment: {e}')
+ raise
+
+ @abstractmethod
+ async def _close(self) -> None:
+ pass
+
+ async def reset(self) -> Observation:
+ r"""Reset the environment to an initial state.
+
+ Returns:
+ Observation: The initial observation for the episode.
+
+ Raises:
+ RuntimeError: If we fail to get the initial observation.
+ """
+
+ if not self._is_setup:
+ await self.setup()
+
+ # Reset state
+ self._current_step = 0
+ self._episode_ended = False
+ self._episode_history = []
+ self._state = self._get_initial_state()
+
+ # Get initial observation
+ observation = self._get_next_observation()
+ if observation is None:
+ raise RuntimeError("Failed to get initial observation")
+
+ self._last_observation = observation
+
+ return observation
+
+ async def step(self, action: Action) -> StepResult:
+ r"""Take a step in the environment using the given action.
+
+ This method updates the environment state based on the LLM's response,
+ computes rewards, checks if the episode is done, and based on that
+ gets the next or final observation.
+
+ Args:
+ action (Action): The action containing the LLM response.
+
+ Returns:
+ StepResult containing next observation, total reward, a dictionary
+ of rewards, done flag, and info.
+
+ Raises:
+ RuntimeError: If the environment is not set up, the episode has
+ ended, or there is no valid current observation.
+ """
+ if self.max_steps and self._current_step >= self.max_steps:
+ return StepResult(
+ observation=self._get_terminal_observation(),
+ reward=0,
+ rewards_dict={},
+ done=True,
+ info={"reason": "max_steps_reached"},
+ )
+
+ if not self._is_setup:
+ raise RuntimeError("Environment not set up. Call setup() first.")
+ if self._episode_ended:
+ raise RuntimeError("Episode has ended. Call reset() first.")
+ if self._last_observation is None:
+ raise RuntimeError("No current observation. Call reset() first.")
+
+ self._current_step += 1
+
+ current_obs: Observation = self._last_observation
+ self._episode_history.append((current_obs, action))
+
+ # Update the environment state based on the action
+ await self._update_state(action)
+
+ # Compute rewards
+ total_reward, rewards_dict = await self.compute_reward()
+
+ # Check termination
+ done = self.is_done()
+
+ # Get next observation based on the updated state
+ next_obs = (
+ self._get_terminal_observation()
+ if done
+ else self._get_next_observation()
+ )
+
+ self._last_observation = next_obs
+ self._episode_ended = done
+
+ return StepResult(
+ observation=next_obs,
+ reward=total_reward,
+ rewards_dict=rewards_dict,
+ done=done,
+ info={
+ "extraction_result": self.extractor.extract(
+ action.llm_response
+ ),
+ "step": self._current_step,
+ "state": self._state, # Updated state
+ },
+ )
+
+ @abstractmethod
+ def _get_initial_state(self) -> Dict[str, Any]:
+ pass
+
+ @abstractmethod
+ async def _update_state(self, action: Action) -> None:
+ pass
+
+ @abstractmethod
+ def _get_next_observation(self) -> Observation:
+ pass
+
+ @abstractmethod
+ def _get_terminal_observation(self) -> Observation:
+ pass
+
+ @abstractmethod
+ async def compute_reward(
+ self,
+ ) -> Tuple[float, Dict[str, float]]:
+ pass
+
+ def is_done(self) -> bool:
+ r"""Check if the episode should terminate.
+
+ This function terminates the episode if the maximum number of
+ steps is reached or if any other terminating criterion is met.
+
+ Returns:
+ bool: A boolean flag.
+ """
+
+ # After too many steps
+ if self.max_steps and self._current_step >= self.max_steps:
+ return True
+
+ # Further termination logic can be implemented in subclass
+ if self._is_done():
+ return True
+
+ return False
+
+ @abstractmethod
+ def _is_done(self) -> bool:
+ pass
+
+ @property
+ def metadata(self) -> Dict[str, Any]:
+ r"""Retrieve the metadata of the environment.
+
+ This provides additional parameters and configuration details.
+
+ Returns:
+ Dict[str, Any]: A copy of the environment's metadata.
+ """
+ return self._metadata.copy()
+
+ @property
+ def current_step(self) -> int:
+ r"""Get the current step number.
+
+ Returns:
+ int: The number of the step we are currently in.
+ """
+ return self._current_step
diff --git a/camel/environments/single_step.py b/camel/environments/single_step.py
new file mode 100644
index 000000000..6eadf5630
--- /dev/null
+++ b/camel/environments/single_step.py
@@ -0,0 +1,293 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+
+
+from abc import abstractmethod
+from typing import Any, Dict, Optional, Tuple
+
+from camel.datasets.base import DataPoint, GenerativeDataset, StaticDataset
+from camel.extractors.base import BaseExtractor
+from camel.logger import get_logger
+from camel.verifiers.base import (
+ BaseVerifier,
+ VerificationResult,
+)
+from camel.verifiers.models import (
+ VerifierInput,
+)
+
+from .models import Action, Observation, StepResult
+
+logger = get_logger(__name__)
+
+
+class SingleStepEnv:
+ r"""A single-step environment for reinforcement learning with LLMs.
+
+ Key Features:
+ - Samples questions from a dataset and asks the LLM
+ - Extracts verifiable information from model responses.
+ - Verifies extracted responses against ground truth.
+ - Computes and assigns rewards based on correctness.
+ - Supports async setup, teardown, and cleanup of resources.
+
+ This class is intended as a foundation for RL experiments involving
+ LLM-based policies, ensuring structured interactions between model
+ actions and verification mechanisms.
+ """
+
+ PLACEHOLDER_OBS = Observation(
+ question="Episode ended. This is just a placeholder."
+ )
+
+ ACCURACY_REWARD = 10
+
+ def __init__(
+ self,
+ dataset: StaticDataset | GenerativeDataset,
+ verifier: BaseVerifier,
+ extractor: BaseExtractor,
+ **kwargs,
+ ) -> None:
+ r"""Initialize the environment.
+
+ Args:
+ dataset: Dataset to sample questions from.
+ verifier: Verifier to check responses.
+ extractor: Extractor to process LLM responses.
+ **kwargs: Additional environment parameters.
+ """
+ self.dataset = dataset
+ self.verifier = verifier
+ self.extractor = extractor
+ self._metadata = kwargs
+
+ # State tracking
+ self._is_setup: bool = False
+ self._state: Optional[DataPoint] = None
+ self._episode_ended: bool = False
+
+ async def setup(self) -> None:
+ r"""Set up the environment by initializing the verifier and extractor.
+
+ This method ensures that the environment is ready for interaction.
+ It sets up necessary components, including the verifier and extractor.
+
+ Raises:
+ Exception: If setup fails due to an internal error.
+ """
+
+ if self._is_setup:
+ return
+
+ try:
+ await self.verifier.setup()
+ await self.extractor.setup()
+
+ self._is_setup = True
+ logger.info('Environment setup completed successfully')
+ except Exception as e:
+ logger.error(f'Failed to setup environment: {e}')
+ raise
+
+ async def close(self) -> None:
+ r"""Clean up and close all resources used by the environment.
+
+ This method shuts down the verifier and extractor, resets the internal
+ state, and ensures that the environment is properly closed.
+
+ Raises:
+ Exception: If an error occurs while closing the environment.
+ """
+
+ if not self._is_setup:
+ return
+
+ try:
+ self._is_setup = False
+ await self.verifier.cleanup()
+ await self.extractor.cleanup()
+ self._state = None
+ self._episode_ended = False
+ logger.info('Environment closed successfully')
+ except Exception as e:
+ logger.error(f'Failed to close environment: {e}')
+ raise
+
+ async def reset(self) -> Observation:
+ r"""Reset the environment and start a new episode.
+
+ This method samples a new data point from the dataset and returns the
+ initial observation.
+
+ Returns:
+ Observation: The first observation of the new episode, including
+ the question.
+
+ Raises:
+ Exception: If the environment is not set up properly.
+ """
+
+ if not self._is_setup:
+ await self.setup()
+
+ self._episode_ended = False
+
+ # Sample a datapoint
+
+ self._state = self.dataset.sample()
+
+ observation = Observation(
+ question=self._state.question, context={}, metadata={}
+ )
+
+ return observation
+
+ async def step(self, action: Action) -> StepResult:
+ r"""Take a step in the environment using the given action.
+
+ This method processes the LLM response, extracts verifiable content,
+ verifies correctness, computes rewards, and ends the episode.
+
+ Args:
+ action (Action): The action containing the LLM response to
+ evaluate.
+
+ Returns:
+ StepResult: Contains the next observation (placeholder), total
+ reward, reward breakdown, completion flag, and additional
+ information.
+
+ Raises:
+ RuntimeError: If the environment is not set up, the episode has
+ ended, or there is no valid current observation.
+ """
+
+ if not self._is_setup:
+ raise RuntimeError("Environment not set up. Call setup() first.")
+ if self._episode_ended:
+ raise RuntimeError("Episode has ended. Call reset() first.")
+ if self._state is None:
+ raise RuntimeError("No current observation. Call reset() first.")
+
+ # extract verifiable part from llm response
+ extraction_result = await self.extractor.extract(action.llm_response)
+
+ if not extraction_result:
+ raise RuntimeError(f"Couldn't extract from {action.llm_response}")
+
+ # verify the extracted
+ verification_result = await self.verifier.verify(
+ VerifierInput(
+ llm_response=extraction_result,
+ ground_truth=self._state.final_answer,
+ )
+ )
+
+ # compute rewards
+ total_reward, rewards_dict = await self._compute_reward(
+ action, extraction_result, verification_result
+ )
+
+ self._episode_ended = True
+
+ return StepResult(
+ observation=self.PLACEHOLDER_OBS,
+ reward=total_reward,
+ rewards_dict=rewards_dict,
+ done=True,
+ info={
+ "extraction_result": extraction_result,
+ "verification_result": verification_result,
+ "state": self._state,
+ },
+ )
+
+ async def _compute_reward(
+ self,
+ action: Action,
+ extraction_result: str,
+ verification_result: VerificationResult,
+ ) -> Tuple[float, Dict[str, float]]:
+ r"""Compute reward scores based on verification results.
+
+ This method calculates the reward based on correctness and any
+ additional custom reward components.
+
+ Args:
+ action (Action): The action taken in the environment.
+ extraction_result (str): The extracted verifiable content from the
+ LLM response.
+ verification_result (VerificationResult): The result of verifying
+ the extracted response.
+
+ Returns:
+ Tuple[float, Dict[str, float]]: A tuple containing:
+ - Total reward (float)
+ - Dictionary of individual reward components.
+
+ Raises:
+ Exception: If an error occurs while computing rewards.
+ """
+
+ rewards: Dict[str, float] = {}
+
+ rewards["correctness"] = (
+ self.ACCURACY_REWARD if verification_result.status else 0.0
+ )
+
+ further_rewards = await self._compute_custom_reward(
+ action, extraction_result, verification_result
+ )
+
+ rewards = rewards | further_rewards
+
+ return sum(rewards.values()), rewards
+
+ @abstractmethod
+ async def _compute_custom_reward(
+ self,
+ action: Action,
+ extraction_result: str,
+ verification_result: VerificationResult,
+ ) -> Dict[str, float]:
+ r"""Compute additional custom reward components.
+
+ This method should be implemented by subclasses to define
+ domain-specific reward calculations.
+
+ Args:
+ action (Action): The action taken in the environment.
+ extraction_result (str): The extracted verifiable content from the
+ LLM response.
+ verification_result (VerificationResult): The result of verifying
+ the extracted response.
+
+ Returns:
+ Dict[str, float]: A dictionary mapping custom reward categories
+ to their values.
+ """
+ pass
+
+ @property
+ def metadata(self) -> Dict[str, Any]:
+ r"""Retrieve the metadata of the environment.
+
+ This provides additional parameters and configuration details.
+
+ Returns:
+ Dict[str, Any]: A copy of the environment's metadata.
+ """
+
+ return self._metadata.copy()
diff --git a/test/environments/test_base_env.py b/test/environments/test_base_env.py
deleted file mode 100644
index 9ac90ab41..000000000
--- a/test/environments/test_base_env.py
+++ /dev/null
@@ -1,288 +0,0 @@
-# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
-
-from datetime import datetime, timezone
-from typing import Dict, Tuple
-from unittest.mock import AsyncMock, MagicMock
-
-import pytest
-
-from camel.environments.base import (
- Action,
- BaseEnvironment,
- Observation,
- StepResult,
-)
-from camel.verifiers.models import VerificationOutcome, VerificationResult
-
-
-def test_action_model():
- r"""Test the Action model initialization and properties."""
- custom_time = datetime(2023, 1, 1, tzinfo=timezone.utc)
- action_full = Action(
- problem_statement="Full problem",
- llm_response="Full response",
- final_answer="Reference solution",
- metadata={"model": "test-model", "temperature": 0.7},
- timestamp=custom_time,
- )
- assert action_full.problem_statement == "Full problem"
- assert action_full.llm_response == "Full response"
- assert action_full.final_answer == "Reference solution"
- assert action_full.metadata == {"model": "test-model", "temperature": 0.7}
- assert action_full.timestamp == custom_time
-
-
-def test_observation_model():
- r"""Test the Observation model initialization and properties."""
- obs_full = Observation(
- question="Complex math problem",
- context={"hint": "Use calculus"},
- metadata={"difficulty": "hard"},
- )
- assert obs_full.question == "Complex math problem"
- assert obs_full.context == {"hint": "Use calculus"}
- assert obs_full.metadata == {"difficulty": "hard"}
-
-
-def test_step_result_model():
- r"""Test the StepResult model initialization and properties."""
- obs = Observation(question="Next question")
- result_full = StepResult(
- observation=obs,
- reward=1.0,
- rewards_dict={"correctness": 0.8, "efficiency": 0.2},
- done=True,
- info={"reason": "task_complete"},
- )
- assert result_full.observation == obs
- assert result_full.reward == 1.0
- assert result_full.rewards_dict == {"correctness": 0.8, "efficiency": 0.2}
- assert result_full.done is True
- assert result_full.info == {"reason": "task_complete"}
-
-
-class MockBaseEnvironment(BaseEnvironment):
- r"""Mock implementation of BaseEnvironment for testing."""
-
- async def setup(self):
- await super().setup()
-
- async def teardown(self):
- await super().teardown()
-
- async def reset(self):
- return await super().reset()
-
- async def step(self, action):
- return await super().step(action)
-
- def _get_initial_state(self):
- return super()._get_initial_state()
-
- def _get_next_observation(self):
- return Observation(question="Test question")
-
- def _get_terminal_observation(self):
- return super()._get_terminal_observation()
-
- async def compute_reward(
- self,
- action: Action,
- extraction_result: str,
- verification_result: VerificationResult,
- ) -> Tuple[float, Dict[str, float]]:
- return await super().compute_reward(
- action, extraction_result, verification_result
- )
-
- async def _compute_reward(
- self, action, extraction_result, verification_result
- ):
- return {"additional_reward": 0.5}
-
-
-async def test_base_environment_lifecycle():
- r"""Test the lifecycle methods of BaseEnvironment."""
- mock_dataset = MagicMock()
- mock_dataset.__len__.return_value = 1
- mock_dataset.__getitem__.return_value = MagicMock(
- question="Test question",
- final_answer="Test answer",
- rationale="Test rationale",
- difficulty="medium",
- metadata={"verified": True},
- )
- mock_dataset.setup = AsyncMock()
- mock_dataset.cleanup = AsyncMock()
-
- mock_verifier = MagicMock()
- mock_verifier.setup = AsyncMock()
- mock_verifier.cleanup = AsyncMock()
- mock_verifier.verify = AsyncMock(
- return_value=VerificationResult(
- status=VerificationOutcome.SUCCESS,
- result="Verification successful",
- feedback="Correct",
- score=1.0,
- )
- )
-
- mock_extractor = MagicMock()
- mock_extractor.setup = AsyncMock()
- mock_extractor.cleanup = AsyncMock()
- mock_extractor.extract = AsyncMock(return_value={"extracted": "content"})
-
- # Create environment
- env = MockBaseEnvironment(
- dataset=mock_dataset,
- verifier=mock_verifier,
- extractor=mock_extractor,
- max_steps=5,
- )
-
- # Test setup
- await env.setup()
- assert env._is_setup is True
- mock_verifier.setup.assert_awaited_once()
-
- # Test reset
- observation = await env.reset()
- assert isinstance(observation, Observation)
- assert env._current_step == 0
- assert env._episode_ended is False
-
- original_step = env.step
-
- async def mock_step(action):
- env._current_step += 1
- return StepResult(
- observation=env._get_next_observation(),
- reward=1.0,
- rewards_dict={"correctness": 1.0, "additional_reward": 0.5},
- done=False,
- info={
- "extraction_result": {"extracted": "content"},
- "verification_result": VerificationResult(
- status=VerificationOutcome.SUCCESS,
- result="Verification successful",
- feedback="Correct",
- score=1.0,
- ),
- "step": env._current_step,
- },
- )
-
- env.step = mock_step
-
- # Test step
- action = Action(
- problem_statement="Test problem",
- llm_response="Test response",
- )
- result = await env.step(action)
- assert isinstance(result, StepResult)
- assert env._current_step == 1
-
- # Restore the original step method
- env.step = original_step
-
- # Test teardown
- await env.teardown()
- assert env._is_setup is False
- mock_verifier.cleanup.assert_awaited_once()
-
-
-async def test_environment_reward_calculation():
- r"""Test the reward calculation in BaseEnvironment."""
- env = MockBaseEnvironment(
- dataset=MagicMock(),
- verifier=MagicMock(),
- extractor=MagicMock(),
- )
-
- # Test compute_reward method
- action = Action(
- problem_statement="Test problem",
- llm_response="Test response",
- )
- extraction_result = {"extracted": "content"}
- verification_result = VerificationResult(
- status=VerificationOutcome.SUCCESS,
- result="Verification successful",
- feedback="Correct",
- score=1.0,
- )
-
- total_reward, rewards_dict = await env.compute_reward(
- action, extraction_result, verification_result
- )
-
- # Check rewards
- assert "correctness" in rewards_dict
- assert rewards_dict["correctness"] == 1.0
- assert "additional_reward" in rewards_dict
- assert rewards_dict["additional_reward"] == 0.5
- assert total_reward == 1.5 # Sum of all rewards
-
-
-async def test_environment_max_steps():
- r"""Test that environment terminates after max_steps."""
- env = MockBaseEnvironment(
- dataset=MagicMock(),
- verifier=MagicMock(),
- extractor=MagicMock(),
- max_steps=2,
- )
- env._is_setup = True
-
- # Take first step
- action = Action(
- problem_statement="Test problem",
- llm_response="Test response",
- )
- env._last_observation = Observation(question="Test question")
-
- original_step = env.step
-
- async def mock_step(action):
- env._current_step += 1
- done = env._current_step >= env.max_steps
- return StepResult(
- observation=env._get_terminal_observation()
- if done
- else env._get_next_observation(),
- reward=1.0,
- rewards_dict={"correctness": 1.0},
- done=done,
- info={"reason": "max_steps_reached"} if done else {},
- )
-
- # Replace the step method
- env.step = mock_step
-
- # First step should not terminate
- result1 = await env.step(action)
- assert result1.done is False
-
- # Second step should terminate due to max_steps
- result2 = await env.step(action)
- assert result2.done is True
- assert "max_steps_reached" in result2.info.get("reason", "")
-
- # Restore the original step method
- env.step = original_step
diff --git a/test/environments/test_multi_step_env.py b/test/environments/test_multi_step_env.py
new file mode 100644
index 000000000..81911b922
--- /dev/null
+++ b/test/environments/test_multi_step_env.py
@@ -0,0 +1,170 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+import re
+from typing import Any, Dict, Tuple
+from unittest.mock import AsyncMock, MagicMock
+
+import pytest
+
+from camel.environments import (
+ Action,
+ Observation,
+ StepResult,
+)
+from camel.environments.multi_step import MultiStepEnv
+
+
+class MockMultiStepEnv(MultiStepEnv):
+ def _get_initial_state(self) -> Dict[str, Any]:
+ """Return the initial state with a step counter."""
+ return {"step": 0}
+
+ async def _update_state(self, action: Action) -> None:
+ """Increment the step counter based on the action."""
+ self._state["step"] += 1
+
+ def _get_next_observation(self) -> Observation:
+ """Return an observation reflecting the current step."""
+ return Observation(question=f"Step {self._state['step']}")
+
+ def _get_terminal_observation(self) -> Observation:
+ """Return the terminal observation when the episode ends."""
+ return Observation(question="Episode ended")
+
+ async def compute_reward(self) -> Tuple[float, Dict[str, float]]:
+ """Return a fixed reward and reward breakdown."""
+ return 1.0, {"step_reward": 1.0}
+
+ def _is_done(self) -> bool:
+ """End the episode after 2 steps."""
+ return self._state["step"] >= 2
+
+ async def _setup(self) -> None:
+ """Perform any setup tasks (empty for this mock)."""
+ pass
+
+ async def _close(self) -> None:
+ """Perform any cleanup tasks (empty for this mock)."""
+ pass
+
+
+async def test_multi_step_env_lifecycle():
+ # Create a mock extractor
+ mock_extractor = MagicMock()
+ mock_extractor.setup = AsyncMock()
+ mock_extractor.cleanup = AsyncMock()
+ mock_extractor.extract = MagicMock(return_value="extracted_answer")
+
+ env = MockMultiStepEnv(extractor=mock_extractor, max_steps=3)
+ await env.setup()
+ assert env._is_setup is True
+ mock_extractor.setup.assert_awaited_once()
+
+ observation = await env.reset()
+ assert isinstance(observation, Observation)
+ assert observation.question == "Step 0"
+
+ # Test first step
+ action1 = Action(llm_response="Response 1")
+ result1 = await env.step(action1)
+ assert isinstance(result1, StepResult)
+ assert result1.reward == 1.0
+ assert result1.done is False
+ assert result1.observation.question == "Step 1"
+ assert env._current_step == 1
+ assert env._state["step"] == 1
+ mock_extractor.extract.assert_called_once_with("Response 1")
+
+ # Test second step
+ action2 = Action(llm_response="Response 2")
+ result2 = await env.step(action2)
+ assert isinstance(result2, StepResult)
+ assert result2.reward == 1.0
+ assert result2.done is True
+ assert result2.observation.question == "Episode ended"
+ assert env._current_step == 2
+ assert env._state["step"] == 2
+ assert mock_extractor.extract.call_count == 2
+ mock_extractor.extract.assert_called_with("Response 2")
+
+ await env.close()
+ assert env._is_setup is False
+ mock_extractor.cleanup.assert_awaited_once()
+
+
+async def test_multi_step_env_error_handling():
+ mock_extractor = MagicMock()
+ mock_extractor.setup = AsyncMock()
+ mock_extractor.cleanup = AsyncMock()
+ mock_extractor.extract = MagicMock(return_value="extracted_answer")
+
+ env = MockMultiStepEnv(extractor=mock_extractor, max_steps=3)
+ action = Action(llm_response="Test response")
+
+ # **Test Case 1: Environment not set up**
+ # Attempt to step without setup, expect RuntimeError
+ with pytest.raises(
+ RuntimeError,
+ match=re.escape("Environment not set up. Call setup() first."),
+ ):
+ await env.step(action)
+
+ # **Test Case 2: Episode has ended**
+ # Setup and reset, then mark episode as ended
+ await env.setup()
+ await env.reset()
+ env._episode_ended = True
+ with pytest.raises(
+ RuntimeError, match=re.escape("Episode has ended. Call reset() first.")
+ ):
+ await env.step(action)
+
+ # **Test Case 3: No current observation**
+ # Reset to initialize, then clear last observation
+ await env.reset()
+ env._last_observation = None
+ with pytest.raises(
+ RuntimeError,
+ match=re.escape("No current observation. Call reset() first."),
+ ):
+ await env.step(action)
+
+ # **Test Case 4: Extractor failure**
+ # Configure extractor to raise an exception
+ mock_extractor.extract = MagicMock(
+ side_effect=Exception("Extractor error")
+ )
+ await env.reset()
+ with pytest.raises(Exception, match="Extractor error"):
+ await env.step(action)
+
+ # **Test Case 5: Maximum steps reached**
+ # Create a new environment with max_steps=1
+ # Reset extractor for Test Case 5
+ mock_extractor.extract = MagicMock(return_value="extracted_answer")
+ env_max_steps = MockMultiStepEnv(extractor=mock_extractor, max_steps=1)
+ await env_max_steps.setup()
+ await env_max_steps.reset()
+ # First step should end the episode due to max_steps
+ result1 = await env_max_steps.step(action)
+ assert (
+ result1.done is True
+ ), "Episode should be done after reaching max_steps"
+ assert env_max_steps._current_step == 1, "Current step should be 1"
+ assert env_max_steps.max_steps == 1, "Max steps should be 1"
+ assert (
+ result1.observation.question == "Episode ended"
+ ), "Should reach terminal observation"
diff --git a/test/environments/test_single_stage_env.py b/test/environments/test_single_stage_env.py
new file mode 100644
index 000000000..047e9af27
--- /dev/null
+++ b/test/environments/test_single_stage_env.py
@@ -0,0 +1,259 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+import re
+from typing import Dict
+from unittest.mock import AsyncMock, MagicMock
+
+import pytest
+
+from camel.datasets import StaticDataset
+from camel.environments import (
+ Action,
+ Observation,
+ StepResult,
+)
+from camel.environments.single_step import SingleStepEnv
+from camel.verifiers.models import VerificationOutcome, VerificationResult
+
+
+class MockSingleStepEnv(SingleStepEnv):
+ async def _compute_custom_reward(
+ self,
+ action: Action,
+ extraction_result: str,
+ verification_result: VerificationResult,
+ ) -> Dict[str, float]:
+ return {"custom_reward": 0.5}
+
+
+async def test_single_step_env_lifecycle():
+ # Define dataset
+ data = [
+ {
+ "question": "What is 2 + 2?",
+ "final_answer": "4",
+ "rationale": "Adding 2 and 2 gives 4.",
+ "metadata": {"difficulty": "easy"},
+ },
+ {
+ "question": "What is the capital of France?",
+ "final_answer": "Paris",
+ "rationale": "Paris is known as the capital city of France.",
+ "metadata": {"difficulty": "easy"},
+ },
+ ]
+ dataset = StaticDataset(data)
+
+ # Define mock verifier
+ mock_verifier = MagicMock()
+ mock_verifier.setup = AsyncMock()
+ mock_verifier.cleanup = AsyncMock()
+ mock_verifier.verify = AsyncMock(
+ return_value=VerificationResult(
+ status=VerificationOutcome.SUCCESS,
+ result="Verification successful",
+ feedback="Correct",
+ score=1.0,
+ )
+ )
+
+ # Define mock extractor
+ mock_extractor = MagicMock()
+ mock_extractor.setup = AsyncMock()
+ mock_extractor.cleanup = AsyncMock()
+ mock_extractor.extract = AsyncMock(return_value="extracted_answer")
+
+ # Initialize the environment with mocked dependencies
+ env = MockSingleStepEnv(
+ dataset=dataset,
+ verifier=mock_verifier,
+ extractor=mock_extractor,
+ )
+
+ # Test setup
+ await env.setup()
+ assert env._is_setup is True
+ mock_verifier.setup.assert_awaited_once()
+ mock_extractor.setup.assert_awaited_once()
+
+ # Test reset
+ observation = await env.reset()
+ assert isinstance(observation, Observation)
+ assert observation.question in [
+ "What is 2 + 2?",
+ "What is the capital of France?",
+ ]
+
+ # Test step
+ action = Action(llm_response="Test response")
+ result = await env.step(action)
+ assert isinstance(result, StepResult)
+ assert result.reward == 10.5 # correctness: 10.0 + custom_reward: 0.5
+ assert result.done is True
+ assert result.observation == SingleStepEnv.PLACEHOLDER_OBS
+ mock_extractor.extract.assert_awaited_once_with("Test response")
+ mock_verifier.verify.assert_awaited_once()
+
+ # Test close
+ await env.close()
+ assert env._is_setup is False
+ mock_verifier.cleanup.assert_awaited_once()
+ mock_extractor.cleanup.assert_awaited_once()
+
+
+async def test_single_step_env_error_handling():
+ # **1. Test Faulty Dataset**
+ faulty_data = [
+ {
+ "question": "What is 2 + 2?",
+ # Missing "final_answer"
+ "rationale": "Adding 2 and 2 gives 4.",
+ "metadata": {"difficulty": "easy"},
+ },
+ {
+ "question": "What is the capital of France?",
+ "final_answer": "Paris",
+ "rationale": "Paris is known as the capital city of France.",
+ "metadata": {"difficulty": "easy"},
+ },
+ ]
+ try:
+ StaticDataset(faulty_data)
+ except Exception as e:
+ assert isinstance(
+ e, ValueError
+ ), "Expected ValueError for faulty dataset"
+
+ # **Valid Dataset for Subsequent Tests**
+ valid_data = [
+ {
+ "question": "What is 2 + 2?",
+ "final_answer": "4",
+ "rationale": "Adding 2 and 2 gives 4.",
+ "metadata": {"difficulty": "easy"},
+ },
+ {
+ "question": "What is the capital of France?",
+ "final_answer": "Paris",
+ "rationale": "Paris is known as the capital city of France.",
+ "metadata": {"difficulty": "easy"},
+ },
+ ]
+ dataset = StaticDataset(valid_data)
+
+ # **2. Test Faulty Extractor**
+ # Mock verifier that fails verification
+ mock_verifier_fail = MagicMock()
+ mock_verifier_fail.setup = AsyncMock()
+ mock_verifier_fail.cleanup = AsyncMock()
+ mock_verifier_fail.verify = AsyncMock(
+ return_value=VerificationResult(
+ status=VerificationOutcome.FAILURE,
+ result="Verification failed",
+ feedback="Incorrect",
+ score=0.0,
+ )
+ )
+ # Mock extractor that returns None (extraction failure)
+ mock_extractor_fail = MagicMock()
+ mock_extractor_fail.setup = AsyncMock()
+ mock_extractor_fail.cleanup = AsyncMock()
+ mock_extractor_fail.extract = AsyncMock(return_value=None)
+
+ env_fail_extractor = MockSingleStepEnv(
+ dataset=dataset,
+ verifier=mock_verifier_fail,
+ extractor=mock_extractor_fail,
+ )
+ await env_fail_extractor.setup()
+ observation = await env_fail_extractor.reset()
+ assert isinstance(
+ observation, Observation
+ ), "Reset should still return an Observation"
+
+ action = Action(llm_response="Test response")
+ with pytest.raises(RuntimeError, match="Couldn't extract from"):
+ await env_fail_extractor.step(action)
+
+ # **3. Test Faulty Verifier**
+ # Mock verifier that raises an exception
+ mock_verifier_exception = MagicMock()
+ mock_verifier_exception.setup = AsyncMock()
+ mock_verifier_exception.cleanup = AsyncMock()
+ mock_verifier_exception.verify = AsyncMock(
+ side_effect=Exception("Verifier error")
+ )
+ # Valid extractor
+ mock_extractor = MagicMock()
+ mock_extractor.setup = AsyncMock()
+ mock_extractor.cleanup = AsyncMock()
+ mock_extractor.extract = AsyncMock(return_value="extracted_answer")
+
+ env_fail_verifier = MockSingleStepEnv(
+ dataset=dataset,
+ verifier=mock_verifier_exception,
+ extractor=mock_extractor,
+ )
+ await env_fail_verifier.setup()
+ observation = await env_fail_verifier.reset()
+ assert isinstance(
+ observation, Observation
+ ), "Reset should still return an Observation"
+
+ with pytest.raises(Exception, match="Verifier error"):
+ await env_fail_verifier.step(action)
+
+ # **4. Test State Mismanagement Scenarios**
+ # a) Environment not set up
+ env_not_setup = MockSingleStepEnv(
+ dataset=dataset,
+ verifier=mock_verifier_fail,
+ extractor=mock_extractor,
+ )
+ with pytest.raises(
+ RuntimeError,
+ match=re.escape("Environment not set up. Call setup() first."),
+ ):
+ await env_not_setup.step(action)
+
+ # b) Episode ended
+ env_episode_ended = MockSingleStepEnv(
+ dataset=dataset,
+ verifier=mock_verifier_fail,
+ extractor=mock_extractor,
+ )
+ await env_episode_ended.setup()
+ await env_episode_ended.reset()
+ env_episode_ended._episode_ended = True
+ with pytest.raises(
+ RuntimeError,
+ match=re.escape("Episode has ended. " "Call reset() first."),
+ ):
+ await env_episode_ended.step(action)
+
+ # c) No current observation
+ env_no_state = MockSingleStepEnv(
+ dataset=dataset,
+ verifier=mock_verifier_fail,
+ extractor=mock_extractor,
+ )
+ await env_no_state.setup()
+ env_no_state._state = None
+ with pytest.raises(
+ RuntimeError,
+ match=re.escape("No current observation. Call reset() first."),
+ ):
+ await env_no_state.step(action)