🐫 CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org
Python agentai-societiesartificial-intelligencecommunicative-aicooperative-aideep-learninglarge-language-modelsmulti-agent-systemsnatural-language-processing
refactor: Clean up SingleStepEnv (#1985)
Apokryphosxd58bdab28eea
diff --git a/camel/environments/models.py b/camel/environments/models.pyindex fe99904a2..8039dfa0d 100644--- a/camel/environments/models.py+++ b/camel/environments/models.py@@ -13,7 +13,7 @@# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========from datetime import datetime, timezone-from typing import Any, Dict, Optional, Protocol+from typing import Any, Dict, Optional, Protocol, Tuplefrom pydantic import BaseModel, Field@@ -33,9 +33,8 @@ class Action(BaseModel):generated (UTC)."""- index: int = Field(- ..., description="Index of the state this action is performed upon"- )+ index: Optional[int] = Field(default=None, description="...")+llm_response: str = Field(description="Generated response from the LLM")metadata: Dict[str, Any] = Field(default_factory=dict,@@ -87,6 +86,13 @@ class StepResult(BaseModel):description="Additional information about the step",)+ def as_tuple(+ self,+ ) -> Tuple[Observation, float, bool, Dict[str, Any]]:+ r"""Returns all fields of the model as a tuple, in declaration order"""+ self.info["rewards_dict"] = self.rewards_dict+ return (self.observation, self.reward, self.done, self.info)+class Environment(Protocol):async def reset(self) -> Observation:diff --git a/camel/environments/single_step.py b/camel/environments/single_step.pyindex 97931e45b..b51d8bbc8 100644--- a/camel/environments/single_step.py+++ b/camel/environments/single_step.py@@ -126,6 +126,7 @@ class SingleStepEnv:await self.verifier.cleanup()self._states = []self._states_done = []+ self.current_batch_size = 0logger.info('Environment closed successfully')except Exception as e:logger.error(f'Failed to close environment: {e}')@@ -157,6 +158,8 @@ class SingleStepEnv:ValueError: If batch size exceeds dataset size.TypeError: If the dataset is of an unsupported type."""+ if batch_size <= 0:+ raise ValueError("Batch size must be positive")if not self._is_setup:logger.warning(@@ -212,16 +215,18 @@ class SingleStepEnv:async def step(self, action: Union[Action, List[Action]]- ) -> Union[StepResult, List[StepResult]]:- r"""Process actions for a subset of states and update their- finished status.+ ) -> Union[+ Tuple[Observation, float, bool, Dict[str, Any]],+ List[Tuple[Observation, float, bool, Dict[str, Any]]],+ ]:+ r"""Process actions for a subset of states and update their finished+ status.Args:- action: Single action or list of actions, where each action- contains an index indicating which state it corresponds to.- The index must be a valid position in the internal _states list- that was populated during the reset() call.-+ action: Single action (for batch_size=1 or micro-batch of size 1)+ or list of actions (for batch_size>=2 with multiple actions).+ Each action must have an index for batch_size>=2, indicating+ which state it corresponds to.Returns:Union[StepResult, List[StepResult]]: StepResult or list of@@ -241,10 +246,62 @@ class SingleStepEnv:if not self._states:raise RuntimeError("No current observation. Call reset() first.")- # Normalize everything to list- actions = [action] if isinstance(action, Action) else action- indices = [act.index for act in actions]+ # Normalize actions into a list for uniform processing+ if self.current_batch_size == 1:+ if isinstance(action, list):+ if len(action) != 1 or not isinstance(action[0], Action):+ raise ValueError(+ "For batch_size=1, expect a single Action or a "+ "list containing exactly one Action"+ )+ elif not isinstance(action, Action):+ raise ValueError(+ "For batch_size=1, expect a single Action or a "+ "list containing exactly one Action"+ )+ if isinstance(action, Action):+ actions = [action]+ else:+ actions = action+ if actions[0].index is None:+ actions[0].index = 0+ if actions[0].index != 0:+ raise ValueError("For batch_size=1, index must be None or 0")++ else: # batch_size >= 2+ if isinstance(action, Action):+ if action.index is None:+ raise ValueError(+ "For batch_size>=2, each Action must have an index"+ )+ if not isinstance(action.index, int):+ raise ValueError("Index must be an integer")+ actions = [action]+ elif isinstance(action, list):+ if not action: # Empty list+ raise ValueError("Action list cannot be empty")+ actions = action+ for act in actions:+ if not isinstance(act, Action):+ raise ValueError(+ "All elements in list must be Action objects"+ )+ if act.index is None:+ raise ValueError(+ "For batch_size>=2, each Action must have an index"+ )+ if not isinstance(act.index, int):+ raise ValueError("Index must be an integer")+ else:+ raise ValueError(+ "For batch_size>=2, expect an Action or list of Actions"+ )+ # Validate indices+ indices: List[int] = []+ for act in actions:+ assert act.index is not None+ indices.append(act.index)if len(set(indices)) != len(indices):raise ValueError("Duplicate state indices in actions.")for idx in indices:@@ -254,7 +311,6 @@ class SingleStepEnv:raise ValueError(f"State at index {idx} is already finished.")num_actions = len(actions)-if self.current_batch_size % num_actions != 0:logger.warning(f"Number of actions ({num_actions}) is not a divisor of "@@ -262,9 +318,9 @@ class SingleStepEnv:)proposed_solutions = [act.llm_response for act in actions]- ground_truths: List[str] = [- self._states[idx].final_answer for idx in indices- ]+ ground_truths: List[str] = []+ for idx in indices:+ ground_truths.append(self._states[idx].final_answer)verification_results = await self.verifier.verify_batch(solutions=proposed_solutions,@@ -276,9 +332,10 @@ class SingleStepEnv:proposed_solutions, verification_results)+ # TODO Batch thisstep_results = []- # TODO: batch thisfor i, action in enumerate(actions):+ assert action.index is not Noneidx = action.indexstep_result = StepResult(observation=self.PLACEHOLDER_OBS,@@ -291,7 +348,7 @@ class SingleStepEnv:"state": self._states[idx],},)- step_results.append(step_result)+ step_results.append(step_result.as_tuple())self._states_done[idx] = Truereturn step_results[0] if len(step_results) == 1 else step_results@@ -315,6 +372,12 @@ class SingleStepEnv:- List of total rewards for each solution.- List of reward component dictionaries for each solution."""+ if len(proposed_solutions) != len(verification_results):+ raise ValueError(+ f"Length mismatch: {len(proposed_solutions)} solutions vs "+ f"{len(verification_results)} verification results"+ )+total_rewards = []rewards_dicts = []@@ -355,9 +418,20 @@ class SingleStepEnv:return {}def _batch_done(self) -> bool:+ r"""Check if all states in the current batch are done.++ Returns:+ bool: True if all states are marked as done, False otherwise.+ """return all(self._states_done)def _batch_started(self) -> bool:+ r"""Check if any state in the current batch is done.++ Returns:+ bool: True if at least one state is marked as done, False+ otherwise.+ """return any(self._states_done)@propertydiff --git a/test/environments/test_single_stage_env.py b/test/environments/test_single_stage_env.pyindex d67831284..319696cf9 100644--- a/test/environments/test_single_stage_env.py+++ b/test/environments/test_single_stage_env.py@@ -21,7 +21,6 @@ from camel.datasets import StaticDatasetfrom camel.environments import (Action,Observation,- StepResult,)from camel.environments.single_step import SingleStepEnvfrom camel.verifiers.models import VerificationOutcome, VerificationResult@@ -37,7 +36,99 @@ class MockSingleStepEnv(SingleStepEnv):@pytest.mark.asyncio-async def test_single_step_env_lifecycle():+async def test_single_step_env_lifecycle_single():+ # Define a sample dataset+ data = [+ {+ "question": "What is 2 + 2?",+ "final_answer": "4",+ "rationale": "Adding 2 and 2 gives 4.",+ "metadata": {"difficulty": "easy"},+ },+ {+ "question": "What is the capital of France?",+ "final_answer": "Paris",+ "rationale": "Paris is known as the capital city of France.",+ "metadata": {"difficulty": "easy"},+ },+ ]+ dataset = StaticDataset(data)++ # Create a mock verifier+ mock_verifier = MagicMock()+ mock_verifier.setup = AsyncMock()+ mock_verifier.cleanup = AsyncMock()+ mock_verifier.verify_batch = AsyncMock(+ side_effect=lambda solutions, ground_truths, **kwargs: [+ VerificationResult(+ status=VerificationOutcome.SUCCESS,+ result="Verification successful",+ feedback="Correct",+ score=1.0,+ )+ for _ in solutions+ ]+ )++ # Initialize the environment+ env = MockSingleStepEnv(dataset=dataset, verifier=mock_verifier)++ # Test setup+ await env.setup()+ assert env._is_setup is True+ mock_verifier.setup.assert_awaited_once()++ # Test reset with batch_size=1+ observation = await env.reset(batch_size=1)+ assert isinstance(+ observation, Observation+ ) # Single Observation, not a list+ assert observation.question in [dp["question"] for dp in data]++ # Test step with a single action (index=0)+ action = Action(index=0, llm_response="4") # Assuming first question+ result = await env.step(action)+ assert isinstance(result, tuple) # Single result tuple, not a list+ next_obs, reward, done, info = result+ assert next_obs == env.PLACEHOLDER_OBS+ assert reward == env.ACCURACY_REWARD + 5 # Accuracy + custom reward+ assert done is True # Single step should end the episode+ assert isinstance(info, dict)+ assert info["rewards_dict"].get("custom_reward", None) == 5++ # Test that stepping again without reset fails+ with pytest.raises(+ RuntimeError,+ match=re.escape("Episodes have ended for batch. Call reset() first."),+ ):+ await env.step(action)++ # Test reset and step again with index=None (should default to 0)+ observation2 = await env.reset(batch_size=1)+ assert isinstance(observation2, Observation)+ action2 = Action(+ index=None, llm_response="Paris"+ ) # Assuming second question+ result2 = await env.step(action2)+ next_obs2, reward2, done2, info2 = result2+ assert next_obs2 == env.PLACEHOLDER_OBS+ assert reward2 == env.ACCURACY_REWARD + 5+ assert done2 is True+ assert info2["rewards_dict"].get("custom_reward", None) == 5++ # Test deterministic sampling with seed+ obs1 = await env.reset(batch_size=1, seed=42)+ obs2 = await env.reset(batch_size=1, seed=42)+ assert obs1.question == obs2.question # Same seed, same question++ # Test close+ await env.close()+ assert env._is_setup is False+ mock_verifier.cleanup.assert_awaited_once()+++async def test_batched_single_step_env_lifecycle():data = [{"question": "What is 2 + 2?",@@ -134,11 +225,12 @@ async def test_single_step_env_lifecycle():results = await env.step(actions)assert isinstance(results, list)assert len(results) == 2- for result in results:- assert isinstance(result, StepResult)- assert result.reward == 15- assert result.done is True- assert result.observation == MockSingleStepEnv.PLACEHOLDER_OBS+ for next_obs, reward, done, info in results:+ assert next_obs == env.PLACEHOLDER_OBS+ assert reward == env.ACCURACY_REWARD + 5 # 5 is custom reward+ assert done+ assert isinstance(info, dict)+ assert info["rewards_dict"].get("custom_reward", None) == 5# Test step with remaining two actionsactions = [@@ -152,11 +244,12 @@ async def test_single_step_env_lifecycle():results = await env.step(actions)assert isinstance(results, list)assert len(results) == 2- for result in results:- assert isinstance(result, StepResult)- assert result.reward == 15- assert result.done is True- assert result.observation == MockSingleStepEnv.PLACEHOLDER_OBS+ for next_obs, reward, done, info in results:+ assert next_obs == env.PLACEHOLDER_OBS+ assert reward == env.ACCURACY_REWARD + 5 # 5 is custom reward+ assert done+ assert isinstance(info, dict)+ assert info["rewards_dict"].get("custom_reward", None) == 5assert env._batch_done()await env.close()@@ -183,7 +276,157 @@ def create_mock_verifier():@pytest.mark.asyncio-async def test_single_step_env_error_handling():+async def test_single_step_env_error_handling_single():+ # Define a valid dataset+ data = [+ {+ "question": "What is 2 + 2?",+ "final_answer": "4",+ "rationale": "Adding 2 and 2 gives 4.",+ "metadata": {"difficulty": "easy"},+ },+ {+ "question": "What is the capital of France?",+ "final_answer": "Paris",+ "rationale": "Paris is known as the capital city of France.",+ "metadata": {"difficulty": "easy"},+ },+ ]+ dataset = StaticDataset(data)++ # **1. Test Faulty Dataset**+ faulty_data = [+ {+ "question": "What is 2 + 2?",+ # Missing "final_answer"+ "rationale": "Adding 2 and 2 gives 4.",+ "metadata": {"difficulty": "easy"},+ },+ ]+ with pytest.raises(ValueError):+ StaticDataset(faulty_data, strict=True)++ # **2. Test Faulty Verifier**+ mock_verifier_exception = MagicMock()+ mock_verifier_exception.setup = AsyncMock()+ mock_verifier_exception.cleanup = AsyncMock()+ mock_verifier_exception.verify_batch = AsyncMock(+ side_effect=Exception("Verifier error")+ )+ env_fail_verifier = SingleStepEnv(+ dataset=dataset,+ verifier=mock_verifier_exception,+ )+ await env_fail_verifier.setup()+ await env_fail_verifier.reset(batch_size=1)+ action = Action(index=0, llm_response="4")+ with pytest.raises(Exception, match="Verifier error"):+ await env_fail_verifier.step(action)++ # **3. Test State Mismanagement Scenarios**+ # a) Step without setup+ env_not_setup = SingleStepEnv(+ dataset=dataset,+ verifier=create_mock_verifier(),+ )+ with pytest.raises(+ RuntimeError,+ match=re.escape("Environment not set up. Call setup() first."),+ ):+ await env_not_setup.step(Action(index=0, llm_response="4"))++ # b) Step after episode is done+ env_episode_ended = SingleStepEnv(+ dataset=dataset,+ verifier=create_mock_verifier(),+ )+ await env_episode_ended.setup()+ await env_episode_ended.reset(batch_size=1)+ await env_episode_ended.step(Action(index=0, llm_response="4"))+ with pytest.raises(+ RuntimeError,+ match=re.escape("Episodes have ended for batch. Call reset() first."),+ ):+ await env_episode_ended.step(Action(index=0, llm_response="4"))++ # c) Step without reset+ env_no_reset = SingleStepEnv(+ dataset=dataset,+ verifier=create_mock_verifier(),+ )+ await env_no_reset.setup()+ with pytest.raises(+ RuntimeError,+ match=re.escape("Episodes have ended for batch. Call reset() first."),+ ):+ await env_no_reset.step(Action(index=0, llm_response="4"))++ # **4. Test Invalid Actions**+ env_invalid_actions = SingleStepEnv(+ dataset=dataset,+ verifier=create_mock_verifier(),+ )+ await env_invalid_actions.setup()+ await env_invalid_actions.reset(batch_size=1)++ # a) Action with invalid index+ with pytest.raises(+ ValueError, match="For batch_size=1, index must be None or 0"+ ):+ await env_invalid_actions.step(Action(index=1, llm_response="4"))++ # b) Providing a list of 2 actions for batch size == 1+ with pytest.raises(+ ValueError,+ match="For batch_size=1, expect a single Action or a list containing "+ "exactly one Action",+ ):+ await env_invalid_actions.step(+ [+ Action(index=0, llm_response="4"),+ Action(index=1, llm_response="3"),+ ]+ )++ # **5. Test Batch Size Issues**+ env_batch_size = SingleStepEnv(+ dataset=dataset,+ verifier=create_mock_verifier(),+ )+ await env_batch_size.setup()+ with pytest.raises(ValueError, match="Batch size must be positive"):+ await env_batch_size.reset(batch_size=0)++ # **6. Test Setup and Close Failures**+ # a) Setup failure+ mock_verifier_setup_fail = MagicMock()+ mock_verifier_setup_fail.setup = AsyncMock(+ side_effect=Exception("Setup failed")+ )+ env_setup_fail = SingleStepEnv(+ dataset=dataset,+ verifier=mock_verifier_setup_fail,+ )+ with pytest.raises(Exception, match="Setup failed"):+ await env_setup_fail.setup()++ # b) Close failure+ mock_verifier_close_fail = MagicMock()+ mock_verifier_close_fail.setup = AsyncMock()+ mock_verifier_close_fail.cleanup = AsyncMock(+ side_effect=Exception("Cleanup failed")+ )+ env_close_fail = SingleStepEnv(+ dataset=dataset,+ verifier=mock_verifier_close_fail,+ )+ await env_close_fail.setup()+ with pytest.raises(Exception, match="Cleanup failed"):+ await env_close_fail.close()+++async def test_batched_single_step_env_error_handling():# **1. Test Faulty Dataset**# Ensure we don't silently handle the errorfaulty_data = [