Skip to content

armgit / camel

Public Imported from GitHub
0

🐫 CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org

refactor: Clean up SingleStepEnv (#1985)

Apokryphosxd58bdab28eea
diff --git a/camel/environments/models.py b/camel/environments/models.py
index fe99904a2..8039dfa0d 100644
--- a/camel/environments/models.py
+++ b/camel/environments/models.py
@@ -13,7 +13,7 @@
# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
from datetime import datetime, timezone
-from typing import Any, Dict, Optional, Protocol
+from typing import Any, Dict, Optional, Protocol, Tuple
from pydantic import BaseModel, Field
@@ -33,9 +33,8 @@ class Action(BaseModel):
generated (UTC).
"""
- index: int = Field(
- ..., description="Index of the state this action is performed upon"
- )
+ index: Optional[int] = Field(default=None, description="...")
+
llm_response: str = Field(description="Generated response from the LLM")
metadata: Dict[str, Any] = Field(
default_factory=dict,
@@ -87,6 +86,13 @@ class StepResult(BaseModel):
description="Additional information about the step",
)
+ def as_tuple(
+ self,
+ ) -> Tuple[Observation, float, bool, Dict[str, Any]]:
+ r"""Returns all fields of the model as a tuple, in declaration order"""
+ self.info["rewards_dict"] = self.rewards_dict
+ return (self.observation, self.reward, self.done, self.info)
+
class Environment(Protocol):
async def reset(self) -> Observation:
diff --git a/camel/environments/single_step.py b/camel/environments/single_step.py
index 97931e45b..b51d8bbc8 100644
--- a/camel/environments/single_step.py
+++ b/camel/environments/single_step.py
@@ -126,6 +126,7 @@ class SingleStepEnv:
await self.verifier.cleanup()
self._states = []
self._states_done = []
+ self.current_batch_size = 0
logger.info('Environment closed successfully')
except Exception as e:
logger.error(f'Failed to close environment: {e}')
@@ -157,6 +158,8 @@ class SingleStepEnv:
ValueError: If batch size exceeds dataset size.
TypeError: If the dataset is of an unsupported type.
"""
+ if batch_size <= 0:
+ raise ValueError("Batch size must be positive")
if not self._is_setup:
logger.warning(
@@ -212,16 +215,18 @@ class SingleStepEnv:
async def step(
self, action: Union[Action, List[Action]]
- ) -> Union[StepResult, List[StepResult]]:
- r"""Process actions for a subset of states and update their
- finished status.
+ ) -> Union[
+ Tuple[Observation, float, bool, Dict[str, Any]],
+ List[Tuple[Observation, float, bool, Dict[str, Any]]],
+ ]:
+ r"""Process actions for a subset of states and update their finished
+ status.
Args:
- action: Single action or list of actions, where each action
- contains an index indicating which state it corresponds to.
- The index must be a valid position in the internal _states list
- that was populated during the reset() call.
-
+ action: Single action (for batch_size=1 or micro-batch of size 1)
+ or list of actions (for batch_size>=2 with multiple actions).
+ Each action must have an index for batch_size>=2, indicating
+ which state it corresponds to.
Returns:
Union[StepResult, List[StepResult]]: StepResult or list of
@@ -241,10 +246,62 @@ class SingleStepEnv:
if not self._states:
raise RuntimeError("No current observation. Call reset() first.")
- # Normalize everything to list
- actions = [action] if isinstance(action, Action) else action
- indices = [act.index for act in actions]
+ # Normalize actions into a list for uniform processing
+ if self.current_batch_size == 1:
+ if isinstance(action, list):
+ if len(action) != 1 or not isinstance(action[0], Action):
+ raise ValueError(
+ "For batch_size=1, expect a single Action or a "
+ "list containing exactly one Action"
+ )
+ elif not isinstance(action, Action):
+ raise ValueError(
+ "For batch_size=1, expect a single Action or a "
+ "list containing exactly one Action"
+ )
+ if isinstance(action, Action):
+ actions = [action]
+ else:
+ actions = action
+ if actions[0].index is None:
+ actions[0].index = 0
+ if actions[0].index != 0:
+ raise ValueError("For batch_size=1, index must be None or 0")
+
+ else: # batch_size >= 2
+ if isinstance(action, Action):
+ if action.index is None:
+ raise ValueError(
+ "For batch_size>=2, each Action must have an index"
+ )
+ if not isinstance(action.index, int):
+ raise ValueError("Index must be an integer")
+ actions = [action]
+ elif isinstance(action, list):
+ if not action: # Empty list
+ raise ValueError("Action list cannot be empty")
+ actions = action
+ for act in actions:
+ if not isinstance(act, Action):
+ raise ValueError(
+ "All elements in list must be Action objects"
+ )
+ if act.index is None:
+ raise ValueError(
+ "For batch_size>=2, each Action must have an index"
+ )
+ if not isinstance(act.index, int):
+ raise ValueError("Index must be an integer")
+ else:
+ raise ValueError(
+ "For batch_size>=2, expect an Action or list of Actions"
+ )
+ # Validate indices
+ indices: List[int] = []
+ for act in actions:
+ assert act.index is not None
+ indices.append(act.index)
if len(set(indices)) != len(indices):
raise ValueError("Duplicate state indices in actions.")
for idx in indices:
@@ -254,7 +311,6 @@ class SingleStepEnv:
raise ValueError(f"State at index {idx} is already finished.")
num_actions = len(actions)
-
if self.current_batch_size % num_actions != 0:
logger.warning(
f"Number of actions ({num_actions}) is not a divisor of "
@@ -262,9 +318,9 @@ class SingleStepEnv:
)
proposed_solutions = [act.llm_response for act in actions]
- ground_truths: List[str] = [
- self._states[idx].final_answer for idx in indices
- ]
+ ground_truths: List[str] = []
+ for idx in indices:
+ ground_truths.append(self._states[idx].final_answer)
verification_results = await self.verifier.verify_batch(
solutions=proposed_solutions,
@@ -276,9 +332,10 @@ class SingleStepEnv:
proposed_solutions, verification_results
)
+ # TODO Batch this
step_results = []
- # TODO: batch this
for i, action in enumerate(actions):
+ assert action.index is not None
idx = action.index
step_result = StepResult(
observation=self.PLACEHOLDER_OBS,
@@ -291,7 +348,7 @@ class SingleStepEnv:
"state": self._states[idx],
},
)
- step_results.append(step_result)
+ step_results.append(step_result.as_tuple())
self._states_done[idx] = True
return step_results[0] if len(step_results) == 1 else step_results
@@ -315,6 +372,12 @@ class SingleStepEnv:
- List of total rewards for each solution.
- List of reward component dictionaries for each solution.
"""
+ if len(proposed_solutions) != len(verification_results):
+ raise ValueError(
+ f"Length mismatch: {len(proposed_solutions)} solutions vs "
+ f"{len(verification_results)} verification results"
+ )
+
total_rewards = []
rewards_dicts = []
@@ -355,9 +418,20 @@ class SingleStepEnv:
return {}
def _batch_done(self) -> bool:
+ r"""Check if all states in the current batch are done.
+
+ Returns:
+ bool: True if all states are marked as done, False otherwise.
+ """
return all(self._states_done)
def _batch_started(self) -> bool:
+ r"""Check if any state in the current batch is done.
+
+ Returns:
+ bool: True if at least one state is marked as done, False
+ otherwise.
+ """
return any(self._states_done)
@property
diff --git a/test/environments/test_single_stage_env.py b/test/environments/test_single_stage_env.py
index d67831284..319696cf9 100644
--- a/test/environments/test_single_stage_env.py
+++ b/test/environments/test_single_stage_env.py
@@ -21,7 +21,6 @@ from camel.datasets import StaticDataset
from camel.environments import (
Action,
Observation,
- StepResult,
)
from camel.environments.single_step import SingleStepEnv
from camel.verifiers.models import VerificationOutcome, VerificationResult
@@ -37,7 +36,99 @@ class MockSingleStepEnv(SingleStepEnv):
@pytest.mark.asyncio
-async def test_single_step_env_lifecycle():
+async def test_single_step_env_lifecycle_single():
+ # Define a sample dataset
+ data = [
+ {
+ "question": "What is 2 + 2?",
+ "final_answer": "4",
+ "rationale": "Adding 2 and 2 gives 4.",
+ "metadata": {"difficulty": "easy"},
+ },
+ {
+ "question": "What is the capital of France?",
+ "final_answer": "Paris",
+ "rationale": "Paris is known as the capital city of France.",
+ "metadata": {"difficulty": "easy"},
+ },
+ ]
+ dataset = StaticDataset(data)
+
+ # Create a mock verifier
+ mock_verifier = MagicMock()
+ mock_verifier.setup = AsyncMock()
+ mock_verifier.cleanup = AsyncMock()
+ mock_verifier.verify_batch = AsyncMock(
+ side_effect=lambda solutions, ground_truths, **kwargs: [
+ VerificationResult(
+ status=VerificationOutcome.SUCCESS,
+ result="Verification successful",
+ feedback="Correct",
+ score=1.0,
+ )
+ for _ in solutions
+ ]
+ )
+
+ # Initialize the environment
+ env = MockSingleStepEnv(dataset=dataset, verifier=mock_verifier)
+
+ # Test setup
+ await env.setup()
+ assert env._is_setup is True
+ mock_verifier.setup.assert_awaited_once()
+
+ # Test reset with batch_size=1
+ observation = await env.reset(batch_size=1)
+ assert isinstance(
+ observation, Observation
+ ) # Single Observation, not a list
+ assert observation.question in [dp["question"] for dp in data]
+
+ # Test step with a single action (index=0)
+ action = Action(index=0, llm_response="4") # Assuming first question
+ result = await env.step(action)
+ assert isinstance(result, tuple) # Single result tuple, not a list
+ next_obs, reward, done, info = result
+ assert next_obs == env.PLACEHOLDER_OBS
+ assert reward == env.ACCURACY_REWARD + 5 # Accuracy + custom reward
+ assert done is True # Single step should end the episode
+ assert isinstance(info, dict)
+ assert info["rewards_dict"].get("custom_reward", None) == 5
+
+ # Test that stepping again without reset fails
+ with pytest.raises(
+ RuntimeError,
+ match=re.escape("Episodes have ended for batch. Call reset() first."),
+ ):
+ await env.step(action)
+
+ # Test reset and step again with index=None (should default to 0)
+ observation2 = await env.reset(batch_size=1)
+ assert isinstance(observation2, Observation)
+ action2 = Action(
+ index=None, llm_response="Paris"
+ ) # Assuming second question
+ result2 = await env.step(action2)
+ next_obs2, reward2, done2, info2 = result2
+ assert next_obs2 == env.PLACEHOLDER_OBS
+ assert reward2 == env.ACCURACY_REWARD + 5
+ assert done2 is True
+ assert info2["rewards_dict"].get("custom_reward", None) == 5
+
+ # Test deterministic sampling with seed
+ obs1 = await env.reset(batch_size=1, seed=42)
+ obs2 = await env.reset(batch_size=1, seed=42)
+ assert obs1.question == obs2.question # Same seed, same question
+
+ # Test close
+ await env.close()
+ assert env._is_setup is False
+ mock_verifier.cleanup.assert_awaited_once()
+
+
+async def test_batched_single_step_env_lifecycle():
data = [
{
"question": "What is 2 + 2?",
@@ -134,11 +225,12 @@ async def test_single_step_env_lifecycle():
results = await env.step(actions)
assert isinstance(results, list)
assert len(results) == 2
- for result in results:
- assert isinstance(result, StepResult)
- assert result.reward == 15
- assert result.done is True
- assert result.observation == MockSingleStepEnv.PLACEHOLDER_OBS
+ for next_obs, reward, done, info in results:
+ assert next_obs == env.PLACEHOLDER_OBS
+ assert reward == env.ACCURACY_REWARD + 5 # 5 is custom reward
+ assert done
+ assert isinstance(info, dict)
+ assert info["rewards_dict"].get("custom_reward", None) == 5
# Test step with remaining two actions
actions = [
@@ -152,11 +244,12 @@ async def test_single_step_env_lifecycle():
results = await env.step(actions)
assert isinstance(results, list)
assert len(results) == 2
- for result in results:
- assert isinstance(result, StepResult)
- assert result.reward == 15
- assert result.done is True
- assert result.observation == MockSingleStepEnv.PLACEHOLDER_OBS
+ for next_obs, reward, done, info in results:
+ assert next_obs == env.PLACEHOLDER_OBS
+ assert reward == env.ACCURACY_REWARD + 5 # 5 is custom reward
+ assert done
+ assert isinstance(info, dict)
+ assert info["rewards_dict"].get("custom_reward", None) == 5
assert env._batch_done()
await env.close()
@@ -183,7 +276,157 @@ def create_mock_verifier():
@pytest.mark.asyncio
-async def test_single_step_env_error_handling():
+async def test_single_step_env_error_handling_single():
+ # Define a valid dataset
+ data = [
+ {
+ "question": "What is 2 + 2?",
+ "final_answer": "4",
+ "rationale": "Adding 2 and 2 gives 4.",
+ "metadata": {"difficulty": "easy"},
+ },
+ {
+ "question": "What is the capital of France?",
+ "final_answer": "Paris",
+ "rationale": "Paris is known as the capital city of France.",
+ "metadata": {"difficulty": "easy"},
+ },
+ ]
+ dataset = StaticDataset(data)
+
+ # **1. Test Faulty Dataset**
+ faulty_data = [
+ {
+ "question": "What is 2 + 2?",
+ # Missing "final_answer"
+ "rationale": "Adding 2 and 2 gives 4.",
+ "metadata": {"difficulty": "easy"},
+ },
+ ]
+ with pytest.raises(ValueError):
+ StaticDataset(faulty_data, strict=True)
+
+ # **2. Test Faulty Verifier**
+ mock_verifier_exception = MagicMock()
+ mock_verifier_exception.setup = AsyncMock()
+ mock_verifier_exception.cleanup = AsyncMock()
+ mock_verifier_exception.verify_batch = AsyncMock(
+ side_effect=Exception("Verifier error")
+ )
+ env_fail_verifier = SingleStepEnv(
+ dataset=dataset,
+ verifier=mock_verifier_exception,
+ )
+ await env_fail_verifier.setup()
+ await env_fail_verifier.reset(batch_size=1)
+ action = Action(index=0, llm_response="4")
+ with pytest.raises(Exception, match="Verifier error"):
+ await env_fail_verifier.step(action)
+
+ # **3. Test State Mismanagement Scenarios**
+ # a) Step without setup
+ env_not_setup = SingleStepEnv(
+ dataset=dataset,
+ verifier=create_mock_verifier(),
+ )
+ with pytest.raises(
+ RuntimeError,
+ match=re.escape("Environment not set up. Call setup() first."),
+ ):
+ await env_not_setup.step(Action(index=0, llm_response="4"))
+
+ # b) Step after episode is done
+ env_episode_ended = SingleStepEnv(
+ dataset=dataset,
+ verifier=create_mock_verifier(),
+ )
+ await env_episode_ended.setup()
+ await env_episode_ended.reset(batch_size=1)
+ await env_episode_ended.step(Action(index=0, llm_response="4"))
+ with pytest.raises(
+ RuntimeError,
+ match=re.escape("Episodes have ended for batch. Call reset() first."),
+ ):
+ await env_episode_ended.step(Action(index=0, llm_response="4"))
+
+ # c) Step without reset
+ env_no_reset = SingleStepEnv(
+ dataset=dataset,
+ verifier=create_mock_verifier(),
+ )
+ await env_no_reset.setup()
+ with pytest.raises(
+ RuntimeError,
+ match=re.escape("Episodes have ended for batch. Call reset() first."),
+ ):
+ await env_no_reset.step(Action(index=0, llm_response="4"))
+
+ # **4. Test Invalid Actions**
+ env_invalid_actions = SingleStepEnv(
+ dataset=dataset,
+ verifier=create_mock_verifier(),
+ )
+ await env_invalid_actions.setup()
+ await env_invalid_actions.reset(batch_size=1)
+
+ # a) Action with invalid index
+ with pytest.raises(
+ ValueError, match="For batch_size=1, index must be None or 0"
+ ):
+ await env_invalid_actions.step(Action(index=1, llm_response="4"))
+
+ # b) Providing a list of 2 actions for batch size == 1
+ with pytest.raises(
+ ValueError,
+ match="For batch_size=1, expect a single Action or a list containing "
+ "exactly one Action",
+ ):
+ await env_invalid_actions.step(
+ [
+ Action(index=0, llm_response="4"),
+ Action(index=1, llm_response="3"),
+ ]
+ )
+
+ # **5. Test Batch Size Issues**
+ env_batch_size = SingleStepEnv(
+ dataset=dataset,
+ verifier=create_mock_verifier(),
+ )
+ await env_batch_size.setup()
+ with pytest.raises(ValueError, match="Batch size must be positive"):
+ await env_batch_size.reset(batch_size=0)
+
+ # **6. Test Setup and Close Failures**
+ # a) Setup failure
+ mock_verifier_setup_fail = MagicMock()
+ mock_verifier_setup_fail.setup = AsyncMock(
+ side_effect=Exception("Setup failed")
+ )
+ env_setup_fail = SingleStepEnv(
+ dataset=dataset,
+ verifier=mock_verifier_setup_fail,
+ )
+ with pytest.raises(Exception, match="Setup failed"):
+ await env_setup_fail.setup()
+
+ # b) Close failure
+ mock_verifier_close_fail = MagicMock()
+ mock_verifier_close_fail.setup = AsyncMock()
+ mock_verifier_close_fail.cleanup = AsyncMock(
+ side_effect=Exception("Cleanup failed")
+ )
+ env_close_fail = SingleStepEnv(
+ dataset=dataset,
+ verifier=mock_verifier_close_fail,
+ )
+ await env_close_fail.setup()
+ with pytest.raises(Exception, match="Cleanup failed"):
+ await env_close_fail.close()
+
+
+async def test_batched_single_step_env_error_handling():
# **1. Test Faulty Dataset**
# Ensure we don't silently handle the error
faulty_data = [