Skip to content

armgit / camel

Public Imported from GitHub
0

🐫 CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org

feat: Implement resetting from a generative dataset (#2027)

Apokryphosxbec98152d3df
diff --git a/camel/datasets/base_generator.py b/camel/datasets/base_generator.py
index e9ca6411f..2e3d611d1 100644
--- a/camel/datasets/base_generator.py
+++ b/camel/datasets/base_generator.py
@@ -39,6 +39,7 @@ class BaseGenerator(abc.ABC, IterableDataset):
def __init__(
self,
seed: int = 42,
+ puffer: int = 20,
cache: Union[str, Path, None] = None,
data_path: Union[str, Path, None] = None,
**kwargs,
@@ -47,6 +48,8 @@ class BaseGenerator(abc.ABC, IterableDataset):
Args:
seed (int): Random seed for reproducibility. (default: :obj:`42`)
+ puffer (int): Amount of DataPoints to be generated when the
+ iterator runs out of DataPoints in data. (default: :obj:`20`)
cache (Union[str, Path, None]): Optional path to save generated
datapoints during iteration. If None is provided, datapoints
will be discarded every 100 generations.
@@ -56,7 +59,7 @@ class BaseGenerator(abc.ABC, IterableDataset):
"""
self._rng = random.Random(seed)
self.cache = Path(cache) if cache else None
-
+ self._puffer = puffer
self._data: List[DataPoint] = []
self._batch_to_save: List[DataPoint] = []
@@ -72,15 +75,27 @@ class BaseGenerator(abc.ABC, IterableDataset):
)
@abc.abstractmethod
- async def generate_new(self, n: int, **kwargs) -> List[DataPoint]:
- r"""Generate n new datapoints.
+ async def generate_new(self, n: int, **kwargs) -> None:
+ r"""Generate n new datapoints and append them to self._data.
+
+ Subclass implementations must generate the specified number of
+ datapoints and append them directly to the `self._data` list.
+ This method should not return the datapoints; the iterator
+ relies on `self._data` being populated.
Args:
- n (int): Number of datapoints to generate.
+ n (int): Number of datapoints to generate and append.
**kwargs: Additional generation parameters.
Returns:
- List[DataPoint]: A list of newly generated datapoints.
+ None: This method should not return anything.
+
+ Example:
+ ```python
+ async def generate_new(self, n: int, **kwargs) -> None:
+ new_points = [DataPoint(...) for _ in range(n)]
+ self._data.extend(new_points)
+ ```
"""
pass
@@ -99,8 +114,7 @@ class BaseGenerator(abc.ABC, IterableDataset):
async def generator():
while True:
if not self._data:
- new_datapoints = await self.generate_new(20)
- self._data.extend(new_datapoints)
+ await self.generate_new(self._puffer)
datapoint = self._data.pop(0)
yield datapoint
self._batch_to_save.append(datapoint)
@@ -137,8 +151,7 @@ class BaseGenerator(abc.ABC, IterableDataset):
while True:
if not self._data:
- new_datapoints = asyncio.run(self.generate_new(20))
- self._data.extend(new_datapoints)
+ asyncio.run(self.generate_new(self._puffer))
datapoint = self._data.pop(0)
yield datapoint
self._batch_to_save.append(datapoint)
diff --git a/camel/datasets/few_shot_generator.py b/camel/datasets/few_shot_generator.py
index f437aac5f..81eb8a708 100644
--- a/camel/datasets/few_shot_generator.py
+++ b/camel/datasets/few_shot_generator.py
@@ -126,7 +126,7 @@ class FewShotGenerator(BaseGenerator):
max_retries: int = 10,
num_examples: int = 3,
**kwargs,
- ) -> List[DataPoint]:
+ ) -> None:
r"""Generates and validates `n` new datapoints through
few-shot prompting, with a retry limit.
@@ -255,4 +255,3 @@ class FewShotGenerator(BaseGenerator):
# Thread-safe way to extend the data list
async with asyncio.Lock():
self._data.extend(valid_data_points)
- return valid_data_points
diff --git a/camel/datasets/self_instruct_generator.py b/camel/datasets/self_instruct_generator.py
index 58d19d6be..974d6f587 100644
--- a/camel/datasets/self_instruct_generator.py
+++ b/camel/datasets/self_instruct_generator.py
@@ -310,7 +310,7 @@ class SelfInstructGenerator(BaseGenerator):
human_sample_count: int = 3,
machine_sample_count: int = 1,
**kwargs,
- ) -> list[DataPoint]:
+ ) -> None:
r"""Generates and validates `n` new datapoints through
self-instruct prompting, with a retry limit.
@@ -324,18 +324,6 @@ class SelfInstructGenerator(BaseGenerator):
(default: :obj:`1`)
**kwargs: Additional keyword arguments.
- Returns:
- List[DataPoint]: A list of newly generated valid datapoints.
-
- Raises:
- TypeError: If the agent's output is not a dictionary (or does not
- match the expected format).
- KeyError: If required keys are missing from the response.
- AttributeError: If the verifier response lacks attributes.
- ValidationError: If a datapoint fails schema validation.
- RuntimeError: If retries are exhausted before `n` valid datapoints
- are generated.
-
Notes:
- Retries on validation failures until `n` valid datapoints exist
or `max_retries` is reached, whichever comes first.
@@ -355,10 +343,12 @@ class SelfInstructGenerator(BaseGenerator):
)
machine_dps_list = list(self.machine_instructions)
- support_machine_dps = random.sample(
- machine_dps_list,
- min(machine_sample_count, len(machine_dps_list)),
- )
+ support_machine_dps = []
+ if machine_dps_list and machine_sample_count > 0:
+ support_machine_dps = random.sample(
+ machine_dps_list,
+ min(machine_sample_count, len(machine_dps_list)),
+ )
question = self.generate_new_instruction(
self.instruction_agent,
support_human_dps,
@@ -423,4 +413,3 @@ class SelfInstructGenerator(BaseGenerator):
async with self._lock:
self._data.extend(valid_data_points)
- return valid_data_points
diff --git a/camel/environments/single_step.py b/camel/environments/single_step.py
index a45f988e9..81ba6f91c 100644
--- a/camel/environments/single_step.py
+++ b/camel/environments/single_step.py
@@ -19,6 +19,7 @@ from camel.datasets import BaseGenerator, DataPoint, StaticDataset
from camel.logger import get_logger
from camel.verifiers.base import (
BaseVerifier,
+ VerificationOutcome,
VerificationResult,
)
@@ -206,9 +207,18 @@ class SingleStepEnv:
return observations[0] if batch_size == 1 else observations
elif isinstance(self.dataset, BaseGenerator):
- raise NotImplementedError(
- "Reset not yet implemented for BaseGenerator datasets."
- )
+ self._states = [
+ await self.dataset.async_sample() for _ in range(batch_size)
+ ]
+ self.current_batch_size = batch_size
+ self._states_done = [False] * batch_size
+
+ observations = [
+ Observation(question=sample.question, context={}, metadata={})
+ for sample in self._states
+ ]
+
+ return observations[0] if batch_size == 1 else observations
else:
raise TypeError(f"Unsupported dataset type: {type(self.dataset)}")
@@ -254,18 +264,19 @@ class SingleStepEnv:
"For batch_size=1, expect a single Action or a "
"list containing exactly one Action"
)
- elif not isinstance(action, Action):
+ actions = action
+ elif isinstance(action, Action):
+ actions = [action]
+ else:
raise ValueError(
"For batch_size=1, expect a single Action or a "
"list containing exactly one Action"
)
- if isinstance(action, Action):
- actions = [action]
- else:
- actions = action
+
+ # For batch_size=1, set index to 0 if not provided
if actions[0].index is None:
actions[0].index = 0
- if actions[0].index != 0:
+ elif actions[0].index != 0:
raise ValueError("For batch_size=1, index must be None or 0")
else: # batch_size >= 2
@@ -322,22 +333,31 @@ class SingleStepEnv:
for idx in indices:
ground_truths.append(self._states[idx].final_answer)
- verification_results = await self.verifier.verify_batch(
- solutions=proposed_solutions,
- reference_answers=ground_truths, # type: ignore [arg-type]
- raise_on_error=True,
- )
+ try:
+ verification_results = await self.verifier.verify_batch(
+ solutions=proposed_solutions,
+ reference_answers=ground_truths, # type: ignore [arg-type]
+ raise_on_error=True,
+ )
+ except Exception as e:
+ logger.error(f"Verification failed: {e}")
+ # Return failed verification results with status=FAILURE
+ verification_results = [
+ VerificationResult(
+ result="",
+ status=VerificationOutcome.FAILURE,
+ error_message=f"Verification error: {e}",
+ )
+ for _ in range(len(proposed_solutions))
+ ]
total_rewards, rewards_dicts = await self._compute_reward_batch(
proposed_solutions, verification_results
)
- # TODO Batch this
- step_results = []
- for i, action in enumerate(actions):
- assert action.index is not None
- idx = action.index
- step_result = StepResult(
+ # Create and return step results in batch
+ step_results = [
+ StepResult(
observation=self.PLACEHOLDER_OBS,
reward=total_rewards[i],
rewards_dict=rewards_dicts[i],
@@ -345,10 +365,12 @@ class SingleStepEnv:
info={
"proposed_solution": proposed_solutions[i],
"verification_result": verification_results[i],
- "state": self._states[idx],
+ "state": self._states[indices[i]],
},
- )
- step_results.append(step_result.as_tuple())
+ ).as_tuple()
+ for i in range(len(actions))
+ ]
+ for _, idx in enumerate(indices):
self._states_done[idx] = True
return step_results[0] if len(step_results) == 1 else step_results
@@ -426,7 +448,7 @@ class SingleStepEnv:
return all(self._states_done)
def _batch_started(self) -> bool:
- r"""Check if any state in the current batch is done.
+ r"""Check if the batch processing has started.
Returns:
bool: True if at least one state is marked as done, False
diff --git a/camel/verifiers/python_verifier.py b/camel/verifiers/python_verifier.py
index f4ea89818..94d20759f 100644
--- a/camel/verifiers/python_verifier.py
+++ b/camel/verifiers/python_verifier.py
@@ -142,7 +142,7 @@ class PythonVerifier(BaseVerifier):
self.venv_path = tempfile.mkdtemp()
try:
subprocess.run(
- ["uv", "venv", self.venv_path],
+ ["uv", "venv", "--python", sys.executable, self.venv_path],
check=True,
capture_output=True,
timeout=self._timeout,
diff --git a/test/datasets/test_base_dataset.py b/test/datasets/test_base_dataset.py
index 75c381246..b9d1d8678 100644
--- a/test/datasets/test_base_dataset.py
+++ b/test/datasets/test_base_dataset.py
@@ -1337,15 +1337,15 @@ async def test_generate_new():
dataset.agent = mock_agent
- new_datapoints = await dataset.generate_new(2)
+ await dataset.generate_new(2)
assert (
- len(new_datapoints) == 2
+ len(dataset._data) == 2
), "Should generate exactly 2 valid datapoints"
assert mock_agent.step.call_count == 3, "Should retry past invalid output"
# Check first correct datapoint (5 + 6 = 11)
- dp1 = new_datapoints[0]
+ dp1 = await dataset.async_sample()
assert dp1.question == "What is 5 + 6?"
assert dp1.rationale == "print(5 + 6)"
assert (
@@ -1354,7 +1354,7 @@ async def test_generate_new():
assert dp1.metadata["synthetic"] == "True"
# Check second correct datapoint (7 + 8 = 15)
- dp2 = new_datapoints[1]
+ dp2 = await dataset.async_sample()
assert dp2.question == "What is 7 + 8?"
assert dp2.rationale == "print(7 + 8)"
assert (
diff --git a/test/datasets/test_self_instruct_generator.py b/test/datasets/test_self_instruct_generator.py
index e19bc8e28..a5c85f1b5 100644
--- a/test/datasets/test_self_instruct_generator.py
+++ b/test/datasets/test_self_instruct_generator.py
@@ -269,15 +269,15 @@ class TestGenerateNew:
return_value="print('New code')"
)
- result = await generator.generate_new(n=1)
+ await generator.generate_new(n=1)
# Verify generated datapoint
- assert len(result) == 1
- assert result[0].question == "New question?"
- assert result[0].rationale == "print('New code')"
- assert result[0].final_answer == "Verified output"
- assert result[0].metadata["synthetic"] == "True"
- assert result[0].metadata["generator"] == "self_instruct"
+ assert len(generator._data) == 1
+ assert generator._data[0].question == "New question?"
+ assert generator._data[0].rationale == "print('New code')"
+ assert generator._data[0].final_answer == "Verified output"
+ assert generator._data[0].metadata["synthetic"] == "True"
+ assert generator._data[0].metadata["generator"] == "self_instruct"
@pytest.mark.asyncio
async def test_generate_new_verification_failure(self, generator):
@@ -304,15 +304,18 @@ class TestGenerateNew:
side_effect=["print('Failed code')", "print('Successful code')"]
)
- result = await generator.generate_new(
+ # Clear any existing data
+ generator._data = []
+
+ await generator.generate_new(
n=1, max_retries=3
) # Reduce retries for test speed
# Verify generated datapoint
- assert len(result) == 1
- assert result[0].question == "Successful question?"
- assert result[0].rationale == "print('Successful code')"
- assert result[0].final_answer == "Verified output"
+ assert len(generator._data) == 1
+ assert generator._data[0].question == "Successful question?"
+ assert generator._data[0].rationale == "print('Successful code')"
+ assert generator._data[0].final_answer == "Verified output"
@pytest.mark.asyncio
async def test_generate_new_validation_error(self, generator):
@@ -330,6 +333,9 @@ class TestGenerateNew:
side_effect=["print('Failed code')", "print('Successful code')"]
)
+ # Clear any existing data
+ generator._data = []
+
# Use a custom DataPoint creation function to simulate validation
# error then success
original_datapoint = DataPoint
@@ -361,12 +367,12 @@ class TestGenerateNew:
'camel.datasets.self_instruct_generator.DataPoint',
side_effect=mock_datapoint_factory,
):
- result = await generator.generate_new(
+ await generator.generate_new(
n=1, max_retries=3
) # Reduce retries for test speed
# Verify generated datapoint
- assert len(result) == 1
- assert result[0].question == "Successful question?"
- assert result[0].rationale == "print('Successful code')"
- assert result[0].final_answer == "Verified output"
+ assert len(generator._data) == 1
+ assert generator._data[0].question == "Successful question?"
+ assert generator._data[0].rationale == "print('Successful code')"
+ assert generator._data[0].final_answer == "Verified output"
diff --git a/test/environments/test_single_stage_env.py b/test/environments/test_single_stage_env.py
index be44429e4..607d6ff70 100644
--- a/test/environments/test_single_stage_env.py
+++ b/test/environments/test_single_stage_env.py
@@ -12,12 +12,12 @@
# limitations under the License.
# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
import re
-from typing import Dict
+from typing import Dict, List
from unittest.mock import AsyncMock, MagicMock
import pytest
-from camel.datasets import StaticDataset
+from camel.datasets import BaseGenerator, DataPoint, StaticDataset
from camel.environments import (
Action,
Observation,
@@ -35,6 +35,108 @@ class MockSingleStepEnv(SingleStepEnv):
return {"custom_reward": 5}
+class MockGenerator(BaseGenerator):
+ def __init__(self, predefined_data: List[Dict], *args, **kwargs):
+ if 'puffer' not in kwargs:
+ kwargs['puffer'] = 1
+ super().__init__(*args, **kwargs)
+ self.predefined_data = predefined_data
+ self.index = 0
+
+ async def generate_new(self, n: int, **kwargs) -> None:
+ if self.index + n > len(self.predefined_data):
+ raise ValueError("Not enough predefined datapoints")
+ new_points = [
+ DataPoint(**data)
+ for data in self.predefined_data[self.index : self.index + n]
+ ]
+ self._data.extend(new_points)
+ self.index += n
+
+
+async def test_single_step_env_lifecycle_single_generator():
+ predefined_data = [
+ {
+ "question": "What is 2 + 2?",
+ "final_answer": "4",
+ "rationale": "Adding 2 and 2 gives 4.",
+ "metadata": {"difficulty": "easy"},
+ },
+ {
+ "question": "What is the capital of France?",
+ "final_answer": "Paris",
+ "rationale": "Paris is known as the capital city of France.",
+ "metadata": {"difficulty": "easy"},
+ },
+ ]
+ dataset = MockGenerator(predefined_data, seed=42)
+
+ mock_verifier = MagicMock()
+ mock_verifier.setup = AsyncMock()
+ mock_verifier.cleanup = AsyncMock()
+ mock_verifier.verify_batch = AsyncMock(
+ side_effect=lambda solutions, reference_answers, **kwargs: [
+ VerificationResult(
+ status=VerificationOutcome.SUCCESS,
+ result="Verification successful",
+ feedback="Correct",
+ score=1.0,
+ )
+ for _ in solutions
+ ]
+ )
+
+ env = MockSingleStepEnv(dataset=dataset, verifier=mock_verifier)
+
+ await env.setup()
+ assert env._is_setup is True
+ mock_verifier.setup.assert_awaited_once()
+
+ # Test reset with batch_size=1
+ observation = await env.reset(batch_size=1)
+ assert isinstance(observation, Observation)
+ assert observation.question == "What is 2 + 2?" # First datapoint
+
+ # Test step with a single action
+ action = Action(index=0, llm_response="4")
+ result = await env.step(action)
+ assert isinstance(result, tuple)
+ next_obs, reward, done, info = result
+ assert next_obs == env.PLACEHOLDER_OBS
+ assert reward == env.ACCURACY_REWARD + 5
+ assert done is True
+ assert info["rewards_dict"].get("custom_reward", None) == 5
+
+ # Test that stepping again without reset fails
+ with pytest.raises(
+ RuntimeError,
+ match=r"Episodes have ended for batch\. Call reset\(\) first\.",
+ ):
+ await env.step(action)
+
+ # Test reset and step again
+ observation2 = await env.reset(batch_size=1)
+ assert isinstance(observation2, Observation)
+ assert (
+ observation2.question == "What is the capital of France?"
+ ) # Second datapoint
+ action2 = Action(index=None, llm_response="Paris")
+ result2 = await env.step(action2)
+ next_obs2, reward2, done2, info2 = result2
+ assert next_obs2 == env.PLACEHOLDER_OBS
+ assert reward2 == env.ACCURACY_REWARD + 5
+ assert done2 is True
+ assert info2["rewards_dict"].get("custom_reward", None) == 5
+
+ # Note: Skipping deterministic sampling test as BaseGenerator
+ # doesn't use seed in the same way
+ # Test close
+ await env.close()
+ assert env._is_setup is False
+ mock_verifier.cleanup.assert_awaited_once()
+
+
@pytest.mark.asyncio
async def test_single_step_env_lifecycle_single():
# Define a sample dataset
@@ -257,6 +359,106 @@ async def test_batched_single_step_env_lifecycle():
mock_verifier.cleanup.assert_awaited_once()
+async def test_batched_single_step_env_lifecycle_generator():
+ predefined_data = [
+ {
+ "question": "What is 2 + 2?",
+ "final_answer": "4",
+ "rationale": "Adding 2 and 2 gives 4.",
+ "metadata": {"difficulty": "easy"},
+ },
+ {
+ "question": "What is the capital of France?",
+ "final_answer": "Paris",
+ "rationale": "Paris is known as the capital city of France.",
+ "metadata": {"difficulty": "easy"},
+ },
+ {
+ "question": "Who wrote 'Romeo and Juliet'?",
+ "final_answer": "Shakespeare",
+ "rationale": "William Shakespeare is the author of 'Romeo and "
+ "Juliet'.",
+ "metadata": {"difficulty": "medium"},
+ },
+ {
+ "question": "What is the boiling point of water in Celsius?",
+ "final_answer": "100",
+ "rationale": "Water boils at 100 degrees Celsius at standard "
+ "pressure.",
+ "metadata": {"difficulty": "easy"},
+ },
+ ]
+ dataset = MockGenerator(predefined_data, seed=42)
+
+ # Create a mock verifier
+ mock_verifier = MagicMock()
+ mock_verifier.setup = AsyncMock()
+ mock_verifier.cleanup = AsyncMock()
+ mock_verifier.verify_batch = AsyncMock(
+ side_effect=lambda solutions, reference_answers, **kwargs: [
+ VerificationResult(
+ status=VerificationOutcome.SUCCESS,
+ result="Verification successful",
+ feedback="Correct",
+ score=1.0,
+ )
+ for _ in solutions
+ ]
+ )
+
+ # Initialize the environment
+ env = MockSingleStepEnv(dataset=dataset, verifier=mock_verifier)
+
+ # Test setup
+ await env.setup()
+ assert env._is_setup is True
+ mock_verifier.setup.assert_awaited_once()
+
+ # Test reset with batch_size=4
+ observations = await env.reset(batch_size=4)
+ assert isinstance(observations, list)
+ assert len(observations) == 4
+ expected_questions = [dp["question"] for dp in predefined_data]
+ for i, obs in enumerate(observations):
+ assert isinstance(obs, Observation)
+ assert obs.question == expected_questions[i]
+
+ # Test step with 2 actions
+ actions = [
+ Action(index=0, llm_response="4"), # First question
+ Action(index=2, llm_response="Shakespeare"), # Third question
+ ]
+ results = await env.step(actions)
+ assert isinstance(results, list)
+ assert len(results) == 2
+ for next_obs, reward, done, info in results:
+ assert next_obs == env.PLACEHOLDER_OBS
+ assert reward == env.ACCURACY_REWARD + 5
+ assert done is True
+ assert info["rewards_dict"].get("custom_reward", None) == 5
+
+ # Test step with remaining 2 actions
+ actions = [
+ Action(index=1, llm_response="Paris"), # Second question
+ Action(index=3, llm_response="100"), # Fourth question
+ ]
+ results = await env.step(actions)
+ assert isinstance(results, list)
+ assert len(results) == 2
+ for next_obs, reward, done, info in results:
+ assert next_obs == env.PLACEHOLDER_OBS
+ assert reward == env.ACCURACY_REWARD + 5
+ assert done is True
+ assert info["rewards_dict"].get("custom_reward", None) == 5
+
+ # Test batch completion
+ assert env._batch_done()
+ await env.close()
+ assert env._is_setup is False
+ mock_verifier.cleanup.assert_awaited_once()
+
+
def create_mock_verifier():
verifier = MagicMock()
verifier.setup = AsyncMock()
@@ -320,8 +522,16 @@ async def test_single_step_env_error_handling_single():
await env_fail_verifier.setup()
await env_fail_verifier.reset(batch_size=1)
action = Action(index=0, llm_response="4")
- with pytest.raises(Exception, match="Verifier error"):
- await env_fail_verifier.step(action)
+
+ step_result = await env_fail_verifier.step(action)
+
+ _, reward, _, info = step_result
+
+ # Verify that the verification result indicates a failure
+ assert info["verification_result"].status == VerificationOutcome.FAILURE
+ assert "Verifier error" in info["verification_result"].error_message
+ # Verify that the reward reflects the failure
+ assert reward == 0.0
# **3. Test State Mismanagement Scenarios**
# a) Step without setup
@@ -506,7 +716,7 @@ async def test_batched_single_step_env_error_handling():
dataset = StaticDataset(data)
# **2. Test Faulty Verifier**
- # Ensure errors in verifier are propagated
+ # Ensure errors in verifier are properly handled
mock_verifier_exception = MagicMock()
mock_verifier_exception.setup = AsyncMock()
mock_verifier_exception.cleanup = AsyncMock()
@@ -520,8 +730,17 @@ async def test_batched_single_step_env_error_handling():
await env_fail_verifier.setup()
await env_fail_verifier.reset(batch_size=1)
action = Action(index=0, llm_response="4")
- with pytest.raises(Exception, match="Verifier error"):
- await env_fail_verifier.step(action)
+
+ step_result = await env_fail_verifier.step(action)
+
+ # Unpack step result
+ _, reward, _, info = step_result
+
+ # Verify that the verification result indicates a failure
+ assert info["verification_result"].status == VerificationOutcome.FAILURE
+ assert "Verifier error" in info["verification_result"].error_message
+ # Verify that the reward reflects the failure
+ assert reward == 0.0
# **3. Test State Mismanagement Scenarios**
# a) Step without setup