🐫 CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org
Python agentai-societiesartificial-intelligencecommunicative-aicooperative-aideep-learninglarge-language-modelsmulti-agent-systemsnatural-language-processing
feat: Implement resetting from a generative dataset (#2027)
Apokryphosxbec98152d3df
diff --git a/camel/datasets/base_generator.py b/camel/datasets/base_generator.pyindex e9ca6411f..2e3d611d1 100644--- a/camel/datasets/base_generator.py+++ b/camel/datasets/base_generator.py@@ -39,6 +39,7 @@ class BaseGenerator(abc.ABC, IterableDataset):def __init__(self,seed: int = 42,+ puffer: int = 20,cache: Union[str, Path, None] = None,data_path: Union[str, Path, None] = None,**kwargs,@@ -47,6 +48,8 @@ class BaseGenerator(abc.ABC, IterableDataset):Args:seed (int): Random seed for reproducibility. (default: :obj:`42`)+ puffer (int): Amount of DataPoints to be generated when the+ iterator runs out of DataPoints in data. (default: :obj:`20`)cache (Union[str, Path, None]): Optional path to save generateddatapoints during iteration. If None is provided, datapointswill be discarded every 100 generations.@@ -56,7 +59,7 @@ class BaseGenerator(abc.ABC, IterableDataset):"""self._rng = random.Random(seed)self.cache = Path(cache) if cache else None-+ self._puffer = pufferself._data: List[DataPoint] = []self._batch_to_save: List[DataPoint] = []@@ -72,15 +75,27 @@ class BaseGenerator(abc.ABC, IterableDataset):)@abc.abstractmethod- async def generate_new(self, n: int, **kwargs) -> List[DataPoint]:- r"""Generate n new datapoints.+ async def generate_new(self, n: int, **kwargs) -> None:+ r"""Generate n new datapoints and append them to self._data.++ Subclass implementations must generate the specified number of+ datapoints and append them directly to the `self._data` list.+ This method should not return the datapoints; the iterator+ relies on `self._data` being populated.Args:- n (int): Number of datapoints to generate.+ n (int): Number of datapoints to generate and append.**kwargs: Additional generation parameters.Returns:- List[DataPoint]: A list of newly generated datapoints.+ None: This method should not return anything.++ Example:+ ```python+ async def generate_new(self, n: int, **kwargs) -> None:+ new_points = [DataPoint(...) for _ in range(n)]+ self._data.extend(new_points)+ ```"""pass@@ -99,8 +114,7 @@ class BaseGenerator(abc.ABC, IterableDataset):async def generator():while True:if not self._data:- new_datapoints = await self.generate_new(20)- self._data.extend(new_datapoints)+ await self.generate_new(self._puffer)datapoint = self._data.pop(0)yield datapointself._batch_to_save.append(datapoint)@@ -137,8 +151,7 @@ class BaseGenerator(abc.ABC, IterableDataset):while True:if not self._data:- new_datapoints = asyncio.run(self.generate_new(20))- self._data.extend(new_datapoints)+ asyncio.run(self.generate_new(self._puffer))datapoint = self._data.pop(0)yield datapointself._batch_to_save.append(datapoint)diff --git a/camel/datasets/few_shot_generator.py b/camel/datasets/few_shot_generator.pyindex f437aac5f..81eb8a708 100644--- a/camel/datasets/few_shot_generator.py+++ b/camel/datasets/few_shot_generator.py@@ -126,7 +126,7 @@ class FewShotGenerator(BaseGenerator):max_retries: int = 10,num_examples: int = 3,**kwargs,- ) -> List[DataPoint]:+ ) -> None:r"""Generates and validates `n` new datapoints throughfew-shot prompting, with a retry limit.@@ -255,4 +255,3 @@ class FewShotGenerator(BaseGenerator):# Thread-safe way to extend the data listasync with asyncio.Lock():self._data.extend(valid_data_points)- return valid_data_pointsdiff --git a/camel/datasets/self_instruct_generator.py b/camel/datasets/self_instruct_generator.pyindex 58d19d6be..974d6f587 100644--- a/camel/datasets/self_instruct_generator.py+++ b/camel/datasets/self_instruct_generator.py@@ -310,7 +310,7 @@ class SelfInstructGenerator(BaseGenerator):human_sample_count: int = 3,machine_sample_count: int = 1,**kwargs,- ) -> list[DataPoint]:+ ) -> None:r"""Generates and validates `n` new datapoints throughself-instruct prompting, with a retry limit.@@ -324,18 +324,6 @@ class SelfInstructGenerator(BaseGenerator):(default: :obj:`1`)**kwargs: Additional keyword arguments.- Returns:- List[DataPoint]: A list of newly generated valid datapoints.-- Raises:- TypeError: If the agent's output is not a dictionary (or does not- match the expected format).- KeyError: If required keys are missing from the response.- AttributeError: If the verifier response lacks attributes.- ValidationError: If a datapoint fails schema validation.- RuntimeError: If retries are exhausted before `n` valid datapoints- are generated.-Notes:- Retries on validation failures until `n` valid datapoints existor `max_retries` is reached, whichever comes first.@@ -355,10 +343,12 @@ class SelfInstructGenerator(BaseGenerator):)machine_dps_list = list(self.machine_instructions)- support_machine_dps = random.sample(- machine_dps_list,- min(machine_sample_count, len(machine_dps_list)),- )+ support_machine_dps = []+ if machine_dps_list and machine_sample_count > 0:+ support_machine_dps = random.sample(+ machine_dps_list,+ min(machine_sample_count, len(machine_dps_list)),+ )question = self.generate_new_instruction(self.instruction_agent,support_human_dps,@@ -423,4 +413,3 @@ class SelfInstructGenerator(BaseGenerator):async with self._lock:self._data.extend(valid_data_points)- return valid_data_pointsdiff --git a/camel/environments/single_step.py b/camel/environments/single_step.pyindex a45f988e9..81ba6f91c 100644--- a/camel/environments/single_step.py+++ b/camel/environments/single_step.py@@ -19,6 +19,7 @@ from camel.datasets import BaseGenerator, DataPoint, StaticDatasetfrom camel.logger import get_loggerfrom camel.verifiers.base import (BaseVerifier,+ VerificationOutcome,VerificationResult,)@@ -206,9 +207,18 @@ class SingleStepEnv:return observations[0] if batch_size == 1 else observationselif isinstance(self.dataset, BaseGenerator):- raise NotImplementedError(- "Reset not yet implemented for BaseGenerator datasets."- )+ self._states = [+ await self.dataset.async_sample() for _ in range(batch_size)+ ]+ self.current_batch_size = batch_size+ self._states_done = [False] * batch_size++ observations = [+ Observation(question=sample.question, context={}, metadata={})+ for sample in self._states+ ]++ return observations[0] if batch_size == 1 else observationselse:raise TypeError(f"Unsupported dataset type: {type(self.dataset)}")@@ -254,18 +264,19 @@ class SingleStepEnv:"For batch_size=1, expect a single Action or a ""list containing exactly one Action")- elif not isinstance(action, Action):+ actions = action+ elif isinstance(action, Action):+ actions = [action]+ else:raise ValueError("For batch_size=1, expect a single Action or a ""list containing exactly one Action")- if isinstance(action, Action):- actions = [action]- else:- actions = action++ # For batch_size=1, set index to 0 if not providedif actions[0].index is None:actions[0].index = 0- if actions[0].index != 0:+ elif actions[0].index != 0:raise ValueError("For batch_size=1, index must be None or 0")else: # batch_size >= 2@@ -322,22 +333,31 @@ class SingleStepEnv:for idx in indices:ground_truths.append(self._states[idx].final_answer)- verification_results = await self.verifier.verify_batch(- solutions=proposed_solutions,- reference_answers=ground_truths, # type: ignore [arg-type]- raise_on_error=True,- )+ try:+ verification_results = await self.verifier.verify_batch(+ solutions=proposed_solutions,+ reference_answers=ground_truths, # type: ignore [arg-type]+ raise_on_error=True,+ )+ except Exception as e:+ logger.error(f"Verification failed: {e}")+ # Return failed verification results with status=FAILURE+ verification_results = [+ VerificationResult(+ result="",+ status=VerificationOutcome.FAILURE,+ error_message=f"Verification error: {e}",+ )+ for _ in range(len(proposed_solutions))+ ]total_rewards, rewards_dicts = await self._compute_reward_batch(proposed_solutions, verification_results)- # TODO Batch this- step_results = []- for i, action in enumerate(actions):- assert action.index is not None- idx = action.index- step_result = StepResult(+ # Create and return step results in batch+ step_results = [+ StepResult(observation=self.PLACEHOLDER_OBS,reward=total_rewards[i],rewards_dict=rewards_dicts[i],@@ -345,10 +365,12 @@ class SingleStepEnv:info={"proposed_solution": proposed_solutions[i],"verification_result": verification_results[i],- "state": self._states[idx],+ "state": self._states[indices[i]],},- )- step_results.append(step_result.as_tuple())+ ).as_tuple()+ for i in range(len(actions))+ ]+ for _, idx in enumerate(indices):self._states_done[idx] = Truereturn step_results[0] if len(step_results) == 1 else step_results@@ -426,7 +448,7 @@ class SingleStepEnv:return all(self._states_done)def _batch_started(self) -> bool:- r"""Check if any state in the current batch is done.+ r"""Check if the batch processing has started.Returns:bool: True if at least one state is marked as done, Falsediff --git a/camel/verifiers/python_verifier.py b/camel/verifiers/python_verifier.pyindex f4ea89818..94d20759f 100644--- a/camel/verifiers/python_verifier.py+++ b/camel/verifiers/python_verifier.py@@ -142,7 +142,7 @@ class PythonVerifier(BaseVerifier):self.venv_path = tempfile.mkdtemp()try:subprocess.run(- ["uv", "venv", self.venv_path],+ ["uv", "venv", "--python", sys.executable, self.venv_path],check=True,capture_output=True,timeout=self._timeout,diff --git a/test/datasets/test_base_dataset.py b/test/datasets/test_base_dataset.pyindex 75c381246..b9d1d8678 100644--- a/test/datasets/test_base_dataset.py+++ b/test/datasets/test_base_dataset.py@@ -1337,15 +1337,15 @@ async def test_generate_new():dataset.agent = mock_agent- new_datapoints = await dataset.generate_new(2)+ await dataset.generate_new(2)assert (- len(new_datapoints) == 2+ len(dataset._data) == 2), "Should generate exactly 2 valid datapoints"assert mock_agent.step.call_count == 3, "Should retry past invalid output"# Check first correct datapoint (5 + 6 = 11)- dp1 = new_datapoints[0]+ dp1 = await dataset.async_sample()assert dp1.question == "What is 5 + 6?"assert dp1.rationale == "print(5 + 6)"assert (@@ -1354,7 +1354,7 @@ async def test_generate_new():assert dp1.metadata["synthetic"] == "True"# Check second correct datapoint (7 + 8 = 15)- dp2 = new_datapoints[1]+ dp2 = await dataset.async_sample()assert dp2.question == "What is 7 + 8?"assert dp2.rationale == "print(7 + 8)"assert (diff --git a/test/datasets/test_self_instruct_generator.py b/test/datasets/test_self_instruct_generator.pyindex e19bc8e28..a5c85f1b5 100644--- a/test/datasets/test_self_instruct_generator.py+++ b/test/datasets/test_self_instruct_generator.py@@ -269,15 +269,15 @@ class TestGenerateNew:return_value="print('New code')")- result = await generator.generate_new(n=1)+ await generator.generate_new(n=1)# Verify generated datapoint- assert len(result) == 1- assert result[0].question == "New question?"- assert result[0].rationale == "print('New code')"- assert result[0].final_answer == "Verified output"- assert result[0].metadata["synthetic"] == "True"- assert result[0].metadata["generator"] == "self_instruct"+ assert len(generator._data) == 1+ assert generator._data[0].question == "New question?"+ assert generator._data[0].rationale == "print('New code')"+ assert generator._data[0].final_answer == "Verified output"+ assert generator._data[0].metadata["synthetic"] == "True"+ assert generator._data[0].metadata["generator"] == "self_instruct"@pytest.mark.asyncioasync def test_generate_new_verification_failure(self, generator):@@ -304,15 +304,18 @@ class TestGenerateNew:side_effect=["print('Failed code')", "print('Successful code')"])- result = await generator.generate_new(+ # Clear any existing data+ generator._data = []++ await generator.generate_new(n=1, max_retries=3) # Reduce retries for test speed# Verify generated datapoint- assert len(result) == 1- assert result[0].question == "Successful question?"- assert result[0].rationale == "print('Successful code')"- assert result[0].final_answer == "Verified output"+ assert len(generator._data) == 1+ assert generator._data[0].question == "Successful question?"+ assert generator._data[0].rationale == "print('Successful code')"+ assert generator._data[0].final_answer == "Verified output"@pytest.mark.asyncioasync def test_generate_new_validation_error(self, generator):@@ -330,6 +333,9 @@ class TestGenerateNew:side_effect=["print('Failed code')", "print('Successful code')"])+ # Clear any existing data+ generator._data = []+# Use a custom DataPoint creation function to simulate validation# error then successoriginal_datapoint = DataPoint@@ -361,12 +367,12 @@ class TestGenerateNew:'camel.datasets.self_instruct_generator.DataPoint',side_effect=mock_datapoint_factory,):- result = await generator.generate_new(+ await generator.generate_new(n=1, max_retries=3) # Reduce retries for test speed# Verify generated datapoint- assert len(result) == 1- assert result[0].question == "Successful question?"- assert result[0].rationale == "print('Successful code')"- assert result[0].final_answer == "Verified output"+ assert len(generator._data) == 1+ assert generator._data[0].question == "Successful question?"+ assert generator._data[0].rationale == "print('Successful code')"+ assert generator._data[0].final_answer == "Verified output"diff --git a/test/environments/test_single_stage_env.py b/test/environments/test_single_stage_env.pyindex be44429e4..607d6ff70 100644--- a/test/environments/test_single_stage_env.py+++ b/test/environments/test_single_stage_env.py@@ -12,12 +12,12 @@# limitations under the License.# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========import re-from typing import Dict+from typing import Dict, Listfrom unittest.mock import AsyncMock, MagicMockimport pytest-from camel.datasets import StaticDataset+from camel.datasets import BaseGenerator, DataPoint, StaticDatasetfrom camel.environments import (Action,Observation,@@ -35,6 +35,108 @@ class MockSingleStepEnv(SingleStepEnv):return {"custom_reward": 5}+class MockGenerator(BaseGenerator):+ def __init__(self, predefined_data: List[Dict], *args, **kwargs):+ if 'puffer' not in kwargs:+ kwargs['puffer'] = 1+ super().__init__(*args, **kwargs)+ self.predefined_data = predefined_data+ self.index = 0++ async def generate_new(self, n: int, **kwargs) -> None:+ if self.index + n > len(self.predefined_data):+ raise ValueError("Not enough predefined datapoints")+ new_points = [+ DataPoint(**data)+ for data in self.predefined_data[self.index : self.index + n]+ ]+ self._data.extend(new_points)+ self.index += n+++async def test_single_step_env_lifecycle_single_generator():+ predefined_data = [+ {+ "question": "What is 2 + 2?",+ "final_answer": "4",+ "rationale": "Adding 2 and 2 gives 4.",+ "metadata": {"difficulty": "easy"},+ },+ {+ "question": "What is the capital of France?",+ "final_answer": "Paris",+ "rationale": "Paris is known as the capital city of France.",+ "metadata": {"difficulty": "easy"},+ },+ ]+ dataset = MockGenerator(predefined_data, seed=42)++ mock_verifier = MagicMock()+ mock_verifier.setup = AsyncMock()+ mock_verifier.cleanup = AsyncMock()+ mock_verifier.verify_batch = AsyncMock(+ side_effect=lambda solutions, reference_answers, **kwargs: [+ VerificationResult(+ status=VerificationOutcome.SUCCESS,+ result="Verification successful",+ feedback="Correct",+ score=1.0,+ )+ for _ in solutions+ ]+ )++ env = MockSingleStepEnv(dataset=dataset, verifier=mock_verifier)++ await env.setup()+ assert env._is_setup is True+ mock_verifier.setup.assert_awaited_once()++ # Test reset with batch_size=1+ observation = await env.reset(batch_size=1)+ assert isinstance(observation, Observation)+ assert observation.question == "What is 2 + 2?" # First datapoint++ # Test step with a single action+ action = Action(index=0, llm_response="4")+ result = await env.step(action)+ assert isinstance(result, tuple)+ next_obs, reward, done, info = result+ assert next_obs == env.PLACEHOLDER_OBS+ assert reward == env.ACCURACY_REWARD + 5+ assert done is True+ assert info["rewards_dict"].get("custom_reward", None) == 5++ # Test that stepping again without reset fails+ with pytest.raises(+ RuntimeError,+ match=r"Episodes have ended for batch\. Call reset\(\) first\.",+ ):+ await env.step(action)++ # Test reset and step again+ observation2 = await env.reset(batch_size=1)+ assert isinstance(observation2, Observation)+ assert (+ observation2.question == "What is the capital of France?"+ ) # Second datapoint+ action2 = Action(index=None, llm_response="Paris")+ result2 = await env.step(action2)+ next_obs2, reward2, done2, info2 = result2+ assert next_obs2 == env.PLACEHOLDER_OBS+ assert reward2 == env.ACCURACY_REWARD + 5+ assert done2 is True+ assert info2["rewards_dict"].get("custom_reward", None) == 5++ # Note: Skipping deterministic sampling test as BaseGenerator+ # doesn't use seed in the same way+ # Test close+ await env.close()+ assert env._is_setup is False+ mock_verifier.cleanup.assert_awaited_once()++@pytest.mark.asyncioasync def test_single_step_env_lifecycle_single():# Define a sample dataset@@ -257,6 +359,106 @@ async def test_batched_single_step_env_lifecycle():mock_verifier.cleanup.assert_awaited_once()+async def test_batched_single_step_env_lifecycle_generator():+ predefined_data = [+ {+ "question": "What is 2 + 2?",+ "final_answer": "4",+ "rationale": "Adding 2 and 2 gives 4.",+ "metadata": {"difficulty": "easy"},+ },+ {+ "question": "What is the capital of France?",+ "final_answer": "Paris",+ "rationale": "Paris is known as the capital city of France.",+ "metadata": {"difficulty": "easy"},+ },+ {+ "question": "Who wrote 'Romeo and Juliet'?",+ "final_answer": "Shakespeare",+ "rationale": "William Shakespeare is the author of 'Romeo and "+ "Juliet'.",+ "metadata": {"difficulty": "medium"},+ },+ {+ "question": "What is the boiling point of water in Celsius?",+ "final_answer": "100",+ "rationale": "Water boils at 100 degrees Celsius at standard "+ "pressure.",+ "metadata": {"difficulty": "easy"},+ },+ ]+ dataset = MockGenerator(predefined_data, seed=42)++ # Create a mock verifier+ mock_verifier = MagicMock()+ mock_verifier.setup = AsyncMock()+ mock_verifier.cleanup = AsyncMock()+ mock_verifier.verify_batch = AsyncMock(+ side_effect=lambda solutions, reference_answers, **kwargs: [+ VerificationResult(+ status=VerificationOutcome.SUCCESS,+ result="Verification successful",+ feedback="Correct",+ score=1.0,+ )+ for _ in solutions+ ]+ )++ # Initialize the environment+ env = MockSingleStepEnv(dataset=dataset, verifier=mock_verifier)++ # Test setup+ await env.setup()+ assert env._is_setup is True+ mock_verifier.setup.assert_awaited_once()++ # Test reset with batch_size=4+ observations = await env.reset(batch_size=4)+ assert isinstance(observations, list)+ assert len(observations) == 4+ expected_questions = [dp["question"] for dp in predefined_data]+ for i, obs in enumerate(observations):+ assert isinstance(obs, Observation)+ assert obs.question == expected_questions[i]++ # Test step with 2 actions+ actions = [+ Action(index=0, llm_response="4"), # First question+ Action(index=2, llm_response="Shakespeare"), # Third question+ ]+ results = await env.step(actions)+ assert isinstance(results, list)+ assert len(results) == 2+ for next_obs, reward, done, info in results:+ assert next_obs == env.PLACEHOLDER_OBS+ assert reward == env.ACCURACY_REWARD + 5+ assert done is True+ assert info["rewards_dict"].get("custom_reward", None) == 5++ # Test step with remaining 2 actions+ actions = [+ Action(index=1, llm_response="Paris"), # Second question+ Action(index=3, llm_response="100"), # Fourth question+ ]+ results = await env.step(actions)+ assert isinstance(results, list)+ assert len(results) == 2+ for next_obs, reward, done, info in results:+ assert next_obs == env.PLACEHOLDER_OBS+ assert reward == env.ACCURACY_REWARD + 5+ assert done is True+ assert info["rewards_dict"].get("custom_reward", None) == 5++ # Test batch completion+ assert env._batch_done()+ await env.close()+ assert env._is_setup is False+ mock_verifier.cleanup.assert_awaited_once()++def create_mock_verifier():verifier = MagicMock()verifier.setup = AsyncMock()@@ -320,8 +522,16 @@ async def test_single_step_env_error_handling_single():await env_fail_verifier.setup()await env_fail_verifier.reset(batch_size=1)action = Action(index=0, llm_response="4")- with pytest.raises(Exception, match="Verifier error"):- await env_fail_verifier.step(action)++ step_result = await env_fail_verifier.step(action)++ _, reward, _, info = step_result++ # Verify that the verification result indicates a failure+ assert info["verification_result"].status == VerificationOutcome.FAILURE+ assert "Verifier error" in info["verification_result"].error_message+ # Verify that the reward reflects the failure+ assert reward == 0.0# **3. Test State Mismanagement Scenarios**# a) Step without setup@@ -506,7 +716,7 @@ async def test_batched_single_step_env_error_handling():dataset = StaticDataset(data)# **2. Test Faulty Verifier**- # Ensure errors in verifier are propagated+ # Ensure errors in verifier are properly handledmock_verifier_exception = MagicMock()mock_verifier_exception.setup = AsyncMock()mock_verifier_exception.cleanup = AsyncMock()@@ -520,8 +730,17 @@ async def test_batched_single_step_env_error_handling():await env_fail_verifier.setup()await env_fail_verifier.reset(batch_size=1)action = Action(index=0, llm_response="4")- with pytest.raises(Exception, match="Verifier error"):- await env_fail_verifier.step(action)++ step_result = await env_fail_verifier.step(action)++ # Unpack step result+ _, reward, _, info = step_result++ # Verify that the verification result indicates a failure+ assert info["verification_result"].status == VerificationOutcome.FAILURE+ assert "Verifier error" in info["verification_result"].error_message+ # Verify that the reward reflects the failure+ assert reward == 0.0# **3. Test State Mismanagement Scenarios**# a) Step without setup