🐫 CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org
Python agentai-societiesartificial-intelligencecommunicative-aicooperative-aideep-learninglarge-language-modelsmulti-agent-systemsnatural-language-processing
feat: Adds test for reset Method for BaseGenerator dataset (#3319)
Tanuj Taneja64bf049ce98e
diff --git a/test/environments/test_single_step_env.py b/test/environments/test_single_step_env.pyindex 64827b7b0..dc3a552b5 100644--- a/test/environments/test_single_step_env.py+++ b/test/environments/test_single_step_env.py@@ -867,6 +867,78 @@ async def test_batched_single_step_env_error_handling():await env_close_fail.close()+def get_dummy_predefined_data(n):+ predefined_data = []+ for i in range(n):+ predefined_data.append(+ {+ "question": f"What is {i} + {i}?",+ "final_answer": str(i + i),+ "rationale": f"Adding {i} and {i} gives {i + i}.",+ "metadata": {"difficulty": "easy"},+ }+ )+ return predefined_data+++async def test_basegenerator_reset_reproducibility_with_identical_seed():+ """Verify that resetting the environment with the same random seed+ produces identical sampling order across consecutive resets."""++ PREDEFINED_DATA_SIZE = 128+ RESET_BATCH_SIZE = 64++ predefined_data = get_dummy_predefined_data(PREDEFINED_DATA_SIZE)+ mock_verifier = create_mock_verifier()+ dataset = MockGenerator(predefined_data, seed=42)+ env = MockSingleStepEnv(dataset=dataset, verifier=mock_verifier)+ await env.setup()++ # First reset with seed=42+ observations1 = await env.reset(batch_size=RESET_BATCH_SIZE, seed=42)+ questions1 = [obs.question for obs in observations1]++ # Second reset with same seed=42 should produce same order+ observations2 = await env.reset(batch_size=RESET_BATCH_SIZE, seed=42)+ questions2 = [obs.question for obs in observations2]++ # Same seed should produce same sampling order+ # given the large batch size)+ assert (+ questions1 == questions2+ ), "Same seed should produce same sampling order"+ await env.close()+++async def test_basegenerator_reset_variability_with_different_seeds():+ """Verify that resetting the environment with different random seeds+ produces distinct sampling orders, confirming stochastic behavior."""++ PREDEFINED_DATA_SIZE = 128+ RESET_BATCH_SIZE = 64++ predefined_data = get_dummy_predefined_data(PREDEFINED_DATA_SIZE)++ dataset = MockGenerator(predefined_data, seed=42)+ env = MockSingleStepEnv(dataset=dataset, verifier=create_mock_verifier())+ await env.setup()++ observations1 = await env.reset(batch_size=RESET_BATCH_SIZE, seed=42)+ questions1 = [obs.question for obs in observations1]++ observations2 = await env.reset(batch_size=RESET_BATCH_SIZE, seed=43)+ questions2 = [obs.question for obs in observations2]++ # Different seeds should produce different orders+ # given the large batch size+ assert (+ questions1 != questions2+ ), "Different seeds should produce different sampling orders"+ await env.close()++@pytest.mark.asyncioasync def test_single_step_env_all_input_types():# Define a simple dataset