Skip to content

armgit / camel

Public Imported from GitHub
0

🐫 CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org

feat: Adds test for reset Method for BaseGenerator dataset (#3319)

Tanuj Taneja64bf049ce98e
diff --git a/test/environments/test_single_step_env.py b/test/environments/test_single_step_env.py
index 64827b7b0..dc3a552b5 100644
--- a/test/environments/test_single_step_env.py
+++ b/test/environments/test_single_step_env.py
@@ -867,6 +867,78 @@ async def test_batched_single_step_env_error_handling():
await env_close_fail.close()
+def get_dummy_predefined_data(n):
+ predefined_data = []
+ for i in range(n):
+ predefined_data.append(
+ {
+ "question": f"What is {i} + {i}?",
+ "final_answer": str(i + i),
+ "rationale": f"Adding {i} and {i} gives {i + i}.",
+ "metadata": {"difficulty": "easy"},
+ }
+ )
+ return predefined_data
+
+
+async def test_basegenerator_reset_reproducibility_with_identical_seed():
+ """Verify that resetting the environment with the same random seed
+ produces identical sampling order across consecutive resets."""
+
+ PREDEFINED_DATA_SIZE = 128
+ RESET_BATCH_SIZE = 64
+
+ predefined_data = get_dummy_predefined_data(PREDEFINED_DATA_SIZE)
+ mock_verifier = create_mock_verifier()
+ dataset = MockGenerator(predefined_data, seed=42)
+ env = MockSingleStepEnv(dataset=dataset, verifier=mock_verifier)
+ await env.setup()
+
+ # First reset with seed=42
+ observations1 = await env.reset(batch_size=RESET_BATCH_SIZE, seed=42)
+ questions1 = [obs.question for obs in observations1]
+
+ # Second reset with same seed=42 should produce same order
+ observations2 = await env.reset(batch_size=RESET_BATCH_SIZE, seed=42)
+ questions2 = [obs.question for obs in observations2]
+
+ # Same seed should produce same sampling order
+ # given the large batch size)
+ assert (
+ questions1 == questions2
+ ), "Same seed should produce same sampling order"
+ await env.close()
+
+
+async def test_basegenerator_reset_variability_with_different_seeds():
+ """Verify that resetting the environment with different random seeds
+ produces distinct sampling orders, confirming stochastic behavior."""
+
+ PREDEFINED_DATA_SIZE = 128
+ RESET_BATCH_SIZE = 64
+
+ predefined_data = get_dummy_predefined_data(PREDEFINED_DATA_SIZE)
+
+ dataset = MockGenerator(predefined_data, seed=42)
+ env = MockSingleStepEnv(dataset=dataset, verifier=create_mock_verifier())
+ await env.setup()
+
+ observations1 = await env.reset(batch_size=RESET_BATCH_SIZE, seed=42)
+ questions1 = [obs.question for obs in observations1]
+
+ observations2 = await env.reset(batch_size=RESET_BATCH_SIZE, seed=43)
+ questions2 = [obs.question for obs in observations2]
+
+ # Different seeds should produce different orders
+ # given the large batch size
+ assert (
+ questions1 != questions2
+ ), "Different seeds should produce different sampling orders"
+ await env.close()
+
+
@pytest.mark.asyncio
async def test_single_step_env_all_input_types():
# Define a simple dataset