🐫 CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org
Python agentai-societiesartificial-intelligencecommunicative-aicooperative-aideep-learninglarge-language-modelsmulti-agent-systemsnatural-language-processing
enhance: SingleStepEnv add timeout (#2116)
Wendong-Fanddd05fffceb7
diff --git a/camel/environments/single_step.py b/camel/environments/single_step.pyindex fd8547d4b..d490dceb9 100644--- a/camel/environments/single_step.py+++ b/camel/environments/single_step.py@@ -12,6 +12,7 @@# limitations under the License.# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+import asyncioimport randomfrom typing import Any, Dict, List, Optional, Tuple, Union@@ -58,6 +59,7 @@ class SingleStepEnv:self,dataset: Union[StaticDataset, BaseGenerator],verifier: BaseVerifier,+ timeout: Optional[float] = 180.0,**kwargs,) -> None:r"""Initialize the SingleStepEnv.@@ -67,12 +69,15 @@ class SingleStepEnv:problems from.verifier (BaseVerifier): Verifier used to evaluate LLM responsesagainst ground-truth answers.+ timeout (Optional[float], optional): The execution timeout in+ seconds. (default: :obj:`180.0`)**kwargs: Optional metadata or configuration values.Notes:This class assumes all interactions are single-step: one question,one LLM response, one reward."""+ self._timeout = timeoutself.dataset = datasetself.verifier = verifierself._metadata = kwargs@@ -279,6 +284,7 @@ class SingleStepEnv:or if `reset()` has not been called.ValueError: If invalid action format, duplicate indices,or out-of-bounds indices are detected.+ asyncio.TimeoutError: If the step execution exceeds the timeout."""if not self._is_setup:@@ -307,18 +313,34 @@ class SingleStepEnv:f"total batch size ({self.current_batch_size})")- indices = [act.index for act in actions]proposed_solutions = [act.llm_response for act in actions]- ground_truths: List[str] = []- for idx in indices:- ground_truths.append(self._states[idx].final_answer)+ ground_truths: List[str] = [+ self._states[idx].final_answer for idx in indices+ ]try:- verification_results = await self.verifier.verify_batch(- solutions=proposed_solutions,- reference_answers=ground_truths, # type: ignore [arg-type]- raise_on_error=True,+ verification_results = await asyncio.wait_for(+ self.verifier.verify_batch(+ solutions=proposed_solutions,+ reference_answers=ground_truths, # type: ignore [arg-type]+ raise_on_error=True,+ ),+ timeout=self._timeout,+ )+ except asyncio.TimeoutError as e:+ logger.error(+ f"Step verification timed out after {self._timeout}s: {e}")+ # Return timeout verification results+ verification_results = [+ VerificationResult(+ result="",+ status=VerificationOutcome.TIMEOUT,+ error_message=f"Verification timed out "+ f"after {self._timeout}s",+ )+ for _ in range(len(proposed_solutions))+ ]except Exception as e:logger.error(f"Verification failed: {e}")# Return failed verification results with status=FAILURE@@ -331,9 +353,55 @@ class SingleStepEnv:for _ in range(len(proposed_solutions))]- total_rewards, rewards_dicts = await self._compute_reward_batch(- proposed_solutions, verification_results- )+ # Track which solutions have been processed and which have timed out+ total_rewards = [0.0] * len(proposed_solutions)+ rewards_dicts = [{"correctness": 0.0}] * len(proposed_solutions)++ try:+ # First try to compute all rewards with a timeout+ computed_rewards, computed_rewards_dicts = await asyncio.wait_for(+ self._compute_reward_batch(+ proposed_solutions, verification_results+ ),+ timeout=self._timeout,+ )+ # If successful, use all the computed values+ total_rewards = computed_rewards+ rewards_dicts = computed_rewards_dicts+ except asyncio.TimeoutError as e:+ logger.error(+ f"Reward computation timed out after {self._timeout}s: {e}"+ )+ # Try to compute rewards one by one to identify which ones time out+ for i, (solution, result) in enumerate(+ zip(proposed_solutions, verification_results)+ ):+ try:+ individual_rewards = await asyncio.wait_for(+ self._compute_custom_reward(solution, result),+ timeout=self._timeout,+ )+ # If successful, calculate the reward for this solution+ correctness_reward = (+ self.ACCURACY_REWARD if result.status else 0.0+ )+ rewards_dict = {+ "correctness": correctness_reward,+ **individual_rewards,+ }+ total_rewards[i] = sum(rewards_dict.values())+ rewards_dicts[i] = rewards_dict+ except asyncio.TimeoutError:+ logger.warning(+ f"Reward computation for solution {i} timed out"+ )+ except Exception as e:+ logger.warning(+ f"Error computing reward for solution {i}: {e}"+ )+ except Exception as e:+ logger.error(f"Reward computation failed: {e}")+# Create and return step results in batchstep_results = [StepResult(