🐫 CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org
Python agentai-societiesartificial-intelligencecommunicative-aicooperative-aideep-learninglarge-language-modelsmulti-agent-systemsnatural-language-processing
[Feature Request] Enhancing robustness of Async operations in core modules #2468 (#2551)
Bonian Hud75ab1b634c9
diff --git a/camel/environments/multi_step.py b/camel/environments/multi_step.pyindex e73b2af6d..1d4645cb2 100644--- a/camel/environments/multi_step.py+++ b/camel/environments/multi_step.py@@ -12,12 +12,14 @@# limitations under the License.# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+import asynciofrom abc import ABC, abstractmethodfrom typing import Any, Dict, List, Optional, Tuplefrom camel.environments.models import Action, Observation, StepResultfrom camel.extractors.base import BaseExtractorfrom camel.logger import get_logger+from camel.utils import TIMEOUT_THRESHOLD, with_timeout_asynclogger = get_logger(__name__)@@ -29,6 +31,7 @@ class MultiStepEnv(ABC):self,extractor: BaseExtractor,max_steps: Optional[int] = None,+ timeout: Optional[float] = TIMEOUT_THRESHOLD,**kwargs,) -> None:r"""Initialize the environment.@@ -36,10 +39,12 @@ class MultiStepEnv(ABC):Args:extractor: Extractor to process LLM responses.max_steps: Maximum steps per episode.+ timeout: The execution timeout in seconds.**kwargs: Additional environment parameters."""self.extractor = extractorself.max_steps = max_steps+ self._timeout = timeoutself._metadata = kwargs# State tracking@@ -109,13 +114,18 @@ class MultiStepEnv(ABC):Raises:RuntimeError: If we fail to get the initial observation.+ asyncio.TimeoutError: If the environment setup times out."""if not self._is_setup:logger.warning("reset() called on un-setup environment. Setting up...")- await self.setup()+ await with_timeout_async(+ self.setup(),+ timeout=self._timeout,+ context="environment setup",+ )# Reset stateself._current_step = 0@@ -151,6 +161,7 @@ class MultiStepEnv(ABC):Raises:RuntimeError: If the environment is not set up, the episode hasended, or there is no valid current observation.+ asyncio.TimeoutError: If the step operation times out."""if self.max_steps and self._current_step >= self.max_steps:return StepResult(@@ -173,11 +184,46 @@ class MultiStepEnv(ABC):current_obs: Observation = self._last_observationself._episode_history.append((current_obs, action))+ # Initialize info dictionary+ info = {}+# Update the environment state based on the action- await self._update_state(action)+ update_timed_out = False+ try:+ await with_timeout_async(+ self._update_state(action),+ timeout=self._timeout,+ context="state update",+ )+ except asyncio.TimeoutError:+ update_timed_out = True++ # If update timed out, return error information+ if update_timed_out:+ return StepResult(+ observation=self._last_observation, # Use last observation+ reward=0.0, # Default reward+ rewards_dict={}, # No rewards on timeout+ done=False, # Episode not done+ info={+ "error": f"State update timed out after {self._timeout}s"+ },+ ).as_tuple()# Compute rewards- total_reward, rewards_dict = await self.compute_reward()+ try:+ total_reward, rewards_dict = await with_timeout_async(+ self.compute_reward(),+ timeout=self._timeout,+ context="reward computation",+ )+ except asyncio.TimeoutError:+ # raise exception to terminate the step operation+ total_reward = 0.0+ rewards_dict = {}+ info["error"] = (+ f"Reward computation timed out after {self._timeout}s"+ )# Check terminationdone = self.is_done()@@ -192,18 +238,38 @@ class MultiStepEnv(ABC):self._last_observation = next_obsself._episode_ended = done+ extraction_result = None+ try:+ extraction_result = await with_timeout_async(+ self.extractor.extract(action.llm_response),+ timeout=self._timeout,+ context="extraction",+ )+ except asyncio.TimeoutError:+ # raise exception to terminate the step operation+ extraction_result = None+ if info is None:+ info = {}+ info["error"] = f"Extraction timed out after {self._timeout}s"++ # If we have no info dict yet (no errors occurred), create one+ if info is None:+ info = {}++ # Add extraction result to info if available and no error occurred+ if "error" not in info and extraction_result is not None:+ info["extraction_result"] = extraction_result++ # Add step and state info+ info["step"] = str(self._current_step)+ info["state"] = str(self._state) # Updated state+return StepResult(observation=next_obs,reward=total_reward,rewards_dict=rewards_dict,done=done,- info={- "extraction_result": await self.extractor.extract(- action.llm_response- ),- "step": self._current_step,- "state": self._state, # Updated state- },+ info=info,).as_tuple()@abstractmethoddiff --git a/camel/environments/single_step.py b/camel/environments/single_step.pyindex d490dceb9..c5325c10f 100644--- a/camel/environments/single_step.py+++ b/camel/environments/single_step.py@@ -18,6 +18,7 @@ from typing import Any, Dict, List, Optional, Tuple, Unionfrom camel.datasets import BaseGenerator, DataPoint, StaticDatasetfrom camel.logger import get_logger+from camel.utils import TIMEOUT_THRESHOLD, with_timeout_asyncfrom camel.verifiers.base import (BaseVerifier,VerificationOutcome,@@ -59,7 +60,7 @@ class SingleStepEnv:self,dataset: Union[StaticDataset, BaseGenerator],verifier: BaseVerifier,- timeout: Optional[float] = 180.0,+ timeout: Optional[float] = TIMEOUT_THRESHOLD,**kwargs,) -> None:r"""Initialize the SingleStepEnv.@@ -77,7 +78,7 @@ class SingleStepEnv:This class assumes all interactions are single-step: one question,one LLM response, one reward."""- self._timeout = timeout+ self._timeout: Optional[float] = timeoutself.dataset = datasetself.verifier = verifierself._metadata = kwargs@@ -163,6 +164,8 @@ class SingleStepEnv:RuntimeError: If called before all previous states are processed.ValueError: If batch size exceeds dataset size.TypeError: If the dataset is of an unsupported type.+ asyncio.TimeoutError: If the environment setup times out or+ dataset sampling times out."""if batch_size <= 0:raise ValueError("Batch size must be positive")@@ -171,7 +174,11 @@ class SingleStepEnv:logger.warning("reset() called on un-setup environment. Setting up...")- await self.setup()+ await with_timeout_async(+ self.setup(),+ timeout=self._timeout,+ context="environment setup",+ )if self._batch_started() and not self._batch_done():logger.error(@@ -218,11 +225,20 @@ class SingleStepEnv:return observations[0] if batch_size == 1 else observationselif isinstance(self.dataset, BaseGenerator):- self._states = [- await self.dataset.async_sample() for _ in range(batch_size)- ]- self.current_batch_size = batch_size- self._states_done = [False] * batch_size+ self._states = []+ for _ in range(batch_size):+ try:+ sample = await with_timeout_async(+ self.dataset.async_sample(),+ timeout=self._timeout,+ context="dataset sampling",+ )+ self._states.append(sample)+ except asyncio.TimeoutError:+ raise++ self.current_batch_size = len(self._states)+ self._states_done = [False] * self.current_batch_sizeobservations = [Observation(@@ -319,18 +335,16 @@ class SingleStepEnv:]try:- verification_results = await asyncio.wait_for(+ verification_results = await with_timeout_async(self.verifier.verify_batch(solutions=proposed_solutions,reference_answers=ground_truths, # type: ignore [arg-type]raise_on_error=True,),timeout=self._timeout,+ context="step verification",)- except asyncio.TimeoutError as e:- logger.error(- f"Step verification timed out after {self._timeout}s: {e}"- )+ except asyncio.TimeoutError:# Return timeout verification resultsverification_results = [VerificationResult(@@ -359,27 +373,29 @@ class SingleStepEnv:try:# First try to compute all rewards with a timeout- computed_rewards, computed_rewards_dicts = await asyncio.wait_for(+ (+ computed_rewards,+ computed_rewards_dicts,+ ) = await with_timeout_async(self._compute_reward_batch(proposed_solutions, verification_results),timeout=self._timeout,+ context="batch reward computation",)# If successful, use all the computed valuestotal_rewards = computed_rewardsrewards_dicts = computed_rewards_dicts- except asyncio.TimeoutError as e:- logger.error(- f"Reward computation timed out after {self._timeout}s: {e}"- )+ except asyncio.TimeoutError:# Try to compute rewards one by one to identify which ones time outfor i, (solution, result) in enumerate(zip(proposed_solutions, verification_results)):try:- individual_rewards = await asyncio.wait_for(+ individual_rewards = await with_timeout_async(self._compute_custom_reward(solution, result),timeout=self._timeout,+ context=f"reward computation for solution {i}",)# If successful, calculate the reward for this solutioncorrectness_reward = (@@ -534,6 +550,9 @@ class SingleStepEnv:Tuple containing:- List of total rewards for each solution.- List of reward component dictionaries for each solution.++ Raises:+ asyncio.TimeoutError: If the custom reward computation times out."""if len(proposed_solutions) != len(verification_results):raise ValueError(@@ -553,10 +572,21 @@ class SingleStepEnv:self.ACCURACY_REWARD if verification_result.status else 0.0)- further_rewards = await self._compute_custom_reward(- solution, verification_result- )- rewards = {**rewards, **further_rewards}+ try:+ # add timeout control+ further_rewards = await with_timeout_async(+ self._compute_custom_reward(solution, verification_result),+ timeout=self._timeout,+ )+ rewards = {**rewards, **further_rewards}+ except asyncio.TimeoutError:+ logger.warning(+ f"Custom reward computation timed out after\+ {self._timeout}s"+ )+ # use empty dictionary as fallback strategy+ further_rewards = {}+ rewards = {**rewards, **further_rewards} # only basic rewardstotal_reward = sum(rewards.values())total_rewards.append(total_reward)diff --git a/camel/environments/tic_tac_toe.py b/camel/environments/tic_tac_toe.pyindex be6f17896..6474c93f3 100644--- a/camel/environments/tic_tac_toe.py+++ b/camel/environments/tic_tac_toe.py@@ -11,6 +11,7 @@# See the License for the specific language governing permissions and# limitations under the License.# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+import asyncioimport mathimport randomimport re@@ -19,6 +20,10 @@ from typing import Any, ClassVar, Dict, List, Literal, Optional, Tuplefrom camel.environments.models import Action, Observationfrom camel.environments.multi_step import MultiStepEnvfrom camel.extractors import BaseExtractor, BaseExtractorStrategy+from camel.logger import get_logger+from camel.utils import with_timeout_async++logger = get_logger(__name__)class MoveExtractor(BaseExtractorStrategy):@@ -254,11 +259,26 @@ class TicTacToeEnv(MultiStepEnv):Returns:None++ Raises:+ asyncio.TimeoutError: If the move extraction times out."""board = self._state["board"]# Attempt to parse the agent's chosen move- extraction_result = await self.extractor.extract(action.llm_response)+ try:+ extraction_result = await with_timeout_async(+ self.extractor.extract(action.llm_response),+ timeout=self._timeout,+ context="move extraction",+ )+ except asyncio.TimeoutError:+ # Handle timeout gracefully+ self._state["last_move_illegal"] = True+ self._state["last_move"] = None+ self._state["extraction_error"] = "Extraction timed out"+ return+if not extraction_result:# Handle extraction failure gracefullyself._state["last_move_illegal"] = Truediff --git a/camel/societies/role_playing.py b/camel/societies/role_playing.pyindex 3df870376..fc00fd136 100644--- a/camel/societies/role_playing.py+++ b/camel/societies/role_playing.py@@ -13,7 +13,14 @@# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========import loggingimport threading-from typing import Dict, List, Optional, Sequence, Tuple, Union+from typing import (+ Dict,+ List,+ Optional,+ Sequence,+ Tuple,+ Union,+)from camel.agents import (ChatAgent,@@ -28,6 +35,7 @@ from camel.models import BaseModelBackendfrom camel.prompts import TextPromptfrom camel.responses import ChatAgentResponsefrom camel.types import RoleType, TaskType+from camel.utils import with_timeout_asynclogger = logging.getLogger(__name__)logger.setLevel(logging.WARNING)@@ -627,7 +635,12 @@ class RolePlaying:user agent terminated the conversation, and any additional userinformation."""- user_response = await self.user_agent.astep(assistant_msg)+ user_response = await with_timeout_async(+ self.user_agent.astep(assistant_msg),+ context=f"waiting for user agent \+ {self.user_agent.role_name} response",+ )+if user_response.terminated or user_response.msgs is None:return (ChatAgentResponse(msgs=[], terminated=False, info={}),@@ -648,7 +661,12 @@ class RolePlaying:):self.user_agent.record_message(user_msg)- assistant_response = await self.assistant_agent.astep(user_msg)+ assistant_response = await with_timeout_async(+ self.assistant_agent.astep(user_msg),+ context=f"waiting for assistant agent \+ {self.assistant_agent.role_name} response",+ )+if assistant_response.terminated or assistant_response.msgs is None:return (ChatAgentResponse(diff --git a/camel/societies/workforce/role_playing_worker.py b/camel/societies/workforce/role_playing_worker.pyindex d437cc84a..1aca0a6b1 100644--- a/camel/societies/workforce/role_playing_worker.py+++ b/camel/societies/workforce/role_playing_worker.py@@ -13,12 +13,14 @@# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========from __future__ import annotations+import asyncioimport jsonfrom typing import Dict, List, Optionalfrom colorama import Forefrom camel.agents.chat_agent import ChatAgent+from camel.logger import get_loggerfrom camel.messages.base import BaseMessagefrom camel.societies import RolePlayingfrom camel.societies.workforce.prompts import (@@ -28,7 +30,10 @@ from camel.societies.workforce.prompts import (from camel.societies.workforce.utils import TaskResultfrom camel.societies.workforce.worker import Workerfrom camel.tasks.task import Task, TaskState, validate_task_content-from camel.utils import print_text_animated+from camel.utils import TIMEOUT_THRESHOLD, print_text_animated++# Set up logging+logger = get_logger(__name__)class RolePlayingWorker(Worker):@@ -82,7 +87,10 @@ class RolePlayingWorker(Worker):self.user_agent_kwargs = user_agent_kwargsasync def _process_task(- self, task: Task, dependencies: List[Task]+ self,+ task: Task,+ dependencies: List[Task],+ timeout: Optional[float] = TIMEOUT_THRESHOLD,) -> TaskState:r"""Processes a task leveraging its dependencies through role-playing.@@ -101,6 +109,10 @@ class RolePlayingWorker(Worker):Returns:TaskState: `TaskState.DONE` if processed successfully, otherwise`TaskState.FAILED`.++ Raises:+ Exception: If an error occurs during role-playing.+ asyncio.TimeoutError: If the role-playing session times out."""dependency_tasks_info = self._get_dep_tasks_info(dependencies)prompt = ROLEPLAY_PROCESS_TASK_PROMPT.format(@@ -121,9 +133,19 @@ class RolePlayingWorker(Worker):chat_history = []while n < self.chat_turn_limit:n += 1- assistant_response, user_response = await role_play_session.astep(- input_msg- )+ try:+ # add a timeout control+ assistant_response, user_response = await asyncio.wait_for(+ role_play_session.astep(input_msg), timeout=timeout+ )+ except asyncio.TimeoutError:+ logger.error("Error: Role playing step timed out.")+ chat_history.append("Error: Role playing step timed out.")+ break+ except Exception as e:+ logger.error(f"Error during role playing step: {e!s}")+ chat_history.append(f"Error during role playing: {e!s}")+ breakif assistant_response.terminated:reason = assistant_response.info['termination_reasons']@@ -160,6 +182,7 @@ class RolePlayingWorker(Worker):f"{Fore.RESET}\n",delay=0.005,)+chat_history.append(f"AI Assistant: {assistant_response.msg.content}")diff --git a/camel/societies/workforce/task_channel.py b/camel/societies/workforce/task_channel.pyindex 63a3cb19e..288fdf14a 100644--- a/camel/societies/workforce/task_channel.py+++ b/camel/societies/workforce/task_channel.py@@ -16,6 +16,7 @@ from enum import Enumfrom typing import Dict, List, Optionalfrom camel.tasks import Task+from camel.utils import TIMEOUT_THRESHOLD, with_timeout_asyncclass PacketStatus(Enum):@@ -78,14 +79,25 @@ class Packet:class TaskChannel:r"""An internal class used by Workforce to manage tasks."""- def __init__(self) -> None:+ def __init__(self, timeout: float = TIMEOUT_THRESHOLD) -> None:self._task_id_list: List[str] = []self._condition = asyncio.Condition()self._task_dict: Dict[str, Packet] = {}+ self._timeout = timeout- async def get_returned_task_by_publisher(self, publisher_id: str) -> Task:+ async def get_returned_task_by_publisher(+ self, publisher_id: str, timeout: float = TIMEOUT_THRESHOLD+ ) -> Task:r"""Get a task from the channel that has been returned by thepublisher.++ Args:+ publisher_id (str): The ID of the publisher+ timeout (float): Maximum time to wait in seconds+ (default: :obj:`180.0`)++ Raises:+ asyncio.TimeoutError: If waiting times out"""async with self._condition:while True:@@ -96,11 +108,27 @@ class TaskChannel:if packet.status != PacketStatus.RETURNED:continuereturn packet.task- await self._condition.wait()- async def get_assigned_task_by_assignee(self, assignee_id: str) -> Task:+ await with_timeout_async(+ self._condition.wait(),+ timeout=timeout,+ context=f"waiting for returned task from publisher\+ {publisher_id}",+ )++ async def get_assigned_task_by_assignee(+ self, assignee_id: str, timeout: float = TIMEOUT_THRESHOLD+ ) -> Task:r"""Get a task from the channel that has been assigned to theassignee.++ Args:+ assignee_id (str): The ID of the assignee+ timeout (float): Maximum time to wait in seconds+ (default: :obj:`180.0`)++ Raises:+ asyncio.TimeoutError: If waiting times out"""async with self._condition:while True:@@ -111,7 +139,13 @@ class TaskChannel:and packet.assignee_id == assignee_id):return packet.task- await self._condition.wait()++ await with_timeout_async(+ self._condition.wait(),+ timeout=timeout,+ context=f"waiting for assigned task to assignee\+ {assignee_id}",+ )async def post_task(self, task: Task, publisher_id: str, assignee_id: strdiff --git a/camel/societies/workforce/worker.py b/camel/societies/workforce/worker.pyindex 899e14eb8..00da0b363 100644--- a/camel/societies/workforce/worker.py+++ b/camel/societies/workforce/worker.py@@ -23,6 +23,7 @@ from camel.societies.workforce.base import BaseNodefrom camel.societies.workforce.task_channel import TaskChannelfrom camel.societies.workforce.utils import check_if_runningfrom camel.tasks.task import Task, TaskState+from camel.utils import with_timeout_asynclogger = logging.getLogger(__name__)@@ -61,7 +62,10 @@ class Worker(BaseNode, ABC):async def _get_assigned_task(self) -> Task:r"""Get the task assigned to this node from the channel."""- return await self._channel.get_assigned_task_by_assignee(self.node_id)+ return await with_timeout_async(+ self._channel.get_assigned_task_by_assignee(self.node_id),+ context=f"getting assigned task for {self.node_id}",+ )@staticmethoddef _get_dep_tasks_info(dependencies: List[Task]) -> str:@@ -90,30 +94,50 @@ class Worker(BaseNode, ABC):while True:# Get the earliest task assigned to this node- task = await self._get_assigned_task()+ task = await with_timeout_async(+ self._get_assigned_task(),+ context=f"getting assigned task for {self.node_id}",+ )+print(f"{Fore.YELLOW}{self} get task {task.id}: {task.content}"f"{Fore.RESET}")+# Get the Task instance of dependencies- dependency_ids = await self._channel.get_dependency_ids()- task_dependencies = [- await self._channel.get_task_by_id(dep_id)- for dep_id in dependency_ids- ]+ dependency_ids = await with_timeout_async(+ self._channel.get_dependency_ids(),+ context=f"getting dependency IDs for task {task.id}",+ )+ task_dependencies = []+ for dep_id in dependency_ids:+ dep_task = await with_timeout_async(+ self._channel.get_task_by_id(dep_id),+ context=f"getting dependency task {dep_id}",+ )+ task_dependencies.append(dep_task)# Process the task- task_state = await self._process_task(task, task_dependencies)+ task_state = await with_timeout_async(+ self._process_task(task, task_dependencies),+ context=f"processing task {task.id}",+ )# Update the result and status of the tasktask.set_state(task_state)- await self._channel.return_task(task.id)+ await with_timeout_async(+ self._channel.return_task(task.id),+ context=f"returning task {task.id}",+ )@check_if_running(False)async def start(self):r"""Start the worker."""- await self._listen_to_channel()+ await with_timeout_async(+ self._listen_to_channel(),+ context=f"listening to channel for {self.node_id}",+ )@check_if_running(True)def stop(self):diff --git a/camel/societies/workforce/workforce.py b/camel/societies/workforce/workforce.pyindex d82f356ca..d7fa6641f 100644--- a/camel/societies/workforce/workforce.py+++ b/camel/societies/workforce/workforce.py@@ -43,7 +43,7 @@ from camel.societies.workforce.worker import Workerfrom camel.tasks.task import Task, TaskState, validate_task_contentfrom camel.toolkits import CodeExecutionToolkit, SearchToolkit, ThinkingToolkitfrom camel.types import ModelPlatformType, ModelType-from camel.utils import dependencies_required+from camel.utils import dependencies_required, with_timeout_asynclogger = get_logger(__name__)@@ -375,10 +375,16 @@ class Workforce(BaseNode):return task_assign_result.assignee_idasync def _post_task(self, task: Task, assignee_id: str) -> None:- await self._channel.post_task(task, self.node_id, assignee_id)+ await with_timeout_async(+ self._channel.post_task(task, self.node_id, assignee_id),+ context=f"posting task {task.id} to assignee {assignee_id}",+ )async def _post_dependency(self, dependency: Task) -> None:- await self._channel.post_dependency(dependency, self.node_id)+ await with_timeout_async(+ self._channel.post_dependency(dependency, self.node_id),+ context=f"posting dependency {dependency.id}",+ )def _create_worker_node_for_task(self, task: Task) -> Worker:r"""Creates a new worker node for a given task and add it to the@@ -449,7 +455,10 @@ class Workforce(BaseNode):r"""Get the task that's published by this node and just get returnedfrom the assignee."""- return await self._channel.get_returned_task_by_publisher(self.node_id)+ return await with_timeout_async(+ self._channel.get_returned_task_by_publisher(self.node_id),+ context=f"getting returned task for publisher {self.node_id}",+ )async def _post_ready_tasks(self) -> None:r"""Send all the pending tasks that have all the dependencies met to@@ -470,40 +479,68 @@ class Workforce(BaseNode):ready_task.compose(self.task_agent)# Remove the subtasks from the channelfor subtask in ready_task.subtasks:- await self._channel.remove_task(subtask.id)+ await with_timeout_async(+ self._channel.remove_task(subtask.id),+ context=f"removing subtask {subtask.id} from channel",+ )# Send the task to the channel as a dependency- await self._post_dependency(ready_task)+ await with_timeout_async(+ self._post_dependency(ready_task),+ context=f"posting dependency task {ready_task.id}",+ )self._pending_tasks.popleft()# Try to send the next task in the pending list- await self._post_ready_tasks()+ await with_timeout_async(+ self._post_ready_tasks(), context="posting next ready task"+ )else:# Directly post the task to the channel if it's a new one# Find a node to assign the taskassignee_id = self._find_assignee(task=ready_task)- await self._post_task(ready_task, assignee_id)+ await with_timeout_async(+ self._post_task(ready_task, assignee_id),+ context=f"posting task {ready_task.id} \+ to assignee {assignee_id}",+ )async def _handle_failed_task(self, task: Task) -> bool:if task.failure_count >= 3:return Truetask.failure_count += 1# Remove the failed task from the channel- await self._channel.remove_task(task.id)+ await with_timeout_async(+ self._channel.remove_task(task.id),+ context=f"removing failed task {task.id} from channel",+ )if task.get_depth() >= 3:# Create a new worker node and reassignassignee = self._create_worker_node_for_task(task)- await self._post_task(task, assignee.node_id)+ await with_timeout_async(+ self._post_task(task, assignee.node_id),+ context=f"posting task {task.id} to new worker\+ {assignee.node_id}",+ )else:subtasks = self._decompose_task(task)# Insert packets at the head of the queueself._pending_tasks.extendleft(reversed(subtasks))- await self._post_ready_tasks()+ await with_timeout_async(+ self._post_ready_tasks(),+ context="posting ready tasks after task failure",+ )return Falseasync def _handle_completed_task(self, task: Task) -> None:# archive the packet, making it into a dependencyself._pending_tasks.popleft()- await self._channel.archive_task(task.id)- await self._post_ready_tasks()+ await with_timeout_async(+ self._channel.archive_task(task.id),+ context=f"archiving task {task.id}",+ )+ await with_timeout_async(+ self._post_ready_tasks(),+ context="posting ready tasks after task completion",+ )async def _graceful_shutdown(self, failed_task: Task) -> None:r"""Handle graceful shutdown with configurable timeout. This is used to@@ -534,14 +571,25 @@ class Workforce(BaseNode):self._running = Truelogger.info(f"Workforce {self.node_id} started.")- await self._post_ready_tasks()+ await with_timeout_async(+ self._post_ready_tasks(),+ context="posting ready tasks at the start",+ )while self._task is None or self._pending_tasks:- returned_task = await self._get_returned_task()+ returned_task = await with_timeout_async(+ self._get_returned_task(), context="getting returned task"+ )if returned_task.state == TaskState.DONE:- await self._handle_completed_task(returned_task)+ await with_timeout_async(+ self._handle_completed_task(returned_task),+ context="handling completed task",+ )elif returned_task.state == TaskState.FAILED:- halt = await self._handle_failed_task(returned_task)+ halt = await with_timeout_async(+ self._handle_failed_task(returned_task),+ context="handling failed task",+ )if not halt:continueprint(@@ -568,7 +616,9 @@ class Workforce(BaseNode):for child in self._children:child_listening_task = asyncio.create_task(child.start())self._child_listening_tasks.append(child_listening_task)- await self._listen_to_channel()+ await with_timeout_async(+ self._listen_to_channel(), context="listening to channel"+ )@check_if_running(True)def stop(self) -> None:diff --git a/camel/utils/__init__.py b/camel/utils/__init__.pyindex 9b8e4c3a8..4e7dff729 100644--- a/camel/utils/__init__.py+++ b/camel/utils/__init__.py@@ -40,6 +40,7 @@ from .commons import (to_pascal,track_agent,with_timeout,+ with_timeout_async,)from .constants import Constantsfrom .deduplication import DeduplicationResult, deduplicate_internally@@ -64,6 +65,8 @@ from .token_counting import (get_model_encoding,)+TIMEOUT_THRESHOLD = Constants.TIMEOUT_THRESHOLD+__all__ = ["print_text_animated","get_prompt_template_key_words",@@ -106,6 +109,8 @@ __all__ = ["sanitize_filename","browser_toolkit_save_auth_cookie","run_async",+ "with_timeout_async",+ "TIMEOUT_THRESHOLD","configure_langfuse","is_langfuse_available","get_current_agent_session_id",diff --git a/camel/utils/commons.py b/camel/utils/commons.pyindex 94f627568..4dd6fbf9d 100644--- a/camel/utils/commons.py+++ b/camel/utils/commons.py@@ -29,6 +29,7 @@ from http import HTTPStatusfrom pathlib import Pathfrom typing import (Any,+ Awaitable,Callable,Dict,List,@@ -51,6 +52,7 @@ from camel.types import TaskTypefrom .constants import ConstantsF = TypeVar('F', bound=Callable[..., Any])+T = TypeVar('T')logger = logging.getLogger(__name__)@@ -1130,3 +1132,28 @@ def run_async(func: Callable[..., Any]) -> Callable[..., Any]:return loop.run_until_complete(func(*args, **kwargs))return wrapper+++async def with_timeout_async(+ coro: Awaitable[T],+ timeout: Optional[float] = Constants.TIMEOUT_THRESHOLD,+ context: str = "operation",+) -> T:+ r"""General timeout wrapper for async operations.++ Args:+ coro: async operation to be executed+ timeout: max wait time (seconds)+ context: context information, used for error messages++ Returns:+ result of the async operation++ Raises:+ asyncio.TimeoutError: if wait times out+ """+ try:+ return await asyncio.wait_for(coro, timeout=timeout)+ except asyncio.TimeoutError:+ logger.error(f"Operation timed out after {timeout}s: {context}")+ raise asyncio.TimeoutError(f"Timed out while {context}")diff --git a/camel/utils/constants.py b/camel/utils/constants.pyindex de8ea1c4c..79c91288a 100644--- a/camel/utils/constants.py+++ b/camel/utils/constants.py@@ -35,3 +35,6 @@ class Constants:# Default similarity threshold value for RAGDEFAULT_SIMILARITY_THRESHOLD = 0.7++ # Default timeout threshold value+ TIMEOUT_THRESHOLD = 180.0diff --git a/camel/verifiers/base.py b/camel/verifiers/base.pyindex 5cd9038bd..ed6f305b8 100644--- a/camel/verifiers/base.py+++ b/camel/verifiers/base.py@@ -18,7 +18,7 @@ from typing import List, Optionalfrom camel.extractors.base import BaseExtractorfrom camel.logger import get_logger-from camel.utils import BatchProcessor+from camel.utils import TIMEOUT_THRESHOLD, BatchProcessor, with_timeout_asyncfrom .models import VerificationOutcome, VerificationResult@@ -47,7 +47,7 @@ class BaseVerifier(ABC):self,extractor: Optional[BaseExtractor] = None,max_parallel: Optional[int] = None,- timeout: Optional[float] = None,+ timeout: Optional[float] = TIMEOUT_THRESHOLD,max_retries: int = 3,retry_delay: float = 1.0,initial_batch_size: Optional[int] = None,@@ -103,7 +103,9 @@ class BaseVerifier(ABC):try:if self.extractor:- await self.extractor.setup()+ await with_timeout_async(+ self.extractor.setup(), context="setting up extractor"+ )batch_size = max(1, self._initial_batch_size or 10)max_parallel = max(1, self._max_parallel or 1)self._batch_processor = BatchProcessor()@@ -113,7 +115,10 @@ class BaseVerifier(ABC):f"batch_size={batch_size}, max_parallel={max_parallel}")- await self._setup(**kwargs)+ await with_timeout_async(+ self._setup(**kwargs),+ context=f"setting up {self.__class__.__name__}",+ )self._is_setup = Trueexcept Exception as e:@@ -121,7 +126,7 @@ class BaseVerifier(ABC):f"Failed to initialize {self.__class__.__name__}: {e!s}")logger.error(error_msg, exc_info=True)- await self.cleanup()+ await with_timeout_async(self.cleanup(), context="cleaning up")raise RuntimeError(error_msg) from e@abstractmethod@@ -144,9 +149,14 @@ class BaseVerifier(ABC):try:if self.extractor:- await self.extractor.cleanup()+ await with_timeout_async(+ self.extractor.cleanup(), context="cleaning up extractor"+ )self._batch_processor = BatchProcessor()- await self._cleanup()+ await with_timeout_async(+ self._cleanup(),+ context=f"cleaning up {self.__class__.__name__}",+ )logger.info(f"{self.__class__.__name__} cleaned up successfully")except Exception as e:@@ -194,7 +204,11 @@ class BaseVerifier(ABC):logger.warning(f"{self.__class__.__name__} not set up, calling setup()")- await self.setup()+ await with_timeout_async(+ self.setup(),+ context="setting \+ up verifier",+ )attempt = 0start_time = time.time()@@ -203,7 +217,10 @@ class BaseVerifier(ABC):# Extract verifiable part of the proposed solution,# if verifier has been initialized with extractor.verifiable_solution = (- await self.extractor.extract(solution)+ await with_timeout_async(+ self.extractor.extract(solution),+ context="extracting solution",+ )if self.extractorelse solution)@@ -222,21 +239,19 @@ class BaseVerifier(ABC):f"Failed to extract verifiable solution on attempt "f"{attempt}, retrying...")- await asyncio.sleep(self._retry_delay)+ await with_timeout_async(+ asyncio.sleep(self._retry_delay),+ context="waiting before retrying",+ )continuetry:- verification_result = (- await asyncio.wait_for(- self._verify_implementation(- verifiable_solution, reference_answer- ),- timeout=self._timeout,- )- if self._timeout- else await self._verify_implementation(+ verification_result = await with_timeout_async(+ self._verify_implementation(verifiable_solution, reference_answer- )+ ),+ timeout=self._timeout,+ context="verifying solution",)verification_result.duration = time.time() - start_time@@ -257,7 +272,10 @@ class BaseVerifier(ABC):logger.warning(f"Verification timeout on attempt {attempt}, retrying...")- await asyncio.sleep(self._retry_delay)+ await with_timeout_async(+ asyncio.sleep(self._retry_delay),+ context="waiting before retrying",+ )except Exception as e:attempt += 1@@ -269,7 +287,10 @@ class BaseVerifier(ABC):duration=time.time() - start_time,metadata={"attempt": attempt},)- await asyncio.sleep(self._retry_delay)+ await with_timeout_async(+ asyncio.sleep(self._retry_delay),+ context="waiting before retrying",+ )return VerificationResult(status=VerificationOutcome.ERROR,@@ -339,7 +360,11 @@ class BaseVerifier(ABC):logger.warning(f"{self.__class__.__name__} not set up, calling setup()")- await self.setup()+ await with_timeout_async(+ self.setup(),+ context="setting\+ up verifier",+ )# Retrieve batch processing settingsmax_workers = getattr(@@ -356,8 +381,10 @@ class BaseVerifier(ABC):start_time = time.time()try:async with semaphore:- verification_result = await self.verify(- solution, reference_answer+ verification_result = await with_timeout_async(+ self.verify(solution, reference_answer),+ timeout=self._timeout,+ context="verifying solution",)processing_time = time.time() - start_timesuccess = (@@ -391,7 +418,11 @@ class BaseVerifier(ABC):)]try:- batch_results = await asyncio.gather(*verification_tasks)+ batch_results = await with_timeout_async(+ asyncio.gather(*verification_tasks),+ timeout=self._timeout,+ context="verifying solutions",+ )all_results.extend(batch_results)except Exception as e:logger.error(diff --git a/camel/verifiers/python_verifier.py b/camel/verifiers/python_verifier.pyindex 7cc2e7932..ca3f71542 100644--- a/camel/verifiers/python_verifier.py+++ b/camel/verifiers/python_verifier.py@@ -24,6 +24,7 @@ from typing import Any, List, Optional, Tuplefrom camel.extractors.base import BaseExtractorfrom camel.logger import get_logger+from camel.utils import with_timeout_asyncfrom camel.verifiers import BaseVerifierfrom .models import VerificationOutcome, VerificationResult@@ -49,7 +50,6 @@ class PythonVerifier(BaseVerifier):def __init__(self,extractor: Optional[BaseExtractor] = None,- timeout: Optional[float] = 30.0,required_packages: Optional[List[str]] = None,float_tolerance: Optional[float] = None,**kwargs,@@ -68,7 +68,7 @@ class PythonVerifier(BaseVerifier):floating point comparisons. (default: :obj:`None`)"""# TODO: Use CAMEL's Interpreter to execute the code- super().__init__(extractor=extractor, timeout=timeout, **kwargs)+ super().__init__(extractor=extractor, **kwargs)self.venv_path: Optional[str] = Noneself.required_packages = required_packages or []self.float_tolerance = float_tolerance@@ -309,8 +309,10 @@ class PythonVerifier(BaseVerifier):)try:- sol_out, sol_err, sol_code = await self._run_code_block(- solution, venv_python+ sol_out, sol_err, sol_code = await with_timeout_async(+ self._run_code_block(solution, venv_python),+ timeout=self._timeout,+ context="running Python solution",)if sol_code != 0:return VerificationResult(@@ -416,8 +418,10 @@ class PythonVerifier(BaseVerifier):stdout=asyncio.subprocess.PIPE,stderr=asyncio.subprocess.PIPE,)- stdout, stderr = await asyncio.wait_for(- proc.communicate(), timeout=self._timeout+ stdout, stderr = await with_timeout_async(+ proc.communicate(),+ timeout=self._timeout,+ context="executing Python code",)os.remove(tmp_path)return (diff --git a/test/environments/test_multi_step_env.py b/test/environments/test_multi_step_env.pyindex 219bcd02e..8e9b6f80f 100644--- a/test/environments/test_multi_step_env.py+++ b/test/environments/test_multi_step_env.py@@ -11,9 +11,10 @@# See the License for the specific language governing permissions and# limitations under the License.# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+import asyncioimport refrom typing import Any, Dict, Tuple-from unittest.mock import AsyncMock, MagicMock+from unittest.mock import AsyncMock, MagicMock, patchimport pytest@@ -166,3 +167,88 @@ async def test_multi_step_env_error_handling():assert env_max_steps._current_step == 1, "Current step should be 1"assert env_max_steps.max_steps == 1, "Max steps should be 1"assert obs.question == "Episode ended", "Should reach terminal observation"+++async def test_multi_step_env_timeout_handling():+ r"""Test timeout handling in MultiStepEnv using mocks instead of sleep."""++ # Create mock functions that raise TimeoutError+ async def mock_update_state_timeout(*args, **kwargs):+ raise asyncio.TimeoutError("Simulated timeout in _update_state")++ async def mock_compute_reward_timeout(*args, **kwargs):+ raise asyncio.TimeoutError("Simulated timeout in compute_reward")++ async def mock_extract_timeout(*args, **kwargs):+ raise asyncio.TimeoutError("Simulated timeout in extract")++ # Create mock extractor+ mock_extractor = MagicMock()+ mock_extractor.setup = AsyncMock()+ mock_extractor.cleanup = AsyncMock()+ mock_extractor.extract = AsyncMock(side_effect=mock_extract_timeout)++ # 1. Test _update_state timeout in step method+ env = MockMultiStepEnv(extractor=mock_extractor, max_steps=3)++ try:+ await env.setup()+ _obs = await env.reset()++ # Replace _update_state with our mocked timeout version+ with patch.object(+ MockMultiStepEnv,+ '_update_state',+ new=AsyncMock(side_effect=mock_update_state_timeout),+ ):+ # The step should not hang - it should handle the timeout+ # and continue+ action = Action(llm_response="test action")+ observation, reward, done, info = await env.step(action)++ # Verify error info is present+ assert "error" in info+ assert "timed out" in info["error"].lower()+ finally:+ # Ensure environment is closed+ await env.close()++ # 2. Test compute_reward timeout in step+ env_reward = MockMultiStepEnv(extractor=mock_extractor, max_steps=3)++ try:+ await env_reward.setup()+ _obs = await env_reward.reset()++ # Replace compute_reward with our mocked timeout version+ with patch.object(+ MockMultiStepEnv,+ 'compute_reward',+ new=AsyncMock(side_effect=mock_compute_reward_timeout),+ ):+ action = Action(llm_response="test action")+ observation, reward, done, info = await env_reward.step(action)++ # Verify reward is default value due to timeout+ assert reward == 0.0+ assert "error" in info+ assert "timed out" in info["error"].lower()+ finally:+ await env_reward.close()++ # 3. Test setup timeout in reset+ setup_env = MockMultiStepEnv(extractor=mock_extractor, max_steps=3)++ # Mock setup to raise TimeoutError+ with patch.object(+ MockMultiStepEnv,+ '_setup',+ side_effect=asyncio.TimeoutError("Simulated timeout in setup"),+ ):+ # Reset should propagate the timeout exception+ with pytest.raises(asyncio.TimeoutError):+ await setup_env.reset()++ # Clean up+ await setup_env.close()diff --git a/test/environments/test_single_step_env.py b/test/environments/test_single_step_env.pyindex 64827b7b0..8dbfba743 100644--- a/test/environments/test_single_step_env.py+++ b/test/environments/test_single_step_env.py@@ -11,9 +11,10 @@# See the License for the specific language governing permissions and# limitations under the License.# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+import asyncioimport refrom typing import Dict, List-from unittest.mock import AsyncMock, MagicMock+from unittest.mock import AsyncMock, MagicMock, patchimport pytest@@ -1012,3 +1013,71 @@ async def test_single_step_env_all_input_types():await env.step([Action(index=2, llm_response="4")])await env.close()+++async def test_single_step_env_timeout_handling():+ r"""Test timeout handling in SingleStepEnv."""+ # Define a valid dataset+ data = [+ {+ "question": "What is 2 + 2?",+ "final_answer": "4",+ "rationale": "Adding 2 and 2 gives 4.",+ "metadata": {"difficulty": "easy"},+ },+ ]+ dataset = StaticDataset(data)++ # **1. Test Custom Reward Timeout**+ # Create a specialized SingleStepEnv that simulates timeout+ class TimeoutCustomRewardEnv(SingleStepEnv):+ async def _compute_custom_reward(self, *args, **kwargs):+ return {"custom_reward": 5.0}++ # Create a patched asyncio.wait_for that raises TimeoutError+ # for _compute_custom_reward+ original_wait_for = asyncio.wait_for++ async def mock_wait_for(coro, timeout, *args, **kwargs):+ # Check if this is the _compute_custom_reward coroutine+ if "_compute_custom_reward" in str(coro):+ raise asyncio.TimeoutError("Simulated timeout")+ # Otherwise, pass through to the original function+ return await original_wait_for(coro, timeout, *args, **kwargs)++ # Create environment with mock verifier+ mock_verifier = create_mock_verifier()+ env = TimeoutCustomRewardEnv(dataset=dataset, verifier=mock_verifier)++ # Setup and reset environment+ await env.setup()+ _obs = await env.reset(batch_size=1)++ # Test step with patched wait_for+ with patch('asyncio.wait_for', side_effect=mock_wait_for):+ action = Action(llm_response="The answer is 4")+ observation, reward, done, info = await env.step(action)++ # Verify that only correctness reward is present+ # (custom reward timed out)+ rewards_dict = info["rewards_dict"]+ assert "correctness" in rewards_dict+ assert "custom_reward" not in rewards_dict+ assert reward == env.ACCURACY_REWARD # Only accuracy reward applied++ # Test cleanup+ await env.close()+ assert env._is_setup is False+ mock_verifier.cleanup.assert_awaited_once()++ # **2. Test Reset Timeout**+ # Create a new environment for reset timeout testing+ reset_env = SingleStepEnv(dataset=dataset, verifier=create_mock_verifier())++ # Patch setup method to raise TimeoutError when called from reset+ with patch.object(+ SingleStepEnv, 'setup', side_effect=asyncio.TimeoutError+ ):+ with pytest.raises(asyncio.TimeoutError):+ await reset_env.reset()diff --git a/test/environments/test_tictactoe_env.py b/test/environments/test_tictactoe_env.pyindex f41aa2198..6d305f7d3 100644--- a/test/environments/test_tictactoe_env.py+++ b/test/environments/test_tictactoe_env.py@@ -347,3 +347,39 @@ async def test_full_game():assert done is Falseawait env.close()+++async def test_extraction_timeout():+ r"""Test timeout handling during move extraction in TicTacToeEnv."""+ import asyncio+ from unittest.mock import patch++ env = TicTacToeEnv()+ await env.setup()+ await env.reset()++ # Store original wait_for to restore later+ original_wait_for = asyncio.wait_for++ async def mock_wait_for(coro, timeout, *args, **kwargs):+ # Check if this is the extractor.extract call+ if "extract" in str(coro):+ raise asyncio.TimeoutError("Simulated timeout during extraction")+ # Otherwise, pass through to the original function+ return await original_wait_for(coro, timeout, *args, **kwargs)++ # Patch asyncio.wait_for to simulate timeout+ with patch('asyncio.wait_for', side_effect=mock_wait_for):+ action = Action(llm_response="<Action>5</Action>")+ obs, reward, done, info = await env.step(action)++ # Verify timeout was handled properly+ state = env._state+ assert state["last_move_illegal"] is True+ assert state["extraction_error"] == "Extraction timed out"+ assert reward == 0.0 # No reward for illegal move+ assert done is False # Game continues+ assert "Your last move was illegal" in obs.question++ await env.close()diff --git a/test/verifiers/test_base_verifier.py b/test/verifiers/test_base_verifier.pyindex 831c8c7c9..800d59ed3 100644--- a/test/verifiers/test_base_verifier.py+++ b/test/verifiers/test_base_verifier.py@@ -18,6 +18,7 @@ from unittest.mock import AsyncMock, patchimport pytest+from camel.utils import with_timeout_asyncfrom camel.verifiers.base import BaseVerifierfrom camel.verifiers.models import VerificationOutcome, VerificationResult@@ -68,7 +69,7 @@ def test_verifier_init():r"""Test BaseVerifier initialization with various parameters."""verifier = TestVerifier()assert verifier._max_parallel is None- assert verifier._timeout is None+ assert verifier._timeout == 180.0assert verifier._max_retries == 3assert verifier._retry_delay == 1.0assert verifier._is_setup is False@@ -194,18 +195,22 @@ async def test_verify_timeout(test_verifier):r"""Test verification timeout."""await test_verifier.setup()- with patch(- "asyncio.wait_for",- side_effect=asyncio.TimeoutError("Simulated timeout"),+ # dont verify directly+ with (+ patch.object(+ test_verifier,+ "_verify_implementation",+ side_effect=asyncio.TimeoutError("Simulated timeout"),+ ),+ patch("asyncio.sleep", new_callable=AsyncMock),):result = await test_verifier.verify(solution="This will timeout",reference_answer="Expected response",)- assert result.status == VerificationOutcome.TIMEOUT- assert "timed out" in result.error_message- assert result.duration > 0+ assert result.status == VerificationOutcome.TIMEOUT+ assert "Verification timed out after all retries" in result.error_messageawait test_verifier.cleanup()@@ -258,9 +263,10 @@ async def test_verify_batch(test_verifier):)solutions = ["Success 1", "Success 2", "This will fail"]- ground_truthes = ["Expected 1", "Expected 2", "Expected 3"]-- results = await test_verifier.verify_batch(solutions, ground_truthes)+ reference_answers = ["reference"] * len(solutions)+ results = await test_verifier.verify_batch(+ solutions, reference_answers+ )assert len(results) == 3assert results[0].status == VerificationOutcome.SUCCESS@@ -388,3 +394,136 @@ async def test_full_verification_flow():finally:await verifier.cleanup()assert verifier._is_setup is False+++async def test_with_timeout_function():+ """Test the with_timeout function handles completions and\+ timeouts correctly"""++ # Test normal operation (successful completion)+ mock_coro = AsyncMock()+ mock_coro.return_value = "success"+ result = await with_timeout_async(mock_coro(), context="test operation")+ assert result == "success"++ # Test timeout handling+ mock_timeout_coro = AsyncMock()+ mock_timeout_coro.side_effect = asyncio.TimeoutError("Simulated timeout")++ with pytest.raises(asyncio.TimeoutError) as exc_info:+ await with_timeout_async(+ mock_timeout_coro(), context="test timeout operation"+ )++ assert "Timed out while test timeout operation" in str(exc_info.value)+++async def test_timeout_in_extractor_setup():+ """Test timeout handling during extractor setup"""+ mock_extractor = AsyncMock()+ mock_extractor.setup.side_effect = asyncio.TimeoutError(+ "Simulated timeout in extractor setup"+ )++ verifier = TestVerifier()+ verifier.extractor = mock_extractor++ with pytest.raises(RuntimeError) as exc_info:+ await verifier.setup()++ assert "Failed to initialize" in str(exc_info.value)+ mock_extractor.setup.assert_called_once()+++async def test_timeout_in_verify_implementation(test_verifier):+ """Test timeout during verify implementation"""+ await test_verifier.setup()++ # Test that a TimeoutError in _verify_implementation is handled with retry+ with (+ patch.object(+ test_verifier,+ "_verify_implementation",+ side_effect=[+ asyncio.TimeoutError("Simulated timeout"),+ VerificationResult(+ status=VerificationOutcome.SUCCESS,+ result="Success after retry",+ ),+ ],+ ),+ patch("asyncio.sleep", new_callable=AsyncMock),+ ):+ result = await test_verifier.verify("solution", "reference")++ assert result.status == VerificationOutcome.SUCCESS+ assert result.result == "Success after retry"+ assert result.metadata["attempt"] == 2 # Second attempt succeeded++ await test_verifier.cleanup()+++async def test_repeated_timeout_in_verify_implementation(test_verifier):+ """Test multiple consecutive timeouts leading to timeout outcome"""+ await test_verifier.setup()+ test_verifier._max_retries = 2 # Set max retries to 2++ # Simulate repeated timeouts exceeding max_retries+ with (+ patch.object(+ test_verifier,+ "_verify_implementation",+ side_effect=asyncio.TimeoutError("Simulated timeout"),+ ),+ patch("asyncio.sleep", new_callable=AsyncMock),+ ):+ result = await test_verifier.verify("solution", "reference")++ # After max retries are exhausted, should return TIMEOUT outcome+ assert result.status == VerificationOutcome.TIMEOUT+ assert "Verification timed out after all retries" in result.error_message+ assert result.metadata["attempt"] == 2 # Attempted max_retries times++ await test_verifier.cleanup()+++async def test_timeout_in_batch_verification(test_verifier):+ """Test timeout handling in batch verification"""+ await test_verifier.setup()++ # Create a mix of successful and timeout solutions+ solutions = ["solution1", "solution2", "timeout"]+ reference_answers = ["reference"] * len(solutions)++ # Mock the verify method to simulate a timeout for specific solutions+ original_verify = test_verifier.verify++ async def mock_verify(solution, reference=None):+ if "timeout" in solution:+ raise asyncio.TimeoutError("Simulated timeout")+ return await original_verify(solution, reference)++ with patch.object(test_verifier, "verify", side_effect=mock_verify):+ results = await test_verifier.verify_batch(+ solutions, reference_answers+ )++ # Should have 3 results+ assert len(results) == 3++ # Check individual results+ success_results = [+ r for r in results if r.status == VerificationOutcome.SUCCESS+ ]++ non_success_results = [+ r for r in results if r.status != VerificationOutcome.SUCCESS+ ]+ assert len(non_success_results) == 1+ assert len(success_results) == 2+ await test_verifier.cleanup()diff --git a/test/verifiers/test_python_verifier.py b/test/verifiers/test_python_verifier.pyindex 2b548869a..914c9c8af 100644--- a/test/verifiers/test_python_verifier.py+++ b/test/verifiers/test_python_verifier.py@@ -12,12 +12,11 @@# limitations under the License.# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========import os+from unittest.mock import patchimport pytest-from camel.verifiers.models import (- VerificationOutcome,-)+from camel.verifiers.models import VerificationOutcomefrom camel.verifiers.python_verifier import PythonVerifier@@ -239,3 +238,50 @@ async def test_verify_with_float_tolerance(), f"Expected failure, got {result}"await verifier.cleanup()+++async def test_python_verifier_run_code_block_timeout(python_verifier):+ """Test timeout handling in _run_code_block method."""+ import asyncio++ await python_verifier.setup(uv=True)++ if python_verifier.venv_path is None:+ python_verifier.venv_path = "/tmp/mock_venv"++ # Mock proc.communicate to raise TimeoutError+ with patch(+ 'asyncio.subprocess.Process.communicate',+ side_effect=asyncio.TimeoutError("Simulated timeout"),+ ):+ with pytest.raises(asyncio.TimeoutError):+ await python_verifier._run_code_block(+ 'print("test")', python_verifier.venv_path + "/bin/python"+ )++ await python_verifier.cleanup()+++async def test_python_verifier_verify_timeout_propagation(python_verifier):+ """Test timeout propagation in the verification implementation."""+ import asyncio++ await python_verifier.setup(uv=True)++ # Mock _run_code_block to raise TimeoutError+ with patch.object(+ python_verifier,+ '_run_code_block',+ side_effect=asyncio.TimeoutError("Simulated timeout"),+ ):+ result = await python_verifier._verify_implementation(+ solution="print('test')", reference_answer=None+ )++ # Check that the timeout is properly handled+ assert result.status == VerificationOutcome.TIMEOUT+ assert "timed out" in result.error_message.lower()++ await python_verifier.cleanup()diff --git a/test/workforce/test_role_play.py b/test/workforce/test_role_play.pynew file mode 100644index 000000000..1666cb3e4--- /dev/null+++ b/test/workforce/test_role_play.py@@ -0,0 +1,258 @@+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========++import asyncio+from unittest.mock import AsyncMock, MagicMock, patch++import pytest++from camel.agents import ChatAgent+from camel.messages import BaseMessage+from camel.responses import ChatAgentResponse+from camel.societies.role_playing import RolePlaying+from camel.utils import with_timeout_async+++async def test_with_timeout_function():+ r"""Test the with_timeout function handles completions and\+ timeouts correctly"""+ # Test normal operation (successful completion)+ mock_coro = AsyncMock()+ mock_coro.return_value = "success"+ result = await with_timeout_async(mock_coro(), context="test operation")+ assert result == "success"++ # Test timeout handling+ mock_timeout_coro = AsyncMock()+ mock_timeout_coro.side_effect = asyncio.TimeoutError("Simulated timeout")++ with pytest.raises(asyncio.TimeoutError) as exc_info:+ await with_timeout_async(+ mock_timeout_coro(), context="test timeout operation"+ )++ assert "Timed out while test timeout operation" in str(exc_info.value)+++async def test_user_agent_timeout():+ r"""Test handling timeout from user agent in RolePlaying.astep"""+ # Create mock agents+ mock_user_agent = AsyncMock(spec=ChatAgent)+ mock_user_agent.astep.side_effect = asyncio.TimeoutError()+ mock_user_agent.role_name = "mock_user"++ # Add model_backend to both mock agents+ mock_user_agent.model_backend = MagicMock()+ mock_user_agent.model_backend.model_config_dict = {}++ mock_assistant_agent = AsyncMock(spec=ChatAgent)+ mock_assistant_agent.model_backend = MagicMock()+ mock_assistant_agent.model_backend.model_config_dict = {}++ # Create RolePlaying instance with mocked agents+ role_playing = RolePlaying("assistant", "user")+ role_playing.assistant_agent = mock_assistant_agent+ role_playing.user_agent = mock_user_agent++ # Create mock message+ mock_msg = MagicMock(spec=BaseMessage)++ # Test that TimeoutError is propagated+ with pytest.raises(asyncio.TimeoutError):+ await role_playing.astep(mock_msg)++ # Verify the user agent's astep was called+ mock_user_agent.astep.assert_called_once_with(mock_msg)+ # Verify the assistant agent's astep was not called+ mock_assistant_agent.astep.assert_not_called()+++async def test_assistant_agent_timeout():+ r"""Test handling timeout from assistant agent in RolePlaying.astep"""+ # Create mock agents with model_backend+ mock_user_agent = AsyncMock(spec=ChatAgent)+ mock_user_agent.model_backend = MagicMock()+ mock_user_agent.model_backend.model_config_dict = {}++ mock_assistant_agent = AsyncMock(spec=ChatAgent)+ mock_assistant_agent.model_backend = MagicMock()+ mock_assistant_agent.model_backend.model_config_dict = {}++ # Configure user_agent to return normal response and assistant_agent+ # to timeout+ mock_user_response = MagicMock(spec=ChatAgentResponse)+ mock_user_response.msgs = [MagicMock(spec=BaseMessage)]+ mock_user_response.terminated = False+ mock_user_agent.astep.return_value = mock_user_response+ mock_user_agent.role_name = "mock_user"++ mock_assistant_agent.astep.side_effect = asyncio.TimeoutError()+ mock_assistant_agent.role_name = "mock_assistant"++ # Create RolePlaying instance with mocked agents+ role_playing = RolePlaying("assistant", "user")+ role_playing.assistant_agent = mock_assistant_agent+ role_playing.user_agent = mock_user_agent+ role_playing._reduce_message_options = lambda msgs: msgs[+ 0+ ] # Mock this function++ # Create mock message+ mock_msg = MagicMock(spec=BaseMessage)++ # Test that TimeoutError is propagated+ with pytest.raises(asyncio.TimeoutError):+ await role_playing.astep(mock_msg)++ # Verify both agents' astep methods were called+ mock_user_agent.astep.assert_called_once_with(mock_msg)+ mock_assistant_agent.astep.assert_called_once()+++async def test_recovery_from_timeout():+ r"""Test RolePlaying can recover from an agent timeout"""+ # Create mock agents with model_backend+ mock_user_agent = AsyncMock(spec=ChatAgent)+ mock_user_agent.model_backend = MagicMock()+ mock_user_agent.model_backend.model_config_dict = {}++ mock_assistant_agent = AsyncMock(spec=ChatAgent)+ mock_assistant_agent.model_backend = MagicMock()+ mock_assistant_agent.model_backend.model_config_dict = {}++ # First call to user_agent.astep will timeout, second will succeed+ call_count = 0++ async def user_agent_side_effect(msg):+ nonlocal call_count+ call_count += 1+ if call_count == 1:+ raise asyncio.TimeoutError("Simulated timeout")++ response = MagicMock(spec=ChatAgentResponse)+ response.msgs = [MagicMock(spec=BaseMessage)]+ response.terminated = False+ response.info = {}+ return response++ mock_user_agent.astep.side_effect = user_agent_side_effect+ mock_user_agent.role_name = "mock_user"++ # Assistant agent returns normal response+ mock_assistant_response = MagicMock(spec=ChatAgentResponse)+ mock_assistant_response.msgs = [MagicMock(spec=BaseMessage)]+ mock_assistant_response.terminated = False+ mock_assistant_response.info = {}+ mock_assistant_agent.astep.return_value = mock_assistant_response+ mock_assistant_agent.role_name = "mock_assistant"++ # Create RolePlaying instance with mocked agents+ role_playing = RolePlaying("assistant", "user")+ role_playing.assistant_agent = mock_assistant_agent+ role_playing.user_agent = mock_user_agent+ role_playing._reduce_message_options = lambda msgs: msgs[+ 0+ ] # Mock this function++ # Create mock message+ mock_msg = MagicMock(spec=BaseMessage)++ # First attempt should raise TimeoutError+ with pytest.raises(asyncio.TimeoutError):+ await role_playing.astep(mock_msg)++ # Second attempt should succeed+ user_response, assistant_response = await role_playing.astep(mock_msg)++ # Verify both responses are returned+ assert user_response is not None+ assert assistant_response is not None+ assert user_response.terminated is False+ assert assistant_response.terminated is False++ # Verify both agents were called the expected number of times+ assert mock_user_agent.astep.call_count == 2+ assert mock_assistant_agent.astep.call_count == 1+++async def test_integration_with_timeout():+ r"""Test the full RolePlaying.astep flow with various timeout points"""++ # Setup patching of with_timeout - properly await the coroutine+ async def mock_with_timeout(coro, **kwargs):+ # Make sure we await the coroutine+ return await coro++ # Create properly structured return values+ mock_user_msg = MagicMock(spec=BaseMessage)+ mock_assistant_msg = MagicMock(spec=BaseMessage)++ mock_user_response = MagicMock(spec=ChatAgentResponse)+ mock_user_response.msgs = [mock_user_msg]+ mock_user_response.terminated = False+ mock_user_response.info = {}++ mock_assistant_response = MagicMock(spec=ChatAgentResponse)+ mock_assistant_response.msgs = [mock_assistant_msg]+ mock_assistant_response.terminated = False+ mock_assistant_response.info = {}++ # Create mocked agents that return proper values+ mock_user_agent = AsyncMock(spec=ChatAgent)+ mock_user_agent.astep.return_value = mock_user_response+ mock_user_agent.role_name = "mock_user"+ mock_user_agent.model_backend = MagicMock()+ mock_user_agent.model_backend.model_config_dict = {}++ mock_assistant_agent = AsyncMock(spec=ChatAgent)+ mock_assistant_agent.astep.return_value = mock_assistant_response+ mock_assistant_agent.role_name = "mock_assistant"+ mock_assistant_agent.model_backend = MagicMock()+ mock_assistant_agent.model_backend.model_config_dict = {}++ # Create RolePlaying instance+ role_playing = RolePlaying("assistant", "user")+ role_playing.assistant_agent = mock_assistant_agent+ role_playing.user_agent = mock_user_agent+ role_playing._reduce_message_options = lambda msgs: msgs[+ 0+ ] # Mock this function++ # Create test message+ test_msg = MagicMock(spec=BaseMessage)++ # Test the normal flow (no timeouts)+ with patch(+ "camel.utils.with_timeout_async",+ side_effect=mock_with_timeout,+ ):+ user_response, assistant_response = await role_playing.astep(test_msg)++ # validate response+ assert len(user_response.msgs) == 1+ assert user_response.terminated is False+ assert user_response.info == {}++ assert len(assistant_response.msgs) == 1+ assert assistant_response.terminated is False+ assert assistant_response.info == {}++ # verify both agents were called+ mock_user_agent.astep.assert_called_with(test_msg)+ mock_assistant_agent.astep.assert_called_with(mock_user_msg)diff --git a/test/workforce/test_worker.py b/test/workforce/test_worker.pynew file mode 100644index 000000000..a3d6534d7--- /dev/null+++ b/test/workforce/test_worker.py@@ -0,0 +1,259 @@+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+import asyncio+from unittest.mock import AsyncMock, MagicMock, patch++import pytest++from camel.societies.workforce.task_channel import TaskChannel+from camel.societies.workforce.worker import Worker+from camel.tasks.task import Task, TaskState+from camel.utils import TIMEOUT_THRESHOLD, with_timeout_async+++class TestTimeoutWorker(Worker):+ r"""Test Worker implementation for timeout testing"""++ def __init__(self, description="Test Worker", node_id=None):+ super().__init__(description, node_id=node_id)++ async def _process_task(self, task, dependencies):+ r"""Simple implementation of task processing method"""+ return TaskState.DONE+++async def test_with_timeout_function():+ r"""Test the basic functionality of with_timeout function"""+ # Test normal case+ mock_coro = AsyncMock()+ mock_coro.return_value = "success"++ result = await with_timeout_async(mock_coro(), timeout=1.0, context="test")+ assert result == "success"++ # Test timeout case by directly mocking timeout error+ mock_failing_coro = AsyncMock()+ mock_failing_coro.side_effect = asyncio.TimeoutError("Simulated timeout")++ with pytest.raises(asyncio.TimeoutError) as excinfo:+ await with_timeout_async(+ mock_failing_coro(), timeout=1.0, context="test timeout"+ )++ assert "Timed out while test timeout" in str(excinfo.value)+++async def test_get_assigned_task_timeout():+ r"""Test timeout handling in _get_assigned_task method"""+ worker = TestTimeoutWorker()++ # Mock TaskChannel to simulate timeout+ mock_channel = AsyncMock(spec=TaskChannel)+ mock_channel.get_assigned_task_by_assignee.side_effect = (+ asyncio.TimeoutError("Simulated timeout")+ )+ worker._channel = mock_channel++ with pytest.raises(asyncio.TimeoutError):+ await worker._get_assigned_task()++ mock_channel.get_assigned_task_by_assignee.assert_called_once()+++async def test_listen_to_channel_recovers_from_timeout():+ r"""Test that _listen_to_channel method recovers from timeouts"""+ worker = TestTimeoutWorker()++ # Mock TaskChannel+ mock_channel = AsyncMock(spec=TaskChannel)++ # Configure get_assigned_task_by_assignee to first throw TimeoutError,+ # then return normal result+ mock_task = MagicMock(spec=Task)+ mock_task.id = "test_task_id"+ mock_task.content = "test content"++ call_count = 0++ async def side_effect(*args, **kwargs):+ nonlocal call_count+ call_count += 1+ if call_count == 1:+ raise asyncio.TimeoutError("Simulated timeout")+ return mock_task++ mock_channel.get_assigned_task_by_assignee.side_effect = side_effect+ mock_channel.get_dependency_ids.return_value = []+ mock_channel.return_task = AsyncMock()++ worker._channel = mock_channel++ async def patched_listen():+ worker._running = True+ try:+ # Only process one task - retry if timeout occurs+ attempts = 0+ max_attempts = 3+ while attempts < max_attempts:+ try:+ await worker._get_assigned_task()+ break # Success, exit the loop+ except asyncio.TimeoutError:+ # This is expected on first attempt, retry+ attempts += 1+ if attempts >= max_attempts:+ raise # Re-raise if max attempts reached+ finally:+ worker._running = False++ async def direct_start():+ return await patched_listen()++ with (+ patch.object(worker, '_listen_to_channel', patched_listen),+ patch.object(worker, 'start', direct_start),+ ):+ # Should not raise exception+ await worker.start()++ # Verify first call failed and then retried+ assert mock_channel.get_assigned_task_by_assignee.call_count == 2+++async def test_worker_with_timeout_integration():+ r"""Test the integration of with_timeout across Worker methods"""+ worker = TestTimeoutWorker()++ # Mock TaskChannel+ mock_channel = AsyncMock(spec=TaskChannel)+ mock_task = MagicMock(spec=Task)+ mock_task.id = "test_task_id"+ mock_task.content = "test content"+ mock_task.set_state = MagicMock()++ mock_channel.get_assigned_task_by_assignee.return_value = mock_task+ mock_channel.get_dependency_ids.return_value = []+ mock_channel.return_task = AsyncMock()++ worker._channel = mock_channel++ ran_once = False++ async def patched_listen():+ nonlocal ran_once+ worker._running = True++ # Add call here to ensure it's always executed regardless of execution+ # path+ await worker._channel.get_dependency_ids()++ if not ran_once:+ ran_once = True+ # Get task+ task = await worker._get_assigned_task()++ # Keep this call for backward compatibility+ # _dependency_ids = await worker._channel.get_dependency_ids()++ task_dependencies = []+ # Process task+ task_state = await with_timeout_async(+ worker._process_task(task, task_dependencies),+ context=f"processing task {task.id}",+ )+ # Update status+ task.set_state(task_state)+ # Return task+ await with_timeout_async(+ worker._channel.return_task(task.id),+ timeout=TIMEOUT_THRESHOLD,+ context=f"returning task {task.id}",+ )+ worker._running = False++ with patch.object(worker, '_listen_to_channel', patched_listen):+ await worker.start()++ # Verify all methods were called correctly+ mock_channel.get_assigned_task_by_assignee.assert_called_once()+ mock_channel.get_dependency_ids.assert_called_once()+ mock_task.set_state.assert_called_once_with(TaskState.DONE)+ mock_channel.return_task.assert_called_once_with(mock_task.id)+++async def test_multiple_timeout_points():+ r"""Test handling timeouts at different points in the worker process"""+ worker = TestTimeoutWorker()++ # Mock TaskChannel that times out during dependency retrieval+ mock_channel = AsyncMock(spec=TaskChannel)+ mock_task = MagicMock(spec=Task)+ mock_task.id = "test_task_id"++ mock_channel.get_assigned_task_by_assignee.return_value = mock_task+ mock_channel.get_dependency_ids.side_effect = asyncio.TimeoutError(+ "Dependency timeout"+ )++ worker._channel = mock_channel++ # Mock _process_task to verify it's not called when dependencies time out+ process_task_mock = AsyncMock()+ worker._process_task = process_task_mock++ # Patch _listen_to_channel to handle one task+ async def patched_listen():+ worker._running = True+ try:+ # Get task+ task = await worker._get_assigned_task()++ # Get dependencies - this will time out+ try:+ dependency_ids = await with_timeout_async(+ worker._channel.get_dependency_ids(),+ context=f"getting dependency IDs for task {task.id}",+ )+ task_dependencies = []+ for dep_id in dependency_ids:+ dep_task = await with_timeout_async(+ worker._channel.get_task_by_id(dep_id),+ context=f"getting dependency task {dep_id}",+ )+ task_dependencies.append(dep_task)++ # Process task - should not reach here due to timeout+ task_state = await worker._process_task(+ task, task_dependencies+ )+ task.set_state(task_state)+ await worker._channel.return_task(task.id)+ except asyncio.TimeoutError:+ # This is expected, continue to next task+ pass+ finally:+ worker._running = False++ with patch.object(worker, '_listen_to_channel', patched_listen):+ await worker.start()++ # Verify get_dependency_ids was called but _process_task was not+ mock_channel.get_assigned_task_by_assignee.assert_called_once()+ mock_channel.get_dependency_ids.assert_called_once()+ process_task_mock.assert_not_called()diff --git a/test/workforce/test_workforce.py b/test/workforce/test_workforce.pynew file mode 100644index 000000000..88de3b26c--- /dev/null+++ b/test/workforce/test_workforce.py@@ -0,0 +1,274 @@+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========++import asyncio+from unittest.mock import AsyncMock, MagicMock, patch++import pytest++from camel.societies.workforce.task_channel import TaskChannel+from camel.societies.workforce.workforce import Workforce+from camel.tasks.task import Task, TaskState+from camel.utils import with_timeout_async+++class TestTimeoutWorkforce(Workforce):+ r"""A test workforce class for testing timeout handling"""++ def __init__(self):+ super().__init__("Test Workforce")+ # Skip the actual initialization to make testing easier+ self._running = False+ self._channel = None+ self._pending_tasks = []+ self._task = None+ self._child_listening_tasks = []+ self._children = []+++async def test_with_timeout_function():+ r"""Test the with_timeout function handles completions and+ timeouts correctly"""+ # Test normal operation (successful completion)+ mock_coro = AsyncMock()+ mock_coro.return_value = "success"+ result = await with_timeout_async(mock_coro(), context="test operation")+ assert result == "success"++ # Test timeout handling+ mock_timeout_coro = AsyncMock()+ mock_timeout_coro.side_effect = asyncio.TimeoutError("Simulated timeout")++ with pytest.raises(asyncio.TimeoutError) as exc_info:+ await with_timeout_async(+ mock_timeout_coro(), context="test timeout operation"+ )++ assert "Timed out while test timeout operation" in str(exc_info.value)+++async def test_post_task_timeout():+ r"""Test timeout handling in _post_task method"""+ workforce = TestTimeoutWorkforce()++ # Mock TaskChannel that times out+ mock_channel = AsyncMock(spec=TaskChannel)+ mock_channel.post_task.side_effect = asyncio.TimeoutError()++ workforce._channel = mock_channel+ mock_task = MagicMock(spec=Task)+ mock_task.id = "test_task_id"++ # Only verify that a TimeoutError is raised, not its specific message+ with pytest.raises(asyncio.TimeoutError):+ await workforce._post_task(mock_task, "test_worker_id")++ # Verify the channel method was called+ mock_channel.post_task.assert_called_once()+++async def test_listen_to_channel_recovers_from_timeout():+ r"""Test that _listen_to_channel method recovers from timeouts when+ getting returned tasks"""+ workforce = TestTimeoutWorkforce()++ # Mock TaskChannel with first call timing out, second call succeeding+ mock_channel = AsyncMock(spec=TaskChannel)+ mock_task = MagicMock(spec=Task)+ mock_task.id = "test_task_id"+ mock_task.state = TaskState.DONE++ # Configure get_returned_task_by_publisher to first throw TimeoutError,+ # then return normal result+ call_count = 0++ async def side_effect(*args, **kwargs):+ nonlocal call_count+ call_count += 1+ if call_count == 1:+ raise asyncio.TimeoutError("Simulated timeout")+ return mock_task++ mock_channel.get_returned_task_by_publisher.side_effect = side_effect+ workforce._channel = mock_channel++ # Mock _handle_completed_task to avoid full implementation+ workforce._handle_completed_task = AsyncMock()++ # Mock _post_ready_tasks to avoid implementation+ workforce._post_ready_tasks = AsyncMock()++ # Patch _listen_to_channel to run only limited iterations+ async def patched_listen():+ workforce._running = True+ workforce._task = mock_task+ workforce._pending_tasks = [mock_task]++ try:+ # Post initial ready tasks+ await workforce._post_ready_tasks()++ # First try to get returned task - should timeout+ try:+ returned_task = await workforce._get_returned_task()+ raise asyncio.TimeoutError("Simulated timeout")+ except asyncio.TimeoutError:+ # This is expected+ pass++ # Second try should succeed+ returned_task = await workforce._get_returned_task()+ # Process the task+ await workforce._handle_completed_task(returned_task)++ finally:+ workforce._running = False++ async def direct_start():+ return await patched_listen()++ with (+ patch.object(workforce, '_listen_to_channel', patched_listen),+ patch.object(workforce, 'start', direct_start),+ ):+ await workforce.start()++ # Verify get_returned_task_by_publisher was called twice (first fails,+ # second succeeds)+ assert mock_channel.get_returned_task_by_publisher.call_count == 2+ # Verify the task was processed+ workforce._handle_completed_task.assert_called_once_with(mock_task)+++async def test_workforce_with_timeout_integration():+ r"""Test the integration of with_timeout across Workforce methods"""+ workforce = TestTimeoutWorkforce()++ # Mock TaskChannel+ mock_channel = AsyncMock(spec=TaskChannel)+ mock_task = MagicMock(spec=Task)+ mock_task.id = "test_task_id"+ mock_task.state = TaskState.DONE++ mock_channel.get_returned_task_by_publisher.return_value = mock_task+ workforce._channel = mock_channel++ # Mock _handle_completed_task to avoid full implementation+ workforce._handle_completed_task = AsyncMock()++ # Mock _post_ready_tasks to avoid implementation+ workforce._post_ready_tasks = AsyncMock()++ # Patch _listen_to_channel to run only once+ ran_once = False++ async def patched_listen():+ nonlocal ran_once+ workforce._running = True+ workforce._task = mock_task+ workforce._pending_tasks = [mock_task]++ if not ran_once:+ ran_once = True+ # Post initial ready tasks+ await with_timeout_async(+ workforce._post_ready_tasks(),+ context="posting ready tasks at start",+ )++ # Get returned task+ returned_task = await with_timeout_async(+ workforce._get_returned_task(), context="getting returned task"+ )++ # Handle completed task+ await with_timeout_async(+ workforce._handle_completed_task(returned_task),+ context="handling completed task",+ )++ workforce._running = False++ with patch.object(workforce, '_listen_to_channel', patched_listen):+ await workforce.start()++ # Verify methods were called+ mock_channel.get_returned_task_by_publisher.assert_called_once()+ workforce._post_ready_tasks.assert_called_once()+ workforce._handle_completed_task.assert_called_once_with(mock_task)+++async def test_multiple_timeout_points():+ r"""Test handling timeouts at different points in the workforce process"""+ workforce = TestTimeoutWorkforce()++ # Mock TaskChannel that times out during get_returned_task+ mock_channel = AsyncMock(spec=TaskChannel)+ mock_task = MagicMock(spec=Task)+ mock_task.id = "test_task_id"+ mock_task.state = TaskState.FAILED++ mock_task.failure_count = 0++ mock_channel.get_returned_task_by_publisher.return_value = mock_task++ # Make _handle_failed_task timeout+ mock_channel.remove_task.side_effect = asyncio.TimeoutError()++ workforce._channel = mock_channel+ workforce._task = mock_task+ workforce._pending_tasks = [mock_task]++ # Patch _listen_to_channel to handle one task with the timeout+ async def patched_listen():+ workforce._running = True+ try:+ # Post initial ready tasks+ await workforce._post_ready_tasks()++ # Get task - should succeed+ returned_task = await workforce._get_returned_task()++ # Handle failed task - should timeout during remove_task+ try:+ await workforce._handle_failed_task(returned_task)+ raise AssertionError("Should have timed out on first attempt")+ except asyncio.TimeoutError:+ # This is expected+ pass++ finally:+ workforce._running = False++ # Mock methods to avoid full implementation+ workforce._post_ready_tasks = AsyncMock()++ async def direct_start():+ return await patched_listen()++ with (+ patch.object(workforce, '_listen_to_channel', patched_listen),+ patch.object(workforce, 'start', direct_start),+ ):+ await workforce.start()++ # Verify get_returned_task_by_publisher was called+ mock_channel.get_returned_task_by_publisher.assert_called_once()+ # Verify remove_task was called (this is where the timeout occurred)+ mock_channel.remove_task.assert_called_once_with(mock_task.id)