Skip to content

armgit / camel

Public Imported from GitHub
0

🐫 CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org

[Feature Request] Enhancing robustness of Async operations in core modules #2468 (#2551)

Bonian Hud75ab1b634c9
diff --git a/camel/environments/multi_step.py b/camel/environments/multi_step.py
index e73b2af6d..1d4645cb2 100644
--- a/camel/environments/multi_step.py
+++ b/camel/environments/multi_step.py
@@ -12,12 +12,14 @@
# limitations under the License.
# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+import asyncio
from abc import ABC, abstractmethod
from typing import Any, Dict, List, Optional, Tuple
from camel.environments.models import Action, Observation, StepResult
from camel.extractors.base import BaseExtractor
from camel.logger import get_logger
+from camel.utils import TIMEOUT_THRESHOLD, with_timeout_async
logger = get_logger(__name__)
@@ -29,6 +31,7 @@ class MultiStepEnv(ABC):
self,
extractor: BaseExtractor,
max_steps: Optional[int] = None,
+ timeout: Optional[float] = TIMEOUT_THRESHOLD,
**kwargs,
) -> None:
r"""Initialize the environment.
@@ -36,10 +39,12 @@ class MultiStepEnv(ABC):
Args:
extractor: Extractor to process LLM responses.
max_steps: Maximum steps per episode.
+ timeout: The execution timeout in seconds.
**kwargs: Additional environment parameters.
"""
self.extractor = extractor
self.max_steps = max_steps
+ self._timeout = timeout
self._metadata = kwargs
# State tracking
@@ -109,13 +114,18 @@ class MultiStepEnv(ABC):
Raises:
RuntimeError: If we fail to get the initial observation.
+ asyncio.TimeoutError: If the environment setup times out.
"""
if not self._is_setup:
logger.warning(
"reset() called on un-setup environment. Setting up..."
)
- await self.setup()
+ await with_timeout_async(
+ self.setup(),
+ timeout=self._timeout,
+ context="environment setup",
+ )
# Reset state
self._current_step = 0
@@ -151,6 +161,7 @@ class MultiStepEnv(ABC):
Raises:
RuntimeError: If the environment is not set up, the episode has
ended, or there is no valid current observation.
+ asyncio.TimeoutError: If the step operation times out.
"""
if self.max_steps and self._current_step >= self.max_steps:
return StepResult(
@@ -173,11 +184,46 @@ class MultiStepEnv(ABC):
current_obs: Observation = self._last_observation
self._episode_history.append((current_obs, action))
+ # Initialize info dictionary
+ info = {}
+
# Update the environment state based on the action
- await self._update_state(action)
+ update_timed_out = False
+ try:
+ await with_timeout_async(
+ self._update_state(action),
+ timeout=self._timeout,
+ context="state update",
+ )
+ except asyncio.TimeoutError:
+ update_timed_out = True
+
+ # If update timed out, return error information
+ if update_timed_out:
+ return StepResult(
+ observation=self._last_observation, # Use last observation
+ reward=0.0, # Default reward
+ rewards_dict={}, # No rewards on timeout
+ done=False, # Episode not done
+ info={
+ "error": f"State update timed out after {self._timeout}s"
+ },
+ ).as_tuple()
# Compute rewards
- total_reward, rewards_dict = await self.compute_reward()
+ try:
+ total_reward, rewards_dict = await with_timeout_async(
+ self.compute_reward(),
+ timeout=self._timeout,
+ context="reward computation",
+ )
+ except asyncio.TimeoutError:
+ # raise exception to terminate the step operation
+ total_reward = 0.0
+ rewards_dict = {}
+ info["error"] = (
+ f"Reward computation timed out after {self._timeout}s"
+ )
# Check termination
done = self.is_done()
@@ -192,18 +238,38 @@ class MultiStepEnv(ABC):
self._last_observation = next_obs
self._episode_ended = done
+ extraction_result = None
+ try:
+ extraction_result = await with_timeout_async(
+ self.extractor.extract(action.llm_response),
+ timeout=self._timeout,
+ context="extraction",
+ )
+ except asyncio.TimeoutError:
+ # raise exception to terminate the step operation
+ extraction_result = None
+ if info is None:
+ info = {}
+ info["error"] = f"Extraction timed out after {self._timeout}s"
+
+ # If we have no info dict yet (no errors occurred), create one
+ if info is None:
+ info = {}
+
+ # Add extraction result to info if available and no error occurred
+ if "error" not in info and extraction_result is not None:
+ info["extraction_result"] = extraction_result
+
+ # Add step and state info
+ info["step"] = str(self._current_step)
+ info["state"] = str(self._state) # Updated state
+
return StepResult(
observation=next_obs,
reward=total_reward,
rewards_dict=rewards_dict,
done=done,
- info={
- "extraction_result": await self.extractor.extract(
- action.llm_response
- ),
- "step": self._current_step,
- "state": self._state, # Updated state
- },
+ info=info,
).as_tuple()
@abstractmethod
diff --git a/camel/environments/single_step.py b/camel/environments/single_step.py
index d490dceb9..c5325c10f 100644
--- a/camel/environments/single_step.py
+++ b/camel/environments/single_step.py
@@ -18,6 +18,7 @@ from typing import Any, Dict, List, Optional, Tuple, Union
from camel.datasets import BaseGenerator, DataPoint, StaticDataset
from camel.logger import get_logger
+from camel.utils import TIMEOUT_THRESHOLD, with_timeout_async
from camel.verifiers.base import (
BaseVerifier,
VerificationOutcome,
@@ -59,7 +60,7 @@ class SingleStepEnv:
self,
dataset: Union[StaticDataset, BaseGenerator],
verifier: BaseVerifier,
- timeout: Optional[float] = 180.0,
+ timeout: Optional[float] = TIMEOUT_THRESHOLD,
**kwargs,
) -> None:
r"""Initialize the SingleStepEnv.
@@ -77,7 +78,7 @@ class SingleStepEnv:
This class assumes all interactions are single-step: one question,
one LLM response, one reward.
"""
- self._timeout = timeout
+ self._timeout: Optional[float] = timeout
self.dataset = dataset
self.verifier = verifier
self._metadata = kwargs
@@ -163,6 +164,8 @@ class SingleStepEnv:
RuntimeError: If called before all previous states are processed.
ValueError: If batch size exceeds dataset size.
TypeError: If the dataset is of an unsupported type.
+ asyncio.TimeoutError: If the environment setup times out or
+ dataset sampling times out.
"""
if batch_size <= 0:
raise ValueError("Batch size must be positive")
@@ -171,7 +174,11 @@ class SingleStepEnv:
logger.warning(
"reset() called on un-setup environment. Setting up..."
)
- await self.setup()
+ await with_timeout_async(
+ self.setup(),
+ timeout=self._timeout,
+ context="environment setup",
+ )
if self._batch_started() and not self._batch_done():
logger.error(
@@ -218,11 +225,20 @@ class SingleStepEnv:
return observations[0] if batch_size == 1 else observations
elif isinstance(self.dataset, BaseGenerator):
- self._states = [
- await self.dataset.async_sample() for _ in range(batch_size)
- ]
- self.current_batch_size = batch_size
- self._states_done = [False] * batch_size
+ self._states = []
+ for _ in range(batch_size):
+ try:
+ sample = await with_timeout_async(
+ self.dataset.async_sample(),
+ timeout=self._timeout,
+ context="dataset sampling",
+ )
+ self._states.append(sample)
+ except asyncio.TimeoutError:
+ raise
+
+ self.current_batch_size = len(self._states)
+ self._states_done = [False] * self.current_batch_size
observations = [
Observation(
@@ -319,18 +335,16 @@ class SingleStepEnv:
]
try:
- verification_results = await asyncio.wait_for(
+ verification_results = await with_timeout_async(
self.verifier.verify_batch(
solutions=proposed_solutions,
reference_answers=ground_truths, # type: ignore [arg-type]
raise_on_error=True,
),
timeout=self._timeout,
+ context="step verification",
)
- except asyncio.TimeoutError as e:
- logger.error(
- f"Step verification timed out after {self._timeout}s: {e}"
- )
+ except asyncio.TimeoutError:
# Return timeout verification results
verification_results = [
VerificationResult(
@@ -359,27 +373,29 @@ class SingleStepEnv:
try:
# First try to compute all rewards with a timeout
- computed_rewards, computed_rewards_dicts = await asyncio.wait_for(
+ (
+ computed_rewards,
+ computed_rewards_dicts,
+ ) = await with_timeout_async(
self._compute_reward_batch(
proposed_solutions, verification_results
),
timeout=self._timeout,
+ context="batch reward computation",
)
# If successful, use all the computed values
total_rewards = computed_rewards
rewards_dicts = computed_rewards_dicts
- except asyncio.TimeoutError as e:
- logger.error(
- f"Reward computation timed out after {self._timeout}s: {e}"
- )
+ except asyncio.TimeoutError:
# Try to compute rewards one by one to identify which ones time out
for i, (solution, result) in enumerate(
zip(proposed_solutions, verification_results)
):
try:
- individual_rewards = await asyncio.wait_for(
+ individual_rewards = await with_timeout_async(
self._compute_custom_reward(solution, result),
timeout=self._timeout,
+ context=f"reward computation for solution {i}",
)
# If successful, calculate the reward for this solution
correctness_reward = (
@@ -534,6 +550,9 @@ class SingleStepEnv:
Tuple containing:
- List of total rewards for each solution.
- List of reward component dictionaries for each solution.
+
+ Raises:
+ asyncio.TimeoutError: If the custom reward computation times out.
"""
if len(proposed_solutions) != len(verification_results):
raise ValueError(
@@ -553,10 +572,21 @@ class SingleStepEnv:
self.ACCURACY_REWARD if verification_result.status else 0.0
)
- further_rewards = await self._compute_custom_reward(
- solution, verification_result
- )
- rewards = {**rewards, **further_rewards}
+ try:
+ # add timeout control
+ further_rewards = await with_timeout_async(
+ self._compute_custom_reward(solution, verification_result),
+ timeout=self._timeout,
+ )
+ rewards = {**rewards, **further_rewards}
+ except asyncio.TimeoutError:
+ logger.warning(
+ f"Custom reward computation timed out after\
+ {self._timeout}s"
+ )
+ # use empty dictionary as fallback strategy
+ further_rewards = {}
+ rewards = {**rewards, **further_rewards} # only basic rewards
total_reward = sum(rewards.values())
total_rewards.append(total_reward)
diff --git a/camel/environments/tic_tac_toe.py b/camel/environments/tic_tac_toe.py
index be6f17896..6474c93f3 100644
--- a/camel/environments/tic_tac_toe.py
+++ b/camel/environments/tic_tac_toe.py
@@ -11,6 +11,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+import asyncio
import math
import random
import re
@@ -19,6 +20,10 @@ from typing import Any, ClassVar, Dict, List, Literal, Optional, Tuple
from camel.environments.models import Action, Observation
from camel.environments.multi_step import MultiStepEnv
from camel.extractors import BaseExtractor, BaseExtractorStrategy
+from camel.logger import get_logger
+from camel.utils import with_timeout_async
+
+logger = get_logger(__name__)
class MoveExtractor(BaseExtractorStrategy):
@@ -254,11 +259,26 @@ class TicTacToeEnv(MultiStepEnv):
Returns:
None
+
+ Raises:
+ asyncio.TimeoutError: If the move extraction times out.
"""
board = self._state["board"]
# Attempt to parse the agent's chosen move
- extraction_result = await self.extractor.extract(action.llm_response)
+ try:
+ extraction_result = await with_timeout_async(
+ self.extractor.extract(action.llm_response),
+ timeout=self._timeout,
+ context="move extraction",
+ )
+ except asyncio.TimeoutError:
+ # Handle timeout gracefully
+ self._state["last_move_illegal"] = True
+ self._state["last_move"] = None
+ self._state["extraction_error"] = "Extraction timed out"
+ return
+
if not extraction_result:
# Handle extraction failure gracefully
self._state["last_move_illegal"] = True
diff --git a/camel/societies/role_playing.py b/camel/societies/role_playing.py
index 3df870376..fc00fd136 100644
--- a/camel/societies/role_playing.py
+++ b/camel/societies/role_playing.py
@@ -13,7 +13,14 @@
# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
import logging
import threading
-from typing import Dict, List, Optional, Sequence, Tuple, Union
+from typing import (
+ Dict,
+ List,
+ Optional,
+ Sequence,
+ Tuple,
+ Union,
+)
from camel.agents import (
ChatAgent,
@@ -28,6 +35,7 @@ from camel.models import BaseModelBackend
from camel.prompts import TextPrompt
from camel.responses import ChatAgentResponse
from camel.types import RoleType, TaskType
+from camel.utils import with_timeout_async
logger = logging.getLogger(__name__)
logger.setLevel(logging.WARNING)
@@ -627,7 +635,12 @@ class RolePlaying:
user agent terminated the conversation, and any additional user
information.
"""
- user_response = await self.user_agent.astep(assistant_msg)
+ user_response = await with_timeout_async(
+ self.user_agent.astep(assistant_msg),
+ context=f"waiting for user agent \
+ {self.user_agent.role_name} response",
+ )
+
if user_response.terminated or user_response.msgs is None:
return (
ChatAgentResponse(msgs=[], terminated=False, info={}),
@@ -648,7 +661,12 @@ class RolePlaying:
):
self.user_agent.record_message(user_msg)
- assistant_response = await self.assistant_agent.astep(user_msg)
+ assistant_response = await with_timeout_async(
+ self.assistant_agent.astep(user_msg),
+ context=f"waiting for assistant agent \
+ {self.assistant_agent.role_name} response",
+ )
+
if assistant_response.terminated or assistant_response.msgs is None:
return (
ChatAgentResponse(
diff --git a/camel/societies/workforce/role_playing_worker.py b/camel/societies/workforce/role_playing_worker.py
index d437cc84a..1aca0a6b1 100644
--- a/camel/societies/workforce/role_playing_worker.py
+++ b/camel/societies/workforce/role_playing_worker.py
@@ -13,12 +13,14 @@
# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
from __future__ import annotations
+import asyncio
import json
from typing import Dict, List, Optional
from colorama import Fore
from camel.agents.chat_agent import ChatAgent
+from camel.logger import get_logger
from camel.messages.base import BaseMessage
from camel.societies import RolePlaying
from camel.societies.workforce.prompts import (
@@ -28,7 +30,10 @@ from camel.societies.workforce.prompts import (
from camel.societies.workforce.utils import TaskResult
from camel.societies.workforce.worker import Worker
from camel.tasks.task import Task, TaskState, validate_task_content
-from camel.utils import print_text_animated
+from camel.utils import TIMEOUT_THRESHOLD, print_text_animated
+
+# Set up logging
+logger = get_logger(__name__)
class RolePlayingWorker(Worker):
@@ -82,7 +87,10 @@ class RolePlayingWorker(Worker):
self.user_agent_kwargs = user_agent_kwargs
async def _process_task(
- self, task: Task, dependencies: List[Task]
+ self,
+ task: Task,
+ dependencies: List[Task],
+ timeout: Optional[float] = TIMEOUT_THRESHOLD,
) -> TaskState:
r"""Processes a task leveraging its dependencies through role-playing.
@@ -101,6 +109,10 @@ class RolePlayingWorker(Worker):
Returns:
TaskState: `TaskState.DONE` if processed successfully, otherwise
`TaskState.FAILED`.
+
+ Raises:
+ Exception: If an error occurs during role-playing.
+ asyncio.TimeoutError: If the role-playing session times out.
"""
dependency_tasks_info = self._get_dep_tasks_info(dependencies)
prompt = ROLEPLAY_PROCESS_TASK_PROMPT.format(
@@ -121,9 +133,19 @@ class RolePlayingWorker(Worker):
chat_history = []
while n < self.chat_turn_limit:
n += 1
- assistant_response, user_response = await role_play_session.astep(
- input_msg
- )
+ try:
+ # add a timeout control
+ assistant_response, user_response = await asyncio.wait_for(
+ role_play_session.astep(input_msg), timeout=timeout
+ )
+ except asyncio.TimeoutError:
+ logger.error("Error: Role playing step timed out.")
+ chat_history.append("Error: Role playing step timed out.")
+ break
+ except Exception as e:
+ logger.error(f"Error during role playing step: {e!s}")
+ chat_history.append(f"Error during role playing: {e!s}")
+ break
if assistant_response.terminated:
reason = assistant_response.info['termination_reasons']
@@ -160,6 +182,7 @@ class RolePlayingWorker(Worker):
f"{Fore.RESET}\n",
delay=0.005,
)
+
chat_history.append(
f"AI Assistant: {assistant_response.msg.content}"
)
diff --git a/camel/societies/workforce/task_channel.py b/camel/societies/workforce/task_channel.py
index 63a3cb19e..288fdf14a 100644
--- a/camel/societies/workforce/task_channel.py
+++ b/camel/societies/workforce/task_channel.py
@@ -16,6 +16,7 @@ from enum import Enum
from typing import Dict, List, Optional
from camel.tasks import Task
+from camel.utils import TIMEOUT_THRESHOLD, with_timeout_async
class PacketStatus(Enum):
@@ -78,14 +79,25 @@ class Packet:
class TaskChannel:
r"""An internal class used by Workforce to manage tasks."""
- def __init__(self) -> None:
+ def __init__(self, timeout: float = TIMEOUT_THRESHOLD) -> None:
self._task_id_list: List[str] = []
self._condition = asyncio.Condition()
self._task_dict: Dict[str, Packet] = {}
+ self._timeout = timeout
- async def get_returned_task_by_publisher(self, publisher_id: str) -> Task:
+ async def get_returned_task_by_publisher(
+ self, publisher_id: str, timeout: float = TIMEOUT_THRESHOLD
+ ) -> Task:
r"""Get a task from the channel that has been returned by the
publisher.
+
+ Args:
+ publisher_id (str): The ID of the publisher
+ timeout (float): Maximum time to wait in seconds
+ (default: :obj:`180.0`)
+
+ Raises:
+ asyncio.TimeoutError: If waiting times out
"""
async with self._condition:
while True:
@@ -96,11 +108,27 @@ class TaskChannel:
if packet.status != PacketStatus.RETURNED:
continue
return packet.task
- await self._condition.wait()
- async def get_assigned_task_by_assignee(self, assignee_id: str) -> Task:
+ await with_timeout_async(
+ self._condition.wait(),
+ timeout=timeout,
+ context=f"waiting for returned task from publisher\
+ {publisher_id}",
+ )
+
+ async def get_assigned_task_by_assignee(
+ self, assignee_id: str, timeout: float = TIMEOUT_THRESHOLD
+ ) -> Task:
r"""Get a task from the channel that has been assigned to the
assignee.
+
+ Args:
+ assignee_id (str): The ID of the assignee
+ timeout (float): Maximum time to wait in seconds
+ (default: :obj:`180.0`)
+
+ Raises:
+ asyncio.TimeoutError: If waiting times out
"""
async with self._condition:
while True:
@@ -111,7 +139,13 @@ class TaskChannel:
and packet.assignee_id == assignee_id
):
return packet.task
- await self._condition.wait()
+
+ await with_timeout_async(
+ self._condition.wait(),
+ timeout=timeout,
+ context=f"waiting for assigned task to assignee\
+ {assignee_id}",
+ )
async def post_task(
self, task: Task, publisher_id: str, assignee_id: str
diff --git a/camel/societies/workforce/worker.py b/camel/societies/workforce/worker.py
index 899e14eb8..00da0b363 100644
--- a/camel/societies/workforce/worker.py
+++ b/camel/societies/workforce/worker.py
@@ -23,6 +23,7 @@ from camel.societies.workforce.base import BaseNode
from camel.societies.workforce.task_channel import TaskChannel
from camel.societies.workforce.utils import check_if_running
from camel.tasks.task import Task, TaskState
+from camel.utils import with_timeout_async
logger = logging.getLogger(__name__)
@@ -61,7 +62,10 @@ class Worker(BaseNode, ABC):
async def _get_assigned_task(self) -> Task:
r"""Get the task assigned to this node from the channel."""
- return await self._channel.get_assigned_task_by_assignee(self.node_id)
+ return await with_timeout_async(
+ self._channel.get_assigned_task_by_assignee(self.node_id),
+ context=f"getting assigned task for {self.node_id}",
+ )
@staticmethod
def _get_dep_tasks_info(dependencies: List[Task]) -> str:
@@ -90,30 +94,50 @@ class Worker(BaseNode, ABC):
while True:
# Get the earliest task assigned to this node
- task = await self._get_assigned_task()
+ task = await with_timeout_async(
+ self._get_assigned_task(),
+ context=f"getting assigned task for {self.node_id}",
+ )
+
print(
f"{Fore.YELLOW}{self} get task {task.id}: {task.content}"
f"{Fore.RESET}"
)
+
# Get the Task instance of dependencies
- dependency_ids = await self._channel.get_dependency_ids()
- task_dependencies = [
- await self._channel.get_task_by_id(dep_id)
- for dep_id in dependency_ids
- ]
+ dependency_ids = await with_timeout_async(
+ self._channel.get_dependency_ids(),
+ context=f"getting dependency IDs for task {task.id}",
+ )
+ task_dependencies = []
+ for dep_id in dependency_ids:
+ dep_task = await with_timeout_async(
+ self._channel.get_task_by_id(dep_id),
+ context=f"getting dependency task {dep_id}",
+ )
+ task_dependencies.append(dep_task)
# Process the task
- task_state = await self._process_task(task, task_dependencies)
+ task_state = await with_timeout_async(
+ self._process_task(task, task_dependencies),
+ context=f"processing task {task.id}",
+ )
# Update the result and status of the task
task.set_state(task_state)
- await self._channel.return_task(task.id)
+ await with_timeout_async(
+ self._channel.return_task(task.id),
+ context=f"returning task {task.id}",
+ )
@check_if_running(False)
async def start(self):
r"""Start the worker."""
- await self._listen_to_channel()
+ await with_timeout_async(
+ self._listen_to_channel(),
+ context=f"listening to channel for {self.node_id}",
+ )
@check_if_running(True)
def stop(self):
diff --git a/camel/societies/workforce/workforce.py b/camel/societies/workforce/workforce.py
index d82f356ca..d7fa6641f 100644
--- a/camel/societies/workforce/workforce.py
+++ b/camel/societies/workforce/workforce.py
@@ -43,7 +43,7 @@ from camel.societies.workforce.worker import Worker
from camel.tasks.task import Task, TaskState, validate_task_content
from camel.toolkits import CodeExecutionToolkit, SearchToolkit, ThinkingToolkit
from camel.types import ModelPlatformType, ModelType
-from camel.utils import dependencies_required
+from camel.utils import dependencies_required, with_timeout_async
logger = get_logger(__name__)
@@ -375,10 +375,16 @@ class Workforce(BaseNode):
return task_assign_result.assignee_id
async def _post_task(self, task: Task, assignee_id: str) -> None:
- await self._channel.post_task(task, self.node_id, assignee_id)
+ await with_timeout_async(
+ self._channel.post_task(task, self.node_id, assignee_id),
+ context=f"posting task {task.id} to assignee {assignee_id}",
+ )
async def _post_dependency(self, dependency: Task) -> None:
- await self._channel.post_dependency(dependency, self.node_id)
+ await with_timeout_async(
+ self._channel.post_dependency(dependency, self.node_id),
+ context=f"posting dependency {dependency.id}",
+ )
def _create_worker_node_for_task(self, task: Task) -> Worker:
r"""Creates a new worker node for a given task and add it to the
@@ -449,7 +455,10 @@ class Workforce(BaseNode):
r"""Get the task that's published by this node and just get returned
from the assignee.
"""
- return await self._channel.get_returned_task_by_publisher(self.node_id)
+ return await with_timeout_async(
+ self._channel.get_returned_task_by_publisher(self.node_id),
+ context=f"getting returned task for publisher {self.node_id}",
+ )
async def _post_ready_tasks(self) -> None:
r"""Send all the pending tasks that have all the dependencies met to
@@ -470,40 +479,68 @@ class Workforce(BaseNode):
ready_task.compose(self.task_agent)
# Remove the subtasks from the channel
for subtask in ready_task.subtasks:
- await self._channel.remove_task(subtask.id)
+ await with_timeout_async(
+ self._channel.remove_task(subtask.id),
+ context=f"removing subtask {subtask.id} from channel",
+ )
# Send the task to the channel as a dependency
- await self._post_dependency(ready_task)
+ await with_timeout_async(
+ self._post_dependency(ready_task),
+ context=f"posting dependency task {ready_task.id}",
+ )
self._pending_tasks.popleft()
# Try to send the next task in the pending list
- await self._post_ready_tasks()
+ await with_timeout_async(
+ self._post_ready_tasks(), context="posting next ready task"
+ )
else:
# Directly post the task to the channel if it's a new one
# Find a node to assign the task
assignee_id = self._find_assignee(task=ready_task)
- await self._post_task(ready_task, assignee_id)
+ await with_timeout_async(
+ self._post_task(ready_task, assignee_id),
+ context=f"posting task {ready_task.id} \
+ to assignee {assignee_id}",
+ )
async def _handle_failed_task(self, task: Task) -> bool:
if task.failure_count >= 3:
return True
task.failure_count += 1
# Remove the failed task from the channel
- await self._channel.remove_task(task.id)
+ await with_timeout_async(
+ self._channel.remove_task(task.id),
+ context=f"removing failed task {task.id} from channel",
+ )
if task.get_depth() >= 3:
# Create a new worker node and reassign
assignee = self._create_worker_node_for_task(task)
- await self._post_task(task, assignee.node_id)
+ await with_timeout_async(
+ self._post_task(task, assignee.node_id),
+ context=f"posting task {task.id} to new worker\
+ {assignee.node_id}",
+ )
else:
subtasks = self._decompose_task(task)
# Insert packets at the head of the queue
self._pending_tasks.extendleft(reversed(subtasks))
- await self._post_ready_tasks()
+ await with_timeout_async(
+ self._post_ready_tasks(),
+ context="posting ready tasks after task failure",
+ )
return False
async def _handle_completed_task(self, task: Task) -> None:
# archive the packet, making it into a dependency
self._pending_tasks.popleft()
- await self._channel.archive_task(task.id)
- await self._post_ready_tasks()
+ await with_timeout_async(
+ self._channel.archive_task(task.id),
+ context=f"archiving task {task.id}",
+ )
+ await with_timeout_async(
+ self._post_ready_tasks(),
+ context="posting ready tasks after task completion",
+ )
async def _graceful_shutdown(self, failed_task: Task) -> None:
r"""Handle graceful shutdown with configurable timeout. This is used to
@@ -534,14 +571,25 @@ class Workforce(BaseNode):
self._running = True
logger.info(f"Workforce {self.node_id} started.")
- await self._post_ready_tasks()
+ await with_timeout_async(
+ self._post_ready_tasks(),
+ context="posting ready tasks at the start",
+ )
while self._task is None or self._pending_tasks:
- returned_task = await self._get_returned_task()
+ returned_task = await with_timeout_async(
+ self._get_returned_task(), context="getting returned task"
+ )
if returned_task.state == TaskState.DONE:
- await self._handle_completed_task(returned_task)
+ await with_timeout_async(
+ self._handle_completed_task(returned_task),
+ context="handling completed task",
+ )
elif returned_task.state == TaskState.FAILED:
- halt = await self._handle_failed_task(returned_task)
+ halt = await with_timeout_async(
+ self._handle_failed_task(returned_task),
+ context="handling failed task",
+ )
if not halt:
continue
print(
@@ -568,7 +616,9 @@ class Workforce(BaseNode):
for child in self._children:
child_listening_task = asyncio.create_task(child.start())
self._child_listening_tasks.append(child_listening_task)
- await self._listen_to_channel()
+ await with_timeout_async(
+ self._listen_to_channel(), context="listening to channel"
+ )
@check_if_running(True)
def stop(self) -> None:
diff --git a/camel/utils/__init__.py b/camel/utils/__init__.py
index 9b8e4c3a8..4e7dff729 100644
--- a/camel/utils/__init__.py
+++ b/camel/utils/__init__.py
@@ -40,6 +40,7 @@ from .commons import (
to_pascal,
track_agent,
with_timeout,
+ with_timeout_async,
)
from .constants import Constants
from .deduplication import DeduplicationResult, deduplicate_internally
@@ -64,6 +65,8 @@ from .token_counting import (
get_model_encoding,
)
+TIMEOUT_THRESHOLD = Constants.TIMEOUT_THRESHOLD
+
__all__ = [
"print_text_animated",
"get_prompt_template_key_words",
@@ -106,6 +109,8 @@ __all__ = [
"sanitize_filename",
"browser_toolkit_save_auth_cookie",
"run_async",
+ "with_timeout_async",
+ "TIMEOUT_THRESHOLD",
"configure_langfuse",
"is_langfuse_available",
"get_current_agent_session_id",
diff --git a/camel/utils/commons.py b/camel/utils/commons.py
index 94f627568..4dd6fbf9d 100644
--- a/camel/utils/commons.py
+++ b/camel/utils/commons.py
@@ -29,6 +29,7 @@ from http import HTTPStatus
from pathlib import Path
from typing import (
Any,
+ Awaitable,
Callable,
Dict,
List,
@@ -51,6 +52,7 @@ from camel.types import TaskType
from .constants import Constants
F = TypeVar('F', bound=Callable[..., Any])
+T = TypeVar('T')
logger = logging.getLogger(__name__)
@@ -1130,3 +1132,28 @@ def run_async(func: Callable[..., Any]) -> Callable[..., Any]:
return loop.run_until_complete(func(*args, **kwargs))
return wrapper
+
+
+async def with_timeout_async(
+ coro: Awaitable[T],
+ timeout: Optional[float] = Constants.TIMEOUT_THRESHOLD,
+ context: str = "operation",
+) -> T:
+ r"""General timeout wrapper for async operations.
+
+ Args:
+ coro: async operation to be executed
+ timeout: max wait time (seconds)
+ context: context information, used for error messages
+
+ Returns:
+ result of the async operation
+
+ Raises:
+ asyncio.TimeoutError: if wait times out
+ """
+ try:
+ return await asyncio.wait_for(coro, timeout=timeout)
+ except asyncio.TimeoutError:
+ logger.error(f"Operation timed out after {timeout}s: {context}")
+ raise asyncio.TimeoutError(f"Timed out while {context}")
diff --git a/camel/utils/constants.py b/camel/utils/constants.py
index de8ea1c4c..79c91288a 100644
--- a/camel/utils/constants.py
+++ b/camel/utils/constants.py
@@ -35,3 +35,6 @@ class Constants:
# Default similarity threshold value for RAG
DEFAULT_SIMILARITY_THRESHOLD = 0.7
+
+ # Default timeout threshold value
+ TIMEOUT_THRESHOLD = 180.0
diff --git a/camel/verifiers/base.py b/camel/verifiers/base.py
index 5cd9038bd..ed6f305b8 100644
--- a/camel/verifiers/base.py
+++ b/camel/verifiers/base.py
@@ -18,7 +18,7 @@ from typing import List, Optional
from camel.extractors.base import BaseExtractor
from camel.logger import get_logger
-from camel.utils import BatchProcessor
+from camel.utils import TIMEOUT_THRESHOLD, BatchProcessor, with_timeout_async
from .models import VerificationOutcome, VerificationResult
@@ -47,7 +47,7 @@ class BaseVerifier(ABC):
self,
extractor: Optional[BaseExtractor] = None,
max_parallel: Optional[int] = None,
- timeout: Optional[float] = None,
+ timeout: Optional[float] = TIMEOUT_THRESHOLD,
max_retries: int = 3,
retry_delay: float = 1.0,
initial_batch_size: Optional[int] = None,
@@ -103,7 +103,9 @@ class BaseVerifier(ABC):
try:
if self.extractor:
- await self.extractor.setup()
+ await with_timeout_async(
+ self.extractor.setup(), context="setting up extractor"
+ )
batch_size = max(1, self._initial_batch_size or 10)
max_parallel = max(1, self._max_parallel or 1)
self._batch_processor = BatchProcessor()
@@ -113,7 +115,10 @@ class BaseVerifier(ABC):
f"batch_size={batch_size}, max_parallel={max_parallel}"
)
- await self._setup(**kwargs)
+ await with_timeout_async(
+ self._setup(**kwargs),
+ context=f"setting up {self.__class__.__name__}",
+ )
self._is_setup = True
except Exception as e:
@@ -121,7 +126,7 @@ class BaseVerifier(ABC):
f"Failed to initialize {self.__class__.__name__}: {e!s}"
)
logger.error(error_msg, exc_info=True)
- await self.cleanup()
+ await with_timeout_async(self.cleanup(), context="cleaning up")
raise RuntimeError(error_msg) from e
@abstractmethod
@@ -144,9 +149,14 @@ class BaseVerifier(ABC):
try:
if self.extractor:
- await self.extractor.cleanup()
+ await with_timeout_async(
+ self.extractor.cleanup(), context="cleaning up extractor"
+ )
self._batch_processor = BatchProcessor()
- await self._cleanup()
+ await with_timeout_async(
+ self._cleanup(),
+ context=f"cleaning up {self.__class__.__name__}",
+ )
logger.info(f"{self.__class__.__name__} cleaned up successfully")
except Exception as e:
@@ -194,7 +204,11 @@ class BaseVerifier(ABC):
logger.warning(
f"{self.__class__.__name__} not set up, calling setup()"
)
- await self.setup()
+ await with_timeout_async(
+ self.setup(),
+ context="setting \
+ up verifier",
+ )
attempt = 0
start_time = time.time()
@@ -203,7 +217,10 @@ class BaseVerifier(ABC):
# Extract verifiable part of the proposed solution,
# if verifier has been initialized with extractor.
verifiable_solution = (
- await self.extractor.extract(solution)
+ await with_timeout_async(
+ self.extractor.extract(solution),
+ context="extracting solution",
+ )
if self.extractor
else solution
)
@@ -222,21 +239,19 @@ class BaseVerifier(ABC):
f"Failed to extract verifiable solution on attempt "
f"{attempt}, retrying..."
)
- await asyncio.sleep(self._retry_delay)
+ await with_timeout_async(
+ asyncio.sleep(self._retry_delay),
+ context="waiting before retrying",
+ )
continue
try:
- verification_result = (
- await asyncio.wait_for(
- self._verify_implementation(
- verifiable_solution, reference_answer
- ),
- timeout=self._timeout,
- )
- if self._timeout
- else await self._verify_implementation(
+ verification_result = await with_timeout_async(
+ self._verify_implementation(
verifiable_solution, reference_answer
- )
+ ),
+ timeout=self._timeout,
+ context="verifying solution",
)
verification_result.duration = time.time() - start_time
@@ -257,7 +272,10 @@ class BaseVerifier(ABC):
logger.warning(
f"Verification timeout on attempt {attempt}, retrying..."
)
- await asyncio.sleep(self._retry_delay)
+ await with_timeout_async(
+ asyncio.sleep(self._retry_delay),
+ context="waiting before retrying",
+ )
except Exception as e:
attempt += 1
@@ -269,7 +287,10 @@ class BaseVerifier(ABC):
duration=time.time() - start_time,
metadata={"attempt": attempt},
)
- await asyncio.sleep(self._retry_delay)
+ await with_timeout_async(
+ asyncio.sleep(self._retry_delay),
+ context="waiting before retrying",
+ )
return VerificationResult(
status=VerificationOutcome.ERROR,
@@ -339,7 +360,11 @@ class BaseVerifier(ABC):
logger.warning(
f"{self.__class__.__name__} not set up, calling setup()"
)
- await self.setup()
+ await with_timeout_async(
+ self.setup(),
+ context="setting\
+ up verifier",
+ )
# Retrieve batch processing settings
max_workers = getattr(
@@ -356,8 +381,10 @@ class BaseVerifier(ABC):
start_time = time.time()
try:
async with semaphore:
- verification_result = await self.verify(
- solution, reference_answer
+ verification_result = await with_timeout_async(
+ self.verify(solution, reference_answer),
+ timeout=self._timeout,
+ context="verifying solution",
)
processing_time = time.time() - start_time
success = (
@@ -391,7 +418,11 @@ class BaseVerifier(ABC):
)
]
try:
- batch_results = await asyncio.gather(*verification_tasks)
+ batch_results = await with_timeout_async(
+ asyncio.gather(*verification_tasks),
+ timeout=self._timeout,
+ context="verifying solutions",
+ )
all_results.extend(batch_results)
except Exception as e:
logger.error(
diff --git a/camel/verifiers/python_verifier.py b/camel/verifiers/python_verifier.py
index 7cc2e7932..ca3f71542 100644
--- a/camel/verifiers/python_verifier.py
+++ b/camel/verifiers/python_verifier.py
@@ -24,6 +24,7 @@ from typing import Any, List, Optional, Tuple
from camel.extractors.base import BaseExtractor
from camel.logger import get_logger
+from camel.utils import with_timeout_async
from camel.verifiers import BaseVerifier
from .models import VerificationOutcome, VerificationResult
@@ -49,7 +50,6 @@ class PythonVerifier(BaseVerifier):
def __init__(
self,
extractor: Optional[BaseExtractor] = None,
- timeout: Optional[float] = 30.0,
required_packages: Optional[List[str]] = None,
float_tolerance: Optional[float] = None,
**kwargs,
@@ -68,7 +68,7 @@ class PythonVerifier(BaseVerifier):
floating point comparisons. (default: :obj:`None`)
"""
# TODO: Use CAMEL's Interpreter to execute the code
- super().__init__(extractor=extractor, timeout=timeout, **kwargs)
+ super().__init__(extractor=extractor, **kwargs)
self.venv_path: Optional[str] = None
self.required_packages = required_packages or []
self.float_tolerance = float_tolerance
@@ -309,8 +309,10 @@ class PythonVerifier(BaseVerifier):
)
try:
- sol_out, sol_err, sol_code = await self._run_code_block(
- solution, venv_python
+ sol_out, sol_err, sol_code = await with_timeout_async(
+ self._run_code_block(solution, venv_python),
+ timeout=self._timeout,
+ context="running Python solution",
)
if sol_code != 0:
return VerificationResult(
@@ -416,8 +418,10 @@ class PythonVerifier(BaseVerifier):
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
)
- stdout, stderr = await asyncio.wait_for(
- proc.communicate(), timeout=self._timeout
+ stdout, stderr = await with_timeout_async(
+ proc.communicate(),
+ timeout=self._timeout,
+ context="executing Python code",
)
os.remove(tmp_path)
return (
diff --git a/test/environments/test_multi_step_env.py b/test/environments/test_multi_step_env.py
index 219bcd02e..8e9b6f80f 100644
--- a/test/environments/test_multi_step_env.py
+++ b/test/environments/test_multi_step_env.py
@@ -11,9 +11,10 @@
# See the License for the specific language governing permissions and
# limitations under the License.
# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+import asyncio
import re
from typing import Any, Dict, Tuple
-from unittest.mock import AsyncMock, MagicMock
+from unittest.mock import AsyncMock, MagicMock, patch
import pytest
@@ -166,3 +167,88 @@ async def test_multi_step_env_error_handling():
assert env_max_steps._current_step == 1, "Current step should be 1"
assert env_max_steps.max_steps == 1, "Max steps should be 1"
assert obs.question == "Episode ended", "Should reach terminal observation"
+
+
+async def test_multi_step_env_timeout_handling():
+ r"""Test timeout handling in MultiStepEnv using mocks instead of sleep."""
+
+ # Create mock functions that raise TimeoutError
+ async def mock_update_state_timeout(*args, **kwargs):
+ raise asyncio.TimeoutError("Simulated timeout in _update_state")
+
+ async def mock_compute_reward_timeout(*args, **kwargs):
+ raise asyncio.TimeoutError("Simulated timeout in compute_reward")
+
+ async def mock_extract_timeout(*args, **kwargs):
+ raise asyncio.TimeoutError("Simulated timeout in extract")
+
+ # Create mock extractor
+ mock_extractor = MagicMock()
+ mock_extractor.setup = AsyncMock()
+ mock_extractor.cleanup = AsyncMock()
+ mock_extractor.extract = AsyncMock(side_effect=mock_extract_timeout)
+
+ # 1. Test _update_state timeout in step method
+ env = MockMultiStepEnv(extractor=mock_extractor, max_steps=3)
+
+ try:
+ await env.setup()
+ _obs = await env.reset()
+
+ # Replace _update_state with our mocked timeout version
+ with patch.object(
+ MockMultiStepEnv,
+ '_update_state',
+ new=AsyncMock(side_effect=mock_update_state_timeout),
+ ):
+ # The step should not hang - it should handle the timeout
+ # and continue
+ action = Action(llm_response="test action")
+ observation, reward, done, info = await env.step(action)
+
+ # Verify error info is present
+ assert "error" in info
+ assert "timed out" in info["error"].lower()
+ finally:
+ # Ensure environment is closed
+ await env.close()
+
+ # 2. Test compute_reward timeout in step
+ env_reward = MockMultiStepEnv(extractor=mock_extractor, max_steps=3)
+
+ try:
+ await env_reward.setup()
+ _obs = await env_reward.reset()
+
+ # Replace compute_reward with our mocked timeout version
+ with patch.object(
+ MockMultiStepEnv,
+ 'compute_reward',
+ new=AsyncMock(side_effect=mock_compute_reward_timeout),
+ ):
+ action = Action(llm_response="test action")
+ observation, reward, done, info = await env_reward.step(action)
+
+ # Verify reward is default value due to timeout
+ assert reward == 0.0
+ assert "error" in info
+ assert "timed out" in info["error"].lower()
+ finally:
+ await env_reward.close()
+
+ # 3. Test setup timeout in reset
+ setup_env = MockMultiStepEnv(extractor=mock_extractor, max_steps=3)
+
+ # Mock setup to raise TimeoutError
+ with patch.object(
+ MockMultiStepEnv,
+ '_setup',
+ side_effect=asyncio.TimeoutError("Simulated timeout in setup"),
+ ):
+ # Reset should propagate the timeout exception
+ with pytest.raises(asyncio.TimeoutError):
+ await setup_env.reset()
+
+ # Clean up
+ await setup_env.close()
diff --git a/test/environments/test_single_step_env.py b/test/environments/test_single_step_env.py
index 64827b7b0..8dbfba743 100644
--- a/test/environments/test_single_step_env.py
+++ b/test/environments/test_single_step_env.py
@@ -11,9 +11,10 @@
# See the License for the specific language governing permissions and
# limitations under the License.
# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+import asyncio
import re
from typing import Dict, List
-from unittest.mock import AsyncMock, MagicMock
+from unittest.mock import AsyncMock, MagicMock, patch
import pytest
@@ -1012,3 +1013,71 @@ async def test_single_step_env_all_input_types():
await env.step([Action(index=2, llm_response="4")])
await env.close()
+
+
+async def test_single_step_env_timeout_handling():
+ r"""Test timeout handling in SingleStepEnv."""
+ # Define a valid dataset
+ data = [
+ {
+ "question": "What is 2 + 2?",
+ "final_answer": "4",
+ "rationale": "Adding 2 and 2 gives 4.",
+ "metadata": {"difficulty": "easy"},
+ },
+ ]
+ dataset = StaticDataset(data)
+
+ # **1. Test Custom Reward Timeout**
+ # Create a specialized SingleStepEnv that simulates timeout
+ class TimeoutCustomRewardEnv(SingleStepEnv):
+ async def _compute_custom_reward(self, *args, **kwargs):
+ return {"custom_reward": 5.0}
+
+ # Create a patched asyncio.wait_for that raises TimeoutError
+ # for _compute_custom_reward
+ original_wait_for = asyncio.wait_for
+
+ async def mock_wait_for(coro, timeout, *args, **kwargs):
+ # Check if this is the _compute_custom_reward coroutine
+ if "_compute_custom_reward" in str(coro):
+ raise asyncio.TimeoutError("Simulated timeout")
+ # Otherwise, pass through to the original function
+ return await original_wait_for(coro, timeout, *args, **kwargs)
+
+ # Create environment with mock verifier
+ mock_verifier = create_mock_verifier()
+ env = TimeoutCustomRewardEnv(dataset=dataset, verifier=mock_verifier)
+
+ # Setup and reset environment
+ await env.setup()
+ _obs = await env.reset(batch_size=1)
+
+ # Test step with patched wait_for
+ with patch('asyncio.wait_for', side_effect=mock_wait_for):
+ action = Action(llm_response="The answer is 4")
+ observation, reward, done, info = await env.step(action)
+
+ # Verify that only correctness reward is present
+ # (custom reward timed out)
+ rewards_dict = info["rewards_dict"]
+ assert "correctness" in rewards_dict
+ assert "custom_reward" not in rewards_dict
+ assert reward == env.ACCURACY_REWARD # Only accuracy reward applied
+
+ # Test cleanup
+ await env.close()
+ assert env._is_setup is False
+ mock_verifier.cleanup.assert_awaited_once()
+
+ # **2. Test Reset Timeout**
+ # Create a new environment for reset timeout testing
+ reset_env = SingleStepEnv(dataset=dataset, verifier=create_mock_verifier())
+
+ # Patch setup method to raise TimeoutError when called from reset
+ with patch.object(
+ SingleStepEnv, 'setup', side_effect=asyncio.TimeoutError
+ ):
+ with pytest.raises(asyncio.TimeoutError):
+ await reset_env.reset()
diff --git a/test/environments/test_tictactoe_env.py b/test/environments/test_tictactoe_env.py
index f41aa2198..6d305f7d3 100644
--- a/test/environments/test_tictactoe_env.py
+++ b/test/environments/test_tictactoe_env.py
@@ -347,3 +347,39 @@ async def test_full_game():
assert done is False
await env.close()
+
+
+async def test_extraction_timeout():
+ r"""Test timeout handling during move extraction in TicTacToeEnv."""
+ import asyncio
+ from unittest.mock import patch
+
+ env = TicTacToeEnv()
+ await env.setup()
+ await env.reset()
+
+ # Store original wait_for to restore later
+ original_wait_for = asyncio.wait_for
+
+ async def mock_wait_for(coro, timeout, *args, **kwargs):
+ # Check if this is the extractor.extract call
+ if "extract" in str(coro):
+ raise asyncio.TimeoutError("Simulated timeout during extraction")
+ # Otherwise, pass through to the original function
+ return await original_wait_for(coro, timeout, *args, **kwargs)
+
+ # Patch asyncio.wait_for to simulate timeout
+ with patch('asyncio.wait_for', side_effect=mock_wait_for):
+ action = Action(llm_response="<Action>5</Action>")
+ obs, reward, done, info = await env.step(action)
+
+ # Verify timeout was handled properly
+ state = env._state
+ assert state["last_move_illegal"] is True
+ assert state["extraction_error"] == "Extraction timed out"
+ assert reward == 0.0 # No reward for illegal move
+ assert done is False # Game continues
+ assert "Your last move was illegal" in obs.question
+
+ await env.close()
diff --git a/test/verifiers/test_base_verifier.py b/test/verifiers/test_base_verifier.py
index 831c8c7c9..800d59ed3 100644
--- a/test/verifiers/test_base_verifier.py
+++ b/test/verifiers/test_base_verifier.py
@@ -18,6 +18,7 @@ from unittest.mock import AsyncMock, patch
import pytest
+from camel.utils import with_timeout_async
from camel.verifiers.base import BaseVerifier
from camel.verifiers.models import VerificationOutcome, VerificationResult
@@ -68,7 +69,7 @@ def test_verifier_init():
r"""Test BaseVerifier initialization with various parameters."""
verifier = TestVerifier()
assert verifier._max_parallel is None
- assert verifier._timeout is None
+ assert verifier._timeout == 180.0
assert verifier._max_retries == 3
assert verifier._retry_delay == 1.0
assert verifier._is_setup is False
@@ -194,18 +195,22 @@ async def test_verify_timeout(test_verifier):
r"""Test verification timeout."""
await test_verifier.setup()
- with patch(
- "asyncio.wait_for",
- side_effect=asyncio.TimeoutError("Simulated timeout"),
+ # dont verify directly
+ with (
+ patch.object(
+ test_verifier,
+ "_verify_implementation",
+ side_effect=asyncio.TimeoutError("Simulated timeout"),
+ ),
+ patch("asyncio.sleep", new_callable=AsyncMock),
):
result = await test_verifier.verify(
solution="This will timeout",
reference_answer="Expected response",
)
- assert result.status == VerificationOutcome.TIMEOUT
- assert "timed out" in result.error_message
- assert result.duration > 0
+ assert result.status == VerificationOutcome.TIMEOUT
+ assert "Verification timed out after all retries" in result.error_message
await test_verifier.cleanup()
@@ -258,9 +263,10 @@ async def test_verify_batch(test_verifier):
)
solutions = ["Success 1", "Success 2", "This will fail"]
- ground_truthes = ["Expected 1", "Expected 2", "Expected 3"]
-
- results = await test_verifier.verify_batch(solutions, ground_truthes)
+ reference_answers = ["reference"] * len(solutions)
+ results = await test_verifier.verify_batch(
+ solutions, reference_answers
+ )
assert len(results) == 3
assert results[0].status == VerificationOutcome.SUCCESS
@@ -388,3 +394,136 @@ async def test_full_verification_flow():
finally:
await verifier.cleanup()
assert verifier._is_setup is False
+
+
+async def test_with_timeout_function():
+ """Test the with_timeout function handles completions and\
+ timeouts correctly"""
+
+ # Test normal operation (successful completion)
+ mock_coro = AsyncMock()
+ mock_coro.return_value = "success"
+ result = await with_timeout_async(mock_coro(), context="test operation")
+ assert result == "success"
+
+ # Test timeout handling
+ mock_timeout_coro = AsyncMock()
+ mock_timeout_coro.side_effect = asyncio.TimeoutError("Simulated timeout")
+
+ with pytest.raises(asyncio.TimeoutError) as exc_info:
+ await with_timeout_async(
+ mock_timeout_coro(), context="test timeout operation"
+ )
+
+ assert "Timed out while test timeout operation" in str(exc_info.value)
+
+
+async def test_timeout_in_extractor_setup():
+ """Test timeout handling during extractor setup"""
+ mock_extractor = AsyncMock()
+ mock_extractor.setup.side_effect = asyncio.TimeoutError(
+ "Simulated timeout in extractor setup"
+ )
+
+ verifier = TestVerifier()
+ verifier.extractor = mock_extractor
+
+ with pytest.raises(RuntimeError) as exc_info:
+ await verifier.setup()
+
+ assert "Failed to initialize" in str(exc_info.value)
+ mock_extractor.setup.assert_called_once()
+
+
+async def test_timeout_in_verify_implementation(test_verifier):
+ """Test timeout during verify implementation"""
+ await test_verifier.setup()
+
+ # Test that a TimeoutError in _verify_implementation is handled with retry
+ with (
+ patch.object(
+ test_verifier,
+ "_verify_implementation",
+ side_effect=[
+ asyncio.TimeoutError("Simulated timeout"),
+ VerificationResult(
+ status=VerificationOutcome.SUCCESS,
+ result="Success after retry",
+ ),
+ ],
+ ),
+ patch("asyncio.sleep", new_callable=AsyncMock),
+ ):
+ result = await test_verifier.verify("solution", "reference")
+
+ assert result.status == VerificationOutcome.SUCCESS
+ assert result.result == "Success after retry"
+ assert result.metadata["attempt"] == 2 # Second attempt succeeded
+
+ await test_verifier.cleanup()
+
+
+async def test_repeated_timeout_in_verify_implementation(test_verifier):
+ """Test multiple consecutive timeouts leading to timeout outcome"""
+ await test_verifier.setup()
+ test_verifier._max_retries = 2 # Set max retries to 2
+
+ # Simulate repeated timeouts exceeding max_retries
+ with (
+ patch.object(
+ test_verifier,
+ "_verify_implementation",
+ side_effect=asyncio.TimeoutError("Simulated timeout"),
+ ),
+ patch("asyncio.sleep", new_callable=AsyncMock),
+ ):
+ result = await test_verifier.verify("solution", "reference")
+
+ # After max retries are exhausted, should return TIMEOUT outcome
+ assert result.status == VerificationOutcome.TIMEOUT
+ assert "Verification timed out after all retries" in result.error_message
+ assert result.metadata["attempt"] == 2 # Attempted max_retries times
+
+ await test_verifier.cleanup()
+
+
+async def test_timeout_in_batch_verification(test_verifier):
+ """Test timeout handling in batch verification"""
+ await test_verifier.setup()
+
+ # Create a mix of successful and timeout solutions
+ solutions = ["solution1", "solution2", "timeout"]
+ reference_answers = ["reference"] * len(solutions)
+
+ # Mock the verify method to simulate a timeout for specific solutions
+ original_verify = test_verifier.verify
+
+ async def mock_verify(solution, reference=None):
+ if "timeout" in solution:
+ raise asyncio.TimeoutError("Simulated timeout")
+ return await original_verify(solution, reference)
+
+ with patch.object(test_verifier, "verify", side_effect=mock_verify):
+ results = await test_verifier.verify_batch(
+ solutions, reference_answers
+ )
+
+ # Should have 3 results
+ assert len(results) == 3
+
+ # Check individual results
+ success_results = [
+ r for r in results if r.status == VerificationOutcome.SUCCESS
+ ]
+
+ non_success_results = [
+ r for r in results if r.status != VerificationOutcome.SUCCESS
+ ]
+ assert len(non_success_results) == 1
+ assert len(success_results) == 2
+ await test_verifier.cleanup()
diff --git a/test/verifiers/test_python_verifier.py b/test/verifiers/test_python_verifier.py
index 2b548869a..914c9c8af 100644
--- a/test/verifiers/test_python_verifier.py
+++ b/test/verifiers/test_python_verifier.py
@@ -12,12 +12,11 @@
# limitations under the License.
# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
import os
+from unittest.mock import patch
import pytest
-from camel.verifiers.models import (
- VerificationOutcome,
-)
+from camel.verifiers.models import VerificationOutcome
from camel.verifiers.python_verifier import PythonVerifier
@@ -239,3 +238,50 @@ async def test_verify_with_float_tolerance(
), f"Expected failure, got {result}"
await verifier.cleanup()
+
+
+async def test_python_verifier_run_code_block_timeout(python_verifier):
+ """Test timeout handling in _run_code_block method."""
+ import asyncio
+
+ await python_verifier.setup(uv=True)
+
+ if python_verifier.venv_path is None:
+ python_verifier.venv_path = "/tmp/mock_venv"
+
+ # Mock proc.communicate to raise TimeoutError
+ with patch(
+ 'asyncio.subprocess.Process.communicate',
+ side_effect=asyncio.TimeoutError("Simulated timeout"),
+ ):
+ with pytest.raises(asyncio.TimeoutError):
+ await python_verifier._run_code_block(
+ 'print("test")', python_verifier.venv_path + "/bin/python"
+ )
+
+ await python_verifier.cleanup()
+
+
+async def test_python_verifier_verify_timeout_propagation(python_verifier):
+ """Test timeout propagation in the verification implementation."""
+ import asyncio
+
+ await python_verifier.setup(uv=True)
+
+ # Mock _run_code_block to raise TimeoutError
+ with patch.object(
+ python_verifier,
+ '_run_code_block',
+ side_effect=asyncio.TimeoutError("Simulated timeout"),
+ ):
+ result = await python_verifier._verify_implementation(
+ solution="print('test')", reference_answer=None
+ )
+
+ # Check that the timeout is properly handled
+ assert result.status == VerificationOutcome.TIMEOUT
+ assert "timed out" in result.error_message.lower()
+
+ await python_verifier.cleanup()
diff --git a/test/workforce/test_role_play.py b/test/workforce/test_role_play.py
new file mode 100644
index 000000000..1666cb3e4
--- /dev/null
+++ b/test/workforce/test_role_play.py
@@ -0,0 +1,258 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+
+import asyncio
+from unittest.mock import AsyncMock, MagicMock, patch
+
+import pytest
+
+from camel.agents import ChatAgent
+from camel.messages import BaseMessage
+from camel.responses import ChatAgentResponse
+from camel.societies.role_playing import RolePlaying
+from camel.utils import with_timeout_async
+
+
+async def test_with_timeout_function():
+ r"""Test the with_timeout function handles completions and\
+ timeouts correctly"""
+ # Test normal operation (successful completion)
+ mock_coro = AsyncMock()
+ mock_coro.return_value = "success"
+ result = await with_timeout_async(mock_coro(), context="test operation")
+ assert result == "success"
+
+ # Test timeout handling
+ mock_timeout_coro = AsyncMock()
+ mock_timeout_coro.side_effect = asyncio.TimeoutError("Simulated timeout")
+
+ with pytest.raises(asyncio.TimeoutError) as exc_info:
+ await with_timeout_async(
+ mock_timeout_coro(), context="test timeout operation"
+ )
+
+ assert "Timed out while test timeout operation" in str(exc_info.value)
+
+
+async def test_user_agent_timeout():
+ r"""Test handling timeout from user agent in RolePlaying.astep"""
+ # Create mock agents
+ mock_user_agent = AsyncMock(spec=ChatAgent)
+ mock_user_agent.astep.side_effect = asyncio.TimeoutError()
+ mock_user_agent.role_name = "mock_user"
+
+ # Add model_backend to both mock agents
+ mock_user_agent.model_backend = MagicMock()
+ mock_user_agent.model_backend.model_config_dict = {}
+
+ mock_assistant_agent = AsyncMock(spec=ChatAgent)
+ mock_assistant_agent.model_backend = MagicMock()
+ mock_assistant_agent.model_backend.model_config_dict = {}
+
+ # Create RolePlaying instance with mocked agents
+ role_playing = RolePlaying("assistant", "user")
+ role_playing.assistant_agent = mock_assistant_agent
+ role_playing.user_agent = mock_user_agent
+
+ # Create mock message
+ mock_msg = MagicMock(spec=BaseMessage)
+
+ # Test that TimeoutError is propagated
+ with pytest.raises(asyncio.TimeoutError):
+ await role_playing.astep(mock_msg)
+
+ # Verify the user agent's astep was called
+ mock_user_agent.astep.assert_called_once_with(mock_msg)
+ # Verify the assistant agent's astep was not called
+ mock_assistant_agent.astep.assert_not_called()
+
+
+async def test_assistant_agent_timeout():
+ r"""Test handling timeout from assistant agent in RolePlaying.astep"""
+ # Create mock agents with model_backend
+ mock_user_agent = AsyncMock(spec=ChatAgent)
+ mock_user_agent.model_backend = MagicMock()
+ mock_user_agent.model_backend.model_config_dict = {}
+
+ mock_assistant_agent = AsyncMock(spec=ChatAgent)
+ mock_assistant_agent.model_backend = MagicMock()
+ mock_assistant_agent.model_backend.model_config_dict = {}
+
+ # Configure user_agent to return normal response and assistant_agent
+ # to timeout
+ mock_user_response = MagicMock(spec=ChatAgentResponse)
+ mock_user_response.msgs = [MagicMock(spec=BaseMessage)]
+ mock_user_response.terminated = False
+ mock_user_agent.astep.return_value = mock_user_response
+ mock_user_agent.role_name = "mock_user"
+
+ mock_assistant_agent.astep.side_effect = asyncio.TimeoutError()
+ mock_assistant_agent.role_name = "mock_assistant"
+
+ # Create RolePlaying instance with mocked agents
+ role_playing = RolePlaying("assistant", "user")
+ role_playing.assistant_agent = mock_assistant_agent
+ role_playing.user_agent = mock_user_agent
+ role_playing._reduce_message_options = lambda msgs: msgs[
+ 0
+ ] # Mock this function
+
+ # Create mock message
+ mock_msg = MagicMock(spec=BaseMessage)
+
+ # Test that TimeoutError is propagated
+ with pytest.raises(asyncio.TimeoutError):
+ await role_playing.astep(mock_msg)
+
+ # Verify both agents' astep methods were called
+ mock_user_agent.astep.assert_called_once_with(mock_msg)
+ mock_assistant_agent.astep.assert_called_once()
+
+
+async def test_recovery_from_timeout():
+ r"""Test RolePlaying can recover from an agent timeout"""
+ # Create mock agents with model_backend
+ mock_user_agent = AsyncMock(spec=ChatAgent)
+ mock_user_agent.model_backend = MagicMock()
+ mock_user_agent.model_backend.model_config_dict = {}
+
+ mock_assistant_agent = AsyncMock(spec=ChatAgent)
+ mock_assistant_agent.model_backend = MagicMock()
+ mock_assistant_agent.model_backend.model_config_dict = {}
+
+ # First call to user_agent.astep will timeout, second will succeed
+ call_count = 0
+
+ async def user_agent_side_effect(msg):
+ nonlocal call_count
+ call_count += 1
+ if call_count == 1:
+ raise asyncio.TimeoutError("Simulated timeout")
+
+ response = MagicMock(spec=ChatAgentResponse)
+ response.msgs = [MagicMock(spec=BaseMessage)]
+ response.terminated = False
+ response.info = {}
+ return response
+
+ mock_user_agent.astep.side_effect = user_agent_side_effect
+ mock_user_agent.role_name = "mock_user"
+
+ # Assistant agent returns normal response
+ mock_assistant_response = MagicMock(spec=ChatAgentResponse)
+ mock_assistant_response.msgs = [MagicMock(spec=BaseMessage)]
+ mock_assistant_response.terminated = False
+ mock_assistant_response.info = {}
+ mock_assistant_agent.astep.return_value = mock_assistant_response
+ mock_assistant_agent.role_name = "mock_assistant"
+
+ # Create RolePlaying instance with mocked agents
+ role_playing = RolePlaying("assistant", "user")
+ role_playing.assistant_agent = mock_assistant_agent
+ role_playing.user_agent = mock_user_agent
+ role_playing._reduce_message_options = lambda msgs: msgs[
+ 0
+ ] # Mock this function
+
+ # Create mock message
+ mock_msg = MagicMock(spec=BaseMessage)
+
+ # First attempt should raise TimeoutError
+ with pytest.raises(asyncio.TimeoutError):
+ await role_playing.astep(mock_msg)
+
+ # Second attempt should succeed
+ user_response, assistant_response = await role_playing.astep(mock_msg)
+
+ # Verify both responses are returned
+ assert user_response is not None
+ assert assistant_response is not None
+ assert user_response.terminated is False
+ assert assistant_response.terminated is False
+
+ # Verify both agents were called the expected number of times
+ assert mock_user_agent.astep.call_count == 2
+ assert mock_assistant_agent.astep.call_count == 1
+
+
+async def test_integration_with_timeout():
+ r"""Test the full RolePlaying.astep flow with various timeout points"""
+
+ # Setup patching of with_timeout - properly await the coroutine
+ async def mock_with_timeout(coro, **kwargs):
+ # Make sure we await the coroutine
+ return await coro
+
+ # Create properly structured return values
+ mock_user_msg = MagicMock(spec=BaseMessage)
+ mock_assistant_msg = MagicMock(spec=BaseMessage)
+
+ mock_user_response = MagicMock(spec=ChatAgentResponse)
+ mock_user_response.msgs = [mock_user_msg]
+ mock_user_response.terminated = False
+ mock_user_response.info = {}
+
+ mock_assistant_response = MagicMock(spec=ChatAgentResponse)
+ mock_assistant_response.msgs = [mock_assistant_msg]
+ mock_assistant_response.terminated = False
+ mock_assistant_response.info = {}
+
+ # Create mocked agents that return proper values
+ mock_user_agent = AsyncMock(spec=ChatAgent)
+ mock_user_agent.astep.return_value = mock_user_response
+ mock_user_agent.role_name = "mock_user"
+ mock_user_agent.model_backend = MagicMock()
+ mock_user_agent.model_backend.model_config_dict = {}
+
+ mock_assistant_agent = AsyncMock(spec=ChatAgent)
+ mock_assistant_agent.astep.return_value = mock_assistant_response
+ mock_assistant_agent.role_name = "mock_assistant"
+ mock_assistant_agent.model_backend = MagicMock()
+ mock_assistant_agent.model_backend.model_config_dict = {}
+
+ # Create RolePlaying instance
+ role_playing = RolePlaying("assistant", "user")
+ role_playing.assistant_agent = mock_assistant_agent
+ role_playing.user_agent = mock_user_agent
+ role_playing._reduce_message_options = lambda msgs: msgs[
+ 0
+ ] # Mock this function
+
+ # Create test message
+ test_msg = MagicMock(spec=BaseMessage)
+
+ # Test the normal flow (no timeouts)
+ with patch(
+ "camel.utils.with_timeout_async",
+ side_effect=mock_with_timeout,
+ ):
+ user_response, assistant_response = await role_playing.astep(test_msg)
+
+ # validate response
+ assert len(user_response.msgs) == 1
+ assert user_response.terminated is False
+ assert user_response.info == {}
+
+ assert len(assistant_response.msgs) == 1
+ assert assistant_response.terminated is False
+ assert assistant_response.info == {}
+
+ # verify both agents were called
+ mock_user_agent.astep.assert_called_with(test_msg)
+ mock_assistant_agent.astep.assert_called_with(mock_user_msg)
diff --git a/test/workforce/test_worker.py b/test/workforce/test_worker.py
new file mode 100644
index 000000000..a3d6534d7
--- /dev/null
+++ b/test/workforce/test_worker.py
@@ -0,0 +1,259 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+import asyncio
+from unittest.mock import AsyncMock, MagicMock, patch
+
+import pytest
+
+from camel.societies.workforce.task_channel import TaskChannel
+from camel.societies.workforce.worker import Worker
+from camel.tasks.task import Task, TaskState
+from camel.utils import TIMEOUT_THRESHOLD, with_timeout_async
+
+
+class TestTimeoutWorker(Worker):
+ r"""Test Worker implementation for timeout testing"""
+
+ def __init__(self, description="Test Worker", node_id=None):
+ super().__init__(description, node_id=node_id)
+
+ async def _process_task(self, task, dependencies):
+ r"""Simple implementation of task processing method"""
+ return TaskState.DONE
+
+
+async def test_with_timeout_function():
+ r"""Test the basic functionality of with_timeout function"""
+ # Test normal case
+ mock_coro = AsyncMock()
+ mock_coro.return_value = "success"
+
+ result = await with_timeout_async(mock_coro(), timeout=1.0, context="test")
+ assert result == "success"
+
+ # Test timeout case by directly mocking timeout error
+ mock_failing_coro = AsyncMock()
+ mock_failing_coro.side_effect = asyncio.TimeoutError("Simulated timeout")
+
+ with pytest.raises(asyncio.TimeoutError) as excinfo:
+ await with_timeout_async(
+ mock_failing_coro(), timeout=1.0, context="test timeout"
+ )
+
+ assert "Timed out while test timeout" in str(excinfo.value)
+
+
+async def test_get_assigned_task_timeout():
+ r"""Test timeout handling in _get_assigned_task method"""
+ worker = TestTimeoutWorker()
+
+ # Mock TaskChannel to simulate timeout
+ mock_channel = AsyncMock(spec=TaskChannel)
+ mock_channel.get_assigned_task_by_assignee.side_effect = (
+ asyncio.TimeoutError("Simulated timeout")
+ )
+ worker._channel = mock_channel
+
+ with pytest.raises(asyncio.TimeoutError):
+ await worker._get_assigned_task()
+
+ mock_channel.get_assigned_task_by_assignee.assert_called_once()
+
+
+async def test_listen_to_channel_recovers_from_timeout():
+ r"""Test that _listen_to_channel method recovers from timeouts"""
+ worker = TestTimeoutWorker()
+
+ # Mock TaskChannel
+ mock_channel = AsyncMock(spec=TaskChannel)
+
+ # Configure get_assigned_task_by_assignee to first throw TimeoutError,
+ # then return normal result
+ mock_task = MagicMock(spec=Task)
+ mock_task.id = "test_task_id"
+ mock_task.content = "test content"
+
+ call_count = 0
+
+ async def side_effect(*args, **kwargs):
+ nonlocal call_count
+ call_count += 1
+ if call_count == 1:
+ raise asyncio.TimeoutError("Simulated timeout")
+ return mock_task
+
+ mock_channel.get_assigned_task_by_assignee.side_effect = side_effect
+ mock_channel.get_dependency_ids.return_value = []
+ mock_channel.return_task = AsyncMock()
+
+ worker._channel = mock_channel
+
+ async def patched_listen():
+ worker._running = True
+ try:
+ # Only process one task - retry if timeout occurs
+ attempts = 0
+ max_attempts = 3
+ while attempts < max_attempts:
+ try:
+ await worker._get_assigned_task()
+ break # Success, exit the loop
+ except asyncio.TimeoutError:
+ # This is expected on first attempt, retry
+ attempts += 1
+ if attempts >= max_attempts:
+ raise # Re-raise if max attempts reached
+ finally:
+ worker._running = False
+
+ async def direct_start():
+ return await patched_listen()
+
+ with (
+ patch.object(worker, '_listen_to_channel', patched_listen),
+ patch.object(worker, 'start', direct_start),
+ ):
+ # Should not raise exception
+ await worker.start()
+
+ # Verify first call failed and then retried
+ assert mock_channel.get_assigned_task_by_assignee.call_count == 2
+
+
+async def test_worker_with_timeout_integration():
+ r"""Test the integration of with_timeout across Worker methods"""
+ worker = TestTimeoutWorker()
+
+ # Mock TaskChannel
+ mock_channel = AsyncMock(spec=TaskChannel)
+ mock_task = MagicMock(spec=Task)
+ mock_task.id = "test_task_id"
+ mock_task.content = "test content"
+ mock_task.set_state = MagicMock()
+
+ mock_channel.get_assigned_task_by_assignee.return_value = mock_task
+ mock_channel.get_dependency_ids.return_value = []
+ mock_channel.return_task = AsyncMock()
+
+ worker._channel = mock_channel
+
+ ran_once = False
+
+ async def patched_listen():
+ nonlocal ran_once
+ worker._running = True
+
+ # Add call here to ensure it's always executed regardless of execution
+ # path
+ await worker._channel.get_dependency_ids()
+
+ if not ran_once:
+ ran_once = True
+ # Get task
+ task = await worker._get_assigned_task()
+
+ # Keep this call for backward compatibility
+ # _dependency_ids = await worker._channel.get_dependency_ids()
+
+ task_dependencies = []
+ # Process task
+ task_state = await with_timeout_async(
+ worker._process_task(task, task_dependencies),
+ context=f"processing task {task.id}",
+ )
+ # Update status
+ task.set_state(task_state)
+ # Return task
+ await with_timeout_async(
+ worker._channel.return_task(task.id),
+ timeout=TIMEOUT_THRESHOLD,
+ context=f"returning task {task.id}",
+ )
+ worker._running = False
+
+ with patch.object(worker, '_listen_to_channel', patched_listen):
+ await worker.start()
+
+ # Verify all methods were called correctly
+ mock_channel.get_assigned_task_by_assignee.assert_called_once()
+ mock_channel.get_dependency_ids.assert_called_once()
+ mock_task.set_state.assert_called_once_with(TaskState.DONE)
+ mock_channel.return_task.assert_called_once_with(mock_task.id)
+
+
+async def test_multiple_timeout_points():
+ r"""Test handling timeouts at different points in the worker process"""
+ worker = TestTimeoutWorker()
+
+ # Mock TaskChannel that times out during dependency retrieval
+ mock_channel = AsyncMock(spec=TaskChannel)
+ mock_task = MagicMock(spec=Task)
+ mock_task.id = "test_task_id"
+
+ mock_channel.get_assigned_task_by_assignee.return_value = mock_task
+ mock_channel.get_dependency_ids.side_effect = asyncio.TimeoutError(
+ "Dependency timeout"
+ )
+
+ worker._channel = mock_channel
+
+ # Mock _process_task to verify it's not called when dependencies time out
+ process_task_mock = AsyncMock()
+ worker._process_task = process_task_mock
+
+ # Patch _listen_to_channel to handle one task
+ async def patched_listen():
+ worker._running = True
+ try:
+ # Get task
+ task = await worker._get_assigned_task()
+
+ # Get dependencies - this will time out
+ try:
+ dependency_ids = await with_timeout_async(
+ worker._channel.get_dependency_ids(),
+ context=f"getting dependency IDs for task {task.id}",
+ )
+ task_dependencies = []
+ for dep_id in dependency_ids:
+ dep_task = await with_timeout_async(
+ worker._channel.get_task_by_id(dep_id),
+ context=f"getting dependency task {dep_id}",
+ )
+ task_dependencies.append(dep_task)
+
+ # Process task - should not reach here due to timeout
+ task_state = await worker._process_task(
+ task, task_dependencies
+ )
+ task.set_state(task_state)
+ await worker._channel.return_task(task.id)
+ except asyncio.TimeoutError:
+ # This is expected, continue to next task
+ pass
+ finally:
+ worker._running = False
+
+ with patch.object(worker, '_listen_to_channel', patched_listen):
+ await worker.start()
+
+ # Verify get_dependency_ids was called but _process_task was not
+ mock_channel.get_assigned_task_by_assignee.assert_called_once()
+ mock_channel.get_dependency_ids.assert_called_once()
+ process_task_mock.assert_not_called()
diff --git a/test/workforce/test_workforce.py b/test/workforce/test_workforce.py
new file mode 100644
index 000000000..88de3b26c
--- /dev/null
+++ b/test/workforce/test_workforce.py
@@ -0,0 +1,274 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+
+import asyncio
+from unittest.mock import AsyncMock, MagicMock, patch
+
+import pytest
+
+from camel.societies.workforce.task_channel import TaskChannel
+from camel.societies.workforce.workforce import Workforce
+from camel.tasks.task import Task, TaskState
+from camel.utils import with_timeout_async
+
+
+class TestTimeoutWorkforce(Workforce):
+ r"""A test workforce class for testing timeout handling"""
+
+ def __init__(self):
+ super().__init__("Test Workforce")
+ # Skip the actual initialization to make testing easier
+ self._running = False
+ self._channel = None
+ self._pending_tasks = []
+ self._task = None
+ self._child_listening_tasks = []
+ self._children = []
+
+
+async def test_with_timeout_function():
+ r"""Test the with_timeout function handles completions and
+ timeouts correctly"""
+ # Test normal operation (successful completion)
+ mock_coro = AsyncMock()
+ mock_coro.return_value = "success"
+ result = await with_timeout_async(mock_coro(), context="test operation")
+ assert result == "success"
+
+ # Test timeout handling
+ mock_timeout_coro = AsyncMock()
+ mock_timeout_coro.side_effect = asyncio.TimeoutError("Simulated timeout")
+
+ with pytest.raises(asyncio.TimeoutError) as exc_info:
+ await with_timeout_async(
+ mock_timeout_coro(), context="test timeout operation"
+ )
+
+ assert "Timed out while test timeout operation" in str(exc_info.value)
+
+
+async def test_post_task_timeout():
+ r"""Test timeout handling in _post_task method"""
+ workforce = TestTimeoutWorkforce()
+
+ # Mock TaskChannel that times out
+ mock_channel = AsyncMock(spec=TaskChannel)
+ mock_channel.post_task.side_effect = asyncio.TimeoutError()
+
+ workforce._channel = mock_channel
+ mock_task = MagicMock(spec=Task)
+ mock_task.id = "test_task_id"
+
+ # Only verify that a TimeoutError is raised, not its specific message
+ with pytest.raises(asyncio.TimeoutError):
+ await workforce._post_task(mock_task, "test_worker_id")
+
+ # Verify the channel method was called
+ mock_channel.post_task.assert_called_once()
+
+
+async def test_listen_to_channel_recovers_from_timeout():
+ r"""Test that _listen_to_channel method recovers from timeouts when
+ getting returned tasks"""
+ workforce = TestTimeoutWorkforce()
+
+ # Mock TaskChannel with first call timing out, second call succeeding
+ mock_channel = AsyncMock(spec=TaskChannel)
+ mock_task = MagicMock(spec=Task)
+ mock_task.id = "test_task_id"
+ mock_task.state = TaskState.DONE
+
+ # Configure get_returned_task_by_publisher to first throw TimeoutError,
+ # then return normal result
+ call_count = 0
+
+ async def side_effect(*args, **kwargs):
+ nonlocal call_count
+ call_count += 1
+ if call_count == 1:
+ raise asyncio.TimeoutError("Simulated timeout")
+ return mock_task
+
+ mock_channel.get_returned_task_by_publisher.side_effect = side_effect
+ workforce._channel = mock_channel
+
+ # Mock _handle_completed_task to avoid full implementation
+ workforce._handle_completed_task = AsyncMock()
+
+ # Mock _post_ready_tasks to avoid implementation
+ workforce._post_ready_tasks = AsyncMock()
+
+ # Patch _listen_to_channel to run only limited iterations
+ async def patched_listen():
+ workforce._running = True
+ workforce._task = mock_task
+ workforce._pending_tasks = [mock_task]
+
+ try:
+ # Post initial ready tasks
+ await workforce._post_ready_tasks()
+
+ # First try to get returned task - should timeout
+ try:
+ returned_task = await workforce._get_returned_task()
+ raise asyncio.TimeoutError("Simulated timeout")
+ except asyncio.TimeoutError:
+ # This is expected
+ pass
+
+ # Second try should succeed
+ returned_task = await workforce._get_returned_task()
+ # Process the task
+ await workforce._handle_completed_task(returned_task)
+
+ finally:
+ workforce._running = False
+
+ async def direct_start():
+ return await patched_listen()
+
+ with (
+ patch.object(workforce, '_listen_to_channel', patched_listen),
+ patch.object(workforce, 'start', direct_start),
+ ):
+ await workforce.start()
+
+ # Verify get_returned_task_by_publisher was called twice (first fails,
+ # second succeeds)
+ assert mock_channel.get_returned_task_by_publisher.call_count == 2
+ # Verify the task was processed
+ workforce._handle_completed_task.assert_called_once_with(mock_task)
+
+
+async def test_workforce_with_timeout_integration():
+ r"""Test the integration of with_timeout across Workforce methods"""
+ workforce = TestTimeoutWorkforce()
+
+ # Mock TaskChannel
+ mock_channel = AsyncMock(spec=TaskChannel)
+ mock_task = MagicMock(spec=Task)
+ mock_task.id = "test_task_id"
+ mock_task.state = TaskState.DONE
+
+ mock_channel.get_returned_task_by_publisher.return_value = mock_task
+ workforce._channel = mock_channel
+
+ # Mock _handle_completed_task to avoid full implementation
+ workforce._handle_completed_task = AsyncMock()
+
+ # Mock _post_ready_tasks to avoid implementation
+ workforce._post_ready_tasks = AsyncMock()
+
+ # Patch _listen_to_channel to run only once
+ ran_once = False
+
+ async def patched_listen():
+ nonlocal ran_once
+ workforce._running = True
+ workforce._task = mock_task
+ workforce._pending_tasks = [mock_task]
+
+ if not ran_once:
+ ran_once = True
+ # Post initial ready tasks
+ await with_timeout_async(
+ workforce._post_ready_tasks(),
+ context="posting ready tasks at start",
+ )
+
+ # Get returned task
+ returned_task = await with_timeout_async(
+ workforce._get_returned_task(), context="getting returned task"
+ )
+
+ # Handle completed task
+ await with_timeout_async(
+ workforce._handle_completed_task(returned_task),
+ context="handling completed task",
+ )
+
+ workforce._running = False
+
+ with patch.object(workforce, '_listen_to_channel', patched_listen):
+ await workforce.start()
+
+ # Verify methods were called
+ mock_channel.get_returned_task_by_publisher.assert_called_once()
+ workforce._post_ready_tasks.assert_called_once()
+ workforce._handle_completed_task.assert_called_once_with(mock_task)
+
+
+async def test_multiple_timeout_points():
+ r"""Test handling timeouts at different points in the workforce process"""
+ workforce = TestTimeoutWorkforce()
+
+ # Mock TaskChannel that times out during get_returned_task
+ mock_channel = AsyncMock(spec=TaskChannel)
+ mock_task = MagicMock(spec=Task)
+ mock_task.id = "test_task_id"
+ mock_task.state = TaskState.FAILED
+
+ mock_task.failure_count = 0
+
+ mock_channel.get_returned_task_by_publisher.return_value = mock_task
+
+ # Make _handle_failed_task timeout
+ mock_channel.remove_task.side_effect = asyncio.TimeoutError()
+
+ workforce._channel = mock_channel
+ workforce._task = mock_task
+ workforce._pending_tasks = [mock_task]
+
+ # Patch _listen_to_channel to handle one task with the timeout
+ async def patched_listen():
+ workforce._running = True
+ try:
+ # Post initial ready tasks
+ await workforce._post_ready_tasks()
+
+ # Get task - should succeed
+ returned_task = await workforce._get_returned_task()
+
+ # Handle failed task - should timeout during remove_task
+ try:
+ await workforce._handle_failed_task(returned_task)
+ raise AssertionError("Should have timed out on first attempt")
+ except asyncio.TimeoutError:
+ # This is expected
+ pass
+
+ finally:
+ workforce._running = False
+
+ # Mock methods to avoid full implementation
+ workforce._post_ready_tasks = AsyncMock()
+
+ async def direct_start():
+ return await patched_listen()
+
+ with (
+ patch.object(workforce, '_listen_to_channel', patched_listen),
+ patch.object(workforce, 'start', direct_start),
+ ):
+ await workforce.start()
+
+ # Verify get_returned_task_by_publisher was called
+ mock_channel.get_returned_task_by_publisher.assert_called_once()
+ # Verify remove_task was called (this is where the timeout occurred)
+ mock_channel.remove_task.assert_called_once_with(mock_task.id)