🐫 CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org
Python agentai-societiesartificial-intelligencecommunicative-aicooperative-aideep-learninglarge-language-modelsmulti-agent-systemsnatural-language-processing
enhance: Workforce Graceful Shutdown (#2552)
Hesam Sheikh6b57645ce7b8
diff --git a/camel/societies/workforce/workforce.py b/camel/societies/workforce/workforce.pyindex 19ab698b2..3c222a253 100644--- a/camel/societies/workforce/workforce.py+++ b/camel/societies/workforce/workforce.py@@ -84,6 +84,10 @@ class Workforce(BaseNode):available parameters.(default: :obj:`None` - creates workers with SearchToolkit,CodeExecutionToolkit, and ThinkingToolkit)+ graceful_shutdown_timeout (float, optional): The timeout in seconds+ for graceful shutdown when a task fails 3 times. During this+ period, the workforce remains active for debugging.+ Set to 0 for immediate shutdown. (default: :obj:`15.0`)Example:>>> # Configure with custom model@@ -108,11 +112,13 @@ class Workforce(BaseNode):coordinator_agent_kwargs: Optional[Dict] = None,task_agent_kwargs: Optional[Dict] = None,new_worker_agent_kwargs: Optional[Dict] = None,+ graceful_shutdown_timeout: float = 15.0,) -> None:super().__init__(description)self._child_listening_tasks: Deque[asyncio.Task] = deque()self._children = children or []self.new_worker_agent_kwargs = new_worker_agent_kwargs+ self.graceful_shutdown_timeout = graceful_shutdown_timeout# Warning messages for default model usageif coordinator_agent_kwargs is None:@@ -488,6 +494,26 @@ class Workforce(BaseNode):await self._channel.archive_task(task.id)await self._post_ready_tasks()+ async def _graceful_shutdown(self, failed_task: Task) -> None:+ r"""Handle graceful shutdown with configurable timeout. This is used to+ keep the workforce running for a while to debug the failed task.++ Args:+ failed_task (Task): The task that failed and triggered shutdown.+ """+ if self.graceful_shutdown_timeout <= 0:+ # Immediate shutdown if timeout is 0 or negative+ return++ logger.warning(+ f"Workforce will shutdown in {self.graceful_shutdown_timeout} "+ f"seconds due to failure. You can use this time to inspect the "+ f"current state of the workforce."+ )++ # Wait for the full timeout period+ await asyncio.sleep(self.graceful_shutdown_timeout)+@check_if_running(False)async def _listen_to_channel(self) -> None:r"""Continuously listen to the channel, post task to the channel and@@ -511,6 +537,8 @@ class Workforce(BaseNode):f"{Fore.RED}Task {returned_task.id} has failed "f"for 3 times, halting the workforce.{Fore.RESET}")+ # Graceful shutdown instead of immediate break+ await self._graceful_shutdown(returned_task)breakelif returned_task.state == TaskState.OPEN:# TODO: multi-layer workforce@@ -562,6 +590,7 @@ class Workforce(BaseNode):coordinator_agent_kwargs={},task_agent_kwargs={},new_worker_agent_kwargs=self.new_worker_agent_kwargs,+ graceful_shutdown_timeout=self.graceful_shutdown_timeout,)new_instance.task_agent = self.task_agent.clone(with_memory)diff --git a/docs/mintlify/reference/camel.societies.workforce.workforce.mdx b/docs/mintlify/reference/camel.societies.workforce.workforce.mdxindex 189f4582d..493a0ab62 100644--- a/docs/mintlify/reference/camel.societies.workforce.workforce.mdx+++ b/docs/mintlify/reference/camel.societies.workforce.workforce.mdx@@ -26,6 +26,7 @@ capabilities- **coordinator_agent_kwargs** (Optional[Dict], optional): Keyword arguments passed directly to the coordinator :obj:`ChatAgent` constructor. The coordinator manages task assignment and failure handling strategies. See :obj:`ChatAgent` documentation for all available parameters. (default: :obj:`None` - uses ModelPlatformType.DEFAULT, ModelType.DEFAULT)- **task_agent_kwargs** (Optional[Dict], optional): Keyword arguments passed directly to the task planning :obj:`ChatAgent` constructor. The task agent handles task decomposition into subtasks and result composition. See :obj:`ChatAgent` documentation for all available parameters. (default: :obj:`None` - uses ModelPlatformType.DEFAULT, ModelType.DEFAULT)- **new_worker_agent_kwargs** (Optional[Dict], optional): Default keyword arguments passed to :obj:`ChatAgent` constructor for workers created dynamically at runtime when existing workers cannot handle failed tasks. See :obj:`ChatAgent` documentation for all available parameters. (default: :obj:`None` - creates workers with SearchToolkit, CodeExecutionToolkit, and ThinkingToolkit)+- **graceful_shutdown_timeout** (float, optional): The timeout in seconds for graceful shutdown when a task fails 3 times. During this period, the workforce remains active for debugging. Set to 0 for immediate shutdown. (default: :obj:`15.0`)<a id="camel.societies.workforce.workforce.Workforce.__init__"></a>@@ -38,7 +39,8 @@ def __init__(children: Optional[List[BaseNode]] = None,coordinator_agent_kwargs: Optional[Dict] = None,task_agent_kwargs: Optional[Dict] = None,- new_worker_agent_kwargs: Optional[Dict] = None+ new_worker_agent_kwargs: Optional[Dict] = None,+ graceful_shutdown_timeout: float = 15.0):```diff --git a/examples/workforce/multiple_single_agents.py b/examples/workforce/multiple_single_agents.pyindex 16794743e..0e8519958 100644--- a/examples/workforce/multiple_single_agents.py+++ b/examples/workforce/multiple_single_agents.py@@ -74,7 +74,10 @@ def main():),)- workforce = Workforce('A travel group')+ workforce = Workforce(+ 'A travel group',+ graceful_shutdown_timeout=30.0, # 30 seconds for debugging+ )workforce.add_single_agent_worker("A tour guide",diff --git a/test/workforce/test_workforce_single_agent.py b/test/workforce/test_workforce_single_agent.pyindex 1af0ad48c..54766df8b 100644--- a/test/workforce/test_workforce_single_agent.py+++ b/test/workforce/test_workforce_single_agent.py@@ -11,16 +11,99 @@# See the License for the specific language governing permissions and# limitations under the License.# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+import time+from typing import List+import pytestfrom camel.agents.chat_agent import ChatAgentfrom camel.messages.base import BaseMessage+from camel.societies.workforce import Workforcefrom camel.societies.workforce.single_agent_worker import SingleAgentWorker-from camel.tasks.task import Task+from camel.tasks.task import Task, TaskState+++class AlwaysFailingWorker(SingleAgentWorker):+ """A worker that always fails tasks for testing purposes."""++ def __init__(self, description: str):+ # Create a dummy agent - won't be used since we override _process_task+ sys_msg = BaseMessage.make_assistant_message(+ role_name="Failing Worker", content="This worker always fails."+ )+ agent = ChatAgent(sys_msg)+ super().__init__(description, agent)++ async def _process_task(+ self, task: Task, dependencies: List[Task]+ ) -> TaskState:+ """Always return failed task."""+ task.state = TaskState.FAILED+ task.result = "Task failed deliberately for testing"+ return TaskState.FAILED+++async def test_graceful_shutdown_immediate_timeout():+ """Test that 0 timeout causes immediate shutdown."""+ # Create workforce with 0 timeout (immediate shutdown)+ workforce = Workforce("Test Workforce", graceful_shutdown_timeout=0.0)++ # Create a task to test with+ task = Task(content="This will fail", id="test_task")++ # Test the graceful shutdown method directly+ start_time = time.time()++ # Call graceful shutdown with 0 timeout (should return immediately)+ await workforce._graceful_shutdown(task)++ end_time = time.time()+ execution_time = end_time - start_time++ # Should complete immediately (well under 0.1 seconds)+ assert (+ execution_time < 0.1+ ), f"Expected immediate shutdown, but took {execution_time:.2f} seconds"+ print(+ f"✓ Immediate shutdown test passed - execution time: "+ f"{execution_time:.3f}s"+ )+++async def test_graceful_shutdown_one_second_timeout():+ """Test that 1 second timeout waits approximately 1 second+ before shutdown."""+ # Create workforce with 1 second timeout+ workforce = Workforce("Test Workforce", graceful_shutdown_timeout=1.0)++ # Create a task to test with+ task = Task(content="This will fail", id="test_task")++ # Test the graceful shutdown method directly+ start_time = time.time()++ # Call graceful shutdown with 1 second timeout+ await workforce._graceful_shutdown(task)++ end_time = time.time()+ execution_time = end_time - start_time++ # Should take approximately 1 second+ # (allow 0.1s margin for processing overhead)+ assert (+ 0.9 <= execution_time <= 1.2+ ), f"Expected ~1 second shutdown, but took {execution_time:.2f} seconds"+ print(+ f"✓ 1-second timeout test passed - execution time: "+ f"{execution_time:.2f}s"+ )@pytest.mark.asyncioasync def test_get_dep_tasks_info():+ """Original test for backwards compatibility."""sys_msg = BaseMessage.make_assistant_message(role_name="programmer",content="You are a python programmer.",