Skip to content

armgit / camel

Public Imported from GitHub
0

🐫 CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org

enhance: Workforce Graceful Shutdown (#2552)

Hesam Sheikh6b57645ce7b8
diff --git a/camel/societies/workforce/workforce.py b/camel/societies/workforce/workforce.py
index 19ab698b2..3c222a253 100644
--- a/camel/societies/workforce/workforce.py
+++ b/camel/societies/workforce/workforce.py
@@ -84,6 +84,10 @@ class Workforce(BaseNode):
available parameters.
(default: :obj:`None` - creates workers with SearchToolkit,
CodeExecutionToolkit, and ThinkingToolkit)
+ graceful_shutdown_timeout (float, optional): The timeout in seconds
+ for graceful shutdown when a task fails 3 times. During this
+ period, the workforce remains active for debugging.
+ Set to 0 for immediate shutdown. (default: :obj:`15.0`)
Example:
>>> # Configure with custom model
@@ -108,11 +112,13 @@ class Workforce(BaseNode):
coordinator_agent_kwargs: Optional[Dict] = None,
task_agent_kwargs: Optional[Dict] = None,
new_worker_agent_kwargs: Optional[Dict] = None,
+ graceful_shutdown_timeout: float = 15.0,
) -> None:
super().__init__(description)
self._child_listening_tasks: Deque[asyncio.Task] = deque()
self._children = children or []
self.new_worker_agent_kwargs = new_worker_agent_kwargs
+ self.graceful_shutdown_timeout = graceful_shutdown_timeout
# Warning messages for default model usage
if coordinator_agent_kwargs is None:
@@ -488,6 +494,26 @@ class Workforce(BaseNode):
await self._channel.archive_task(task.id)
await self._post_ready_tasks()
+ async def _graceful_shutdown(self, failed_task: Task) -> None:
+ r"""Handle graceful shutdown with configurable timeout. This is used to
+ keep the workforce running for a while to debug the failed task.
+
+ Args:
+ failed_task (Task): The task that failed and triggered shutdown.
+ """
+ if self.graceful_shutdown_timeout <= 0:
+ # Immediate shutdown if timeout is 0 or negative
+ return
+
+ logger.warning(
+ f"Workforce will shutdown in {self.graceful_shutdown_timeout} "
+ f"seconds due to failure. You can use this time to inspect the "
+ f"current state of the workforce."
+ )
+
+ # Wait for the full timeout period
+ await asyncio.sleep(self.graceful_shutdown_timeout)
+
@check_if_running(False)
async def _listen_to_channel(self) -> None:
r"""Continuously listen to the channel, post task to the channel and
@@ -511,6 +537,8 @@ class Workforce(BaseNode):
f"{Fore.RED}Task {returned_task.id} has failed "
f"for 3 times, halting the workforce.{Fore.RESET}"
)
+ # Graceful shutdown instead of immediate break
+ await self._graceful_shutdown(returned_task)
break
elif returned_task.state == TaskState.OPEN:
# TODO: multi-layer workforce
@@ -562,6 +590,7 @@ class Workforce(BaseNode):
coordinator_agent_kwargs={},
task_agent_kwargs={},
new_worker_agent_kwargs=self.new_worker_agent_kwargs,
+ graceful_shutdown_timeout=self.graceful_shutdown_timeout,
)
new_instance.task_agent = self.task_agent.clone(with_memory)
diff --git a/docs/mintlify/reference/camel.societies.workforce.workforce.mdx b/docs/mintlify/reference/camel.societies.workforce.workforce.mdx
index 189f4582d..493a0ab62 100644
--- a/docs/mintlify/reference/camel.societies.workforce.workforce.mdx
+++ b/docs/mintlify/reference/camel.societies.workforce.workforce.mdx
@@ -26,6 +26,7 @@ capabilities
- **coordinator_agent_kwargs** (Optional[Dict], optional): Keyword arguments passed directly to the coordinator :obj:`ChatAgent` constructor. The coordinator manages task assignment and failure handling strategies. See :obj:`ChatAgent` documentation for all available parameters. (default: :obj:`None` - uses ModelPlatformType.DEFAULT, ModelType.DEFAULT)
- **task_agent_kwargs** (Optional[Dict], optional): Keyword arguments passed directly to the task planning :obj:`ChatAgent` constructor. The task agent handles task decomposition into subtasks and result composition. See :obj:`ChatAgent` documentation for all available parameters. (default: :obj:`None` - uses ModelPlatformType.DEFAULT, ModelType.DEFAULT)
- **new_worker_agent_kwargs** (Optional[Dict], optional): Default keyword arguments passed to :obj:`ChatAgent` constructor for workers created dynamically at runtime when existing workers cannot handle failed tasks. See :obj:`ChatAgent` documentation for all available parameters. (default: :obj:`None` - creates workers with SearchToolkit, CodeExecutionToolkit, and ThinkingToolkit)
+- **graceful_shutdown_timeout** (float, optional): The timeout in seconds for graceful shutdown when a task fails 3 times. During this period, the workforce remains active for debugging. Set to 0 for immediate shutdown. (default: :obj:`15.0`)
<a id="camel.societies.workforce.workforce.Workforce.__init__"></a>
@@ -38,7 +39,8 @@ def __init__(
children: Optional[List[BaseNode]] = None,
coordinator_agent_kwargs: Optional[Dict] = None,
task_agent_kwargs: Optional[Dict] = None,
- new_worker_agent_kwargs: Optional[Dict] = None
+ new_worker_agent_kwargs: Optional[Dict] = None,
+ graceful_shutdown_timeout: float = 15.0
):
```
diff --git a/examples/workforce/multiple_single_agents.py b/examples/workforce/multiple_single_agents.py
index 16794743e..0e8519958 100644
--- a/examples/workforce/multiple_single_agents.py
+++ b/examples/workforce/multiple_single_agents.py
@@ -74,7 +74,10 @@ def main():
),
)
- workforce = Workforce('A travel group')
+ workforce = Workforce(
+ 'A travel group',
+ graceful_shutdown_timeout=30.0, # 30 seconds for debugging
+ )
workforce.add_single_agent_worker(
"A tour guide",
diff --git a/test/workforce/test_workforce_single_agent.py b/test/workforce/test_workforce_single_agent.py
index 1af0ad48c..54766df8b 100644
--- a/test/workforce/test_workforce_single_agent.py
+++ b/test/workforce/test_workforce_single_agent.py
@@ -11,16 +11,99 @@
# See the License for the specific language governing permissions and
# limitations under the License.
# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+import time
+from typing import List
+
import pytest
from camel.agents.chat_agent import ChatAgent
from camel.messages.base import BaseMessage
+from camel.societies.workforce import Workforce
from camel.societies.workforce.single_agent_worker import SingleAgentWorker
-from camel.tasks.task import Task
+from camel.tasks.task import Task, TaskState
+
+
+class AlwaysFailingWorker(SingleAgentWorker):
+ """A worker that always fails tasks for testing purposes."""
+
+ def __init__(self, description: str):
+ # Create a dummy agent - won't be used since we override _process_task
+ sys_msg = BaseMessage.make_assistant_message(
+ role_name="Failing Worker", content="This worker always fails."
+ )
+ agent = ChatAgent(sys_msg)
+ super().__init__(description, agent)
+
+ async def _process_task(
+ self, task: Task, dependencies: List[Task]
+ ) -> TaskState:
+ """Always return failed task."""
+ task.state = TaskState.FAILED
+ task.result = "Task failed deliberately for testing"
+ return TaskState.FAILED
+
+
+async def test_graceful_shutdown_immediate_timeout():
+ """Test that 0 timeout causes immediate shutdown."""
+ # Create workforce with 0 timeout (immediate shutdown)
+ workforce = Workforce("Test Workforce", graceful_shutdown_timeout=0.0)
+
+ # Create a task to test with
+ task = Task(content="This will fail", id="test_task")
+
+ # Test the graceful shutdown method directly
+ start_time = time.time()
+
+ # Call graceful shutdown with 0 timeout (should return immediately)
+ await workforce._graceful_shutdown(task)
+
+ end_time = time.time()
+ execution_time = end_time - start_time
+
+ # Should complete immediately (well under 0.1 seconds)
+ assert (
+ execution_time < 0.1
+ ), f"Expected immediate shutdown, but took {execution_time:.2f} seconds"
+ print(
+ f"✓ Immediate shutdown test passed - execution time: "
+ f"{execution_time:.3f}s"
+ )
+
+
+async def test_graceful_shutdown_one_second_timeout():
+ """Test that 1 second timeout waits approximately 1 second
+ before shutdown."""
+ # Create workforce with 1 second timeout
+ workforce = Workforce("Test Workforce", graceful_shutdown_timeout=1.0)
+
+ # Create a task to test with
+ task = Task(content="This will fail", id="test_task")
+
+ # Test the graceful shutdown method directly
+ start_time = time.time()
+
+ # Call graceful shutdown with 1 second timeout
+ await workforce._graceful_shutdown(task)
+
+ end_time = time.time()
+ execution_time = end_time - start_time
+
+ # Should take approximately 1 second
+ # (allow 0.1s margin for processing overhead)
+ assert (
+ 0.9 <= execution_time <= 1.2
+ ), f"Expected ~1 second shutdown, but took {execution_time:.2f} seconds"
+ print(
+ f"✓ 1-second timeout test passed - execution time: "
+ f"{execution_time:.2f}s"
+ )
@pytest.mark.asyncio
async def test_get_dep_tasks_info():
+ """Original test for backwards compatibility."""
sys_msg = BaseMessage.make_assistant_message(
role_name="programmer",
content="You are a python programmer.",