Skip to content

armgit / camel

Public Imported from GitHub
0

🐫 CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org

feat: add callback support for workforce events and metrics logging (#3257)

Lei Zhang150c6f51fff4
diff --git a/camel/societies/workforce/events.py b/camel/societies/workforce/events.py
new file mode 100644
index 000000000..280073931
--- /dev/null
+++ b/camel/societies/workforce/events.py
@@ -0,0 +1,122 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+from __future__ import annotations
+
+from datetime import datetime, timezone
+from typing import Any, Dict, List, Literal, Optional, Union
+
+from pydantic import BaseModel, ConfigDict, Field
+
+
+class WorkforceEventBase(BaseModel):
+ model_config = ConfigDict(frozen=True, extra='forbid')
+ event_type: Literal[
+ "task_decomposed",
+ "task_created",
+ "task_assigned",
+ "task_started",
+ "task_completed",
+ "task_failed",
+ "worker_created",
+ "worker_deleted",
+ "queue_status",
+ "all_tasks_completed",
+ ]
+ metadata: Optional[Dict[str, Any]] = None
+ timestamp: datetime = Field(
+ default_factory=lambda: datetime.now(timezone.utc)
+ )
+
+
+class WorkerCreatedEvent(WorkforceEventBase):
+ event_type: Literal["worker_created"] = "worker_created"
+ worker_id: str
+ worker_type: str
+ role: str
+
+
+class WorkerDeletedEvent(WorkforceEventBase):
+ event_type: Literal["worker_deleted"] = "worker_deleted"
+ worker_id: str
+ reason: Optional[str] = None
+
+
+class TaskDecomposedEvent(WorkforceEventBase):
+ event_type: Literal["task_decomposed"] = "task_decomposed"
+ parent_task_id: str
+ subtask_ids: List[str]
+
+
+class TaskCreatedEvent(WorkforceEventBase):
+ event_type: Literal["task_created"] = "task_created"
+ task_id: str
+ description: str
+ parent_task_id: Optional[str] = None
+ task_type: Optional[str] = None
+
+
+class TaskAssignedEvent(WorkforceEventBase):
+ event_type: Literal["task_assigned"] = "task_assigned"
+ task_id: str
+ worker_id: str
+ queue_time_seconds: Optional[float] = None
+ dependencies: Optional[List[str]] = None
+
+
+class TaskStartedEvent(WorkforceEventBase):
+ event_type: Literal["task_started"] = "task_started"
+ task_id: str
+ worker_id: str
+
+
+class TaskCompletedEvent(WorkforceEventBase):
+ event_type: Literal["task_completed"] = "task_completed"
+ task_id: str
+ worker_id: str
+ result_summary: Optional[str] = None
+ processing_time_seconds: Optional[float] = None
+ token_usage: Optional[Dict[str, int]] = None
+
+
+class TaskFailedEvent(WorkforceEventBase):
+ event_type: Literal["task_failed"] = "task_failed"
+ task_id: str
+ error_message: str
+ worker_id: Optional[str] = None
+
+
+class AllTasksCompletedEvent(WorkforceEventBase):
+ event_type: Literal["all_tasks_completed"] = "all_tasks_completed"
+
+
+class QueueStatusEvent(WorkforceEventBase):
+ event_type: Literal["queue_status"] = "queue_status"
+ queue_name: str
+ length: int
+ pending_task_ids: Optional[List[str]] = None
+ metadata: Optional[Dict[str, Any]] = None
+
+
+WorkforceEvent = Union[
+ TaskDecomposedEvent,
+ TaskCreatedEvent,
+ TaskAssignedEvent,
+ TaskStartedEvent,
+ TaskCompletedEvent,
+ TaskFailedEvent,
+ WorkerCreatedEvent,
+ WorkerDeletedEvent,
+ AllTasksCompletedEvent,
+ QueueStatusEvent,
+]
diff --git a/camel/societies/workforce/workforce.py b/camel/societies/workforce/workforce.py
index e5c468643..e9787b7d8 100644
--- a/camel/societies/workforce/workforce.py
+++ b/camel/societies/workforce/workforce.py
@@ -37,6 +37,9 @@ from typing import (
cast,
)
+from .workforce_callback import WorkforceCallback
+from .workforce_metrics import WorkforceMetrics
+
if TYPE_CHECKING:
from camel.utils.context_utils import ContextUtility
@@ -89,6 +92,16 @@ from camel.toolkits import (
from camel.types import ModelPlatformType, ModelType
from camel.utils import dependencies_required
+from .events import (
+ AllTasksCompletedEvent,
+ TaskAssignedEvent,
+ TaskCompletedEvent,
+ TaskCreatedEvent,
+ TaskDecomposedEvent,
+ TaskFailedEvent,
+ TaskStartedEvent,
+ WorkerCreatedEvent,
+)
from .workforce_logger import WorkforceLogger
if os.environ.get("TRACEROOT_ENABLED", "False").lower() == "true":
@@ -205,6 +218,17 @@ class Workforce(BaseNode):
support native structured output. When disabled, the workforce
uses the native response_format parameter.
(default: :obj:`True`)
+ callbacks (Optional[List[WorkforceCallback]], optional): A list of
+ callback handlers to observe and record workforce lifecycle events
+ and metrics (e.g., task creation/assignment/start/completion/
+ failure, worker creation/deletion, all-tasks-completed). All
+ items must be instances of :class:`WorkforceCallback`, otherwise
+ a :class:`ValueError` is raised. If none of the provided
+ callbacks implement :class:`WorkforceMetrics`, a built-in
+ :class:`WorkforceLogger` (implements both callback and metrics)
+ is added automatically. If at least one provided callback
+ implements :class:`WorkforceMetrics`, no default logger is added.
+ (default: :obj:`None`)
Example:
>>> import asyncio
@@ -257,6 +281,7 @@ class Workforce(BaseNode):
share_memory: bool = False,
use_structured_output_handler: bool = True,
task_timeout_seconds: Optional[float] = None,
+ callbacks: Optional[List[WorkforceCallback]] = None,
) -> None:
super().__init__(description)
self._child_listening_tasks: Deque[
@@ -272,7 +297,6 @@ class Workforce(BaseNode):
)
if self.use_structured_output_handler:
self.structured_handler = StructuredOutputHandler()
- self.metrics_logger = WorkforceLogger(workforce_id=self.node_id)
self._task: Optional[Task] = None
self._pending_tasks: Deque[Task] = deque()
self._task_dependencies: Dict[str, List[str]] = {}
@@ -297,15 +321,7 @@ class Workforce(BaseNode):
self.snapshot_interval: float = 30.0
# Shared memory UUID tracking to prevent re-sharing duplicates
self._shared_memory_uuids: Set[str] = set()
- if self.metrics_logger:
- for child in self._children:
- worker_type = type(child).__name__
- role_or_desc = child.description
- self.metrics_logger.log_worker_created(
- worker_id=child.node_id,
- worker_type=worker_type,
- role=role_or_desc,
- )
+ self._initialize_callbacks(callbacks)
# Set up coordinator agent with default system message
coord_agent_sys_msg = BaseMessage.make_assistant_message(
@@ -465,6 +481,54 @@ class Workforce(BaseNode):
# Helper for propagating pause control to externally supplied agents
# ------------------------------------------------------------------
+ def _initialize_callbacks(
+ self, callbacks: Optional[List[WorkforceCallback]]
+ ) -> None:
+ r"""Validate, register, and prime workforce callbacks."""
+ self._callbacks: List[WorkforceCallback] = []
+
+ if callbacks:
+ for cb in callbacks:
+ if isinstance(cb, WorkforceCallback):
+ self._callbacks.append(cb)
+ else:
+ raise ValueError(
+ "All callbacks must be instances of WorkforceCallback"
+ )
+
+ has_metrics_callback = any(
+ isinstance(cb, WorkforceMetrics) for cb in self._callbacks
+ )
+
+ if not has_metrics_callback:
+ self._callbacks.append(WorkforceLogger(workforce_id=self.node_id))
+ else:
+ logger.info(
+ "WorkforceMetrics implementation detected. Skipping default "
+ "WorkforceLogger addition."
+ )
+
+ for child in self._children:
+ self._notify_worker_created(child)
+
+ def _notify_worker_created(
+ self,
+ worker_node: BaseNode,
+ *,
+ worker_type: Optional[str] = None,
+ role: Optional[str] = None,
+ metadata: Optional[Dict[str, Any]] = None,
+ ) -> None:
+ r"""Emit a worker-created event to all registered callbacks."""
+ event = WorkerCreatedEvent(
+ worker_id=worker_node.node_id,
+ worker_type=worker_type or type(worker_node).__name__,
+ role=role or worker_node.description,
+ metadata=metadata,
+ )
+ for cb in self._callbacks:
+ cb.log_worker_created(event)
+
def _get_or_create_shared_context_utility(
self,
session_id: Optional[str] = None,
@@ -1115,19 +1179,23 @@ class Workforce(BaseNode):
else:
subtasks = subtasks_result
- if self.metrics_logger and subtasks:
- self.metrics_logger.log_task_decomposed(
+ if subtasks:
+ task_decomposed_event = TaskDecomposedEvent(
parent_task_id=task.id,
subtask_ids=[st.id for st in subtasks],
)
+ for cb in self._callbacks:
+ cb.log_task_decomposed(task_decomposed_event)
for subtask in subtasks:
- self.metrics_logger.log_task_created(
+ task_created_event = TaskCreatedEvent(
task_id=subtask.id,
description=subtask.content,
parent_task_id=task.id,
task_type=subtask.type,
metadata=subtask.additional_info,
)
+ for cb in self._callbacks:
+ cb.log_task_created(task_created_event)
# Insert subtasks at the head of the queue
self._pending_tasks.extendleft(reversed(subtasks))
@@ -1640,13 +1708,15 @@ class Workforce(BaseNode):
self._task = task
task.state = TaskState.FAILED
- if self.metrics_logger:
- self.metrics_logger.log_task_created(
- task_id=task.id,
- description=task.content,
- task_type=task.type,
- metadata=task.additional_info,
- )
+ task_created_event = TaskCreatedEvent(
+ task_id=task.id,
+ description=task.content,
+ task_type=task.type,
+ metadata=task.additional_info,
+ )
+ for cb in self._callbacks:
+ cb.log_task_created(task_created_event)
+
# The agent tend to be overconfident on the whole task, so we
# decompose the task into subtasks first
subtasks_result = self._decompose_task(task)
@@ -1660,18 +1730,23 @@ class Workforce(BaseNode):
else:
# This is a regular list (non-streaming mode)
subtasks = subtasks_result
- if self.metrics_logger and subtasks:
- self.metrics_logger.log_task_decomposed(
- parent_task_id=task.id, subtask_ids=[st.id for st in subtasks]
+ if subtasks:
+ task_decomposed_event = TaskDecomposedEvent(
+ parent_task_id=task.id,
+ subtask_ids=[st.id for st in subtasks],
)
+ for cb in self._callbacks:
+ cb.log_task_decomposed(task_decomposed_event)
for subtask in subtasks:
- self.metrics_logger.log_task_created(
+ task_created_event = TaskCreatedEvent(
task_id=subtask.id,
description=subtask.content,
parent_task_id=task.id,
task_type=subtask.type,
metadata=subtask.additional_info,
)
+ for cb in self._callbacks:
+ cb.log_task_created(task_created_event)
if subtasks:
# _pending_tasks will contain both undecomposed
@@ -1990,12 +2065,10 @@ class Workforce(BaseNode):
# If workforce is paused, start the worker's listening task
self._start_child_node_when_paused(worker_node.start())
- if self.metrics_logger:
- self.metrics_logger.log_worker_created(
- worker_id=worker_node.node_id,
- worker_type='SingleAgentWorker',
- role=worker_node.description,
- )
+ self._notify_worker_created(
+ worker_node,
+ worker_type='SingleAgentWorker',
+ )
return self
def add_role_playing_worker(
@@ -2069,12 +2142,10 @@ class Workforce(BaseNode):
# If workforce is paused, start the worker's listening task
self._start_child_node_when_paused(worker_node.start())
- if self.metrics_logger:
- self.metrics_logger.log_worker_created(
- worker_id=worker_node.node_id,
- worker_type='RolePlayingWorker',
- role=worker_node.description,
- )
+ self._notify_worker_created(
+ worker_node,
+ worker_type='RolePlayingWorker',
+ )
return self
def add_workforce(self, workforce: Workforce) -> Workforce:
@@ -2151,10 +2222,9 @@ class Workforce(BaseNode):
# No active loop, directly set the event
self._pause_event.set()
- if hasattr(self, 'metrics_logger') and self.metrics_logger is not None:
- self.metrics_logger.reset_task_data()
- else:
- self.metrics_logger = WorkforceLogger(workforce_id=self.node_id)
+ for cb in self._callbacks:
+ if isinstance(cb, WorkforceMetrics):
+ cb.reset_task_data()
def save_workflow_memories(
self,
@@ -2983,10 +3053,11 @@ class Workforce(BaseNode):
task.assigned_worker_id = assignee_id
- if self.metrics_logger:
- self.metrics_logger.log_task_started(
- task_id=task.id, worker_id=assignee_id
- )
+ task_started_event = TaskStartedEvent(
+ task_id=task.id, worker_id=assignee_id
+ )
+ for cb in self._callbacks:
+ cb.log_task_started(task_started_event)
try:
await self._channel.post_task(task, self.node_id, assignee_id)
@@ -3130,13 +3201,13 @@ class Workforce(BaseNode):
print(f"{Fore.CYAN}{new_node} created.{Fore.RESET}")
self._children.append(new_node)
- if self.metrics_logger:
- self.metrics_logger.log_worker_created(
- worker_id=new_node.node_id,
- worker_type='SingleAgentWorker',
- role=new_node_conf.role,
- metadata={'description': new_node_conf.description},
- )
+
+ self._notify_worker_created(
+ new_node,
+ worker_type='SingleAgentWorker',
+ role=new_node_conf.role,
+ metadata={'description': new_node_conf.description},
+ )
self._child_listening_tasks.append(
asyncio.create_task(new_node.start())
)
@@ -3237,22 +3308,24 @@ class Workforce(BaseNode):
batch_result = await self._find_assignee(tasks_to_assign)
logger.debug(
f"Coordinator returned assignments:\n"
- f"{json.dumps(batch_result.dict(), indent=2)}"
+ f"{json.dumps(batch_result.model_dump(), indent=2)}"
)
for assignment in batch_result.assignments:
self._task_dependencies[assignment.task_id] = (
assignment.dependencies
)
self._assignees[assignment.task_id] = assignment.assignee_id
- if self.metrics_logger:
+
+ task_assigned_event = TaskAssignedEvent(
+ task_id=assignment.task_id,
+ worker_id=assignment.assignee_id,
+ dependencies=assignment.dependencies,
+ queue_time_seconds=None,
+ )
+ for cb in self._callbacks:
# queue_time_seconds can be derived by logger if task
# creation time is logged
- self.metrics_logger.log_task_assigned(
- task_id=assignment.task_id,
- worker_id=assignment.assignee_id,
- dependencies=assignment.dependencies,
- queue_time_seconds=None,
- )
+ cb.log_task_assigned(task_assigned_event)
# Step 2: Iterate through all pending tasks and post those that are
# ready
@@ -3369,21 +3442,19 @@ class Workforce(BaseNode):
)
# Log the failure to metrics
- if self.metrics_logger:
- self.metrics_logger.log_task_failed(
- task_id=task.id,
- worker_id=task.assigned_worker_id
- or "unknown",
- error_message=task.result,
- metadata={
- 'failure_reason': (
- 'dependency_failure'
- ),
- 'failed_dependencies': (
- permanently_failed_deps
- ),
- },
- )
+ task_failed_event = TaskFailedEvent(
+ task_id=task.id,
+ worker_id=task.assigned_worker_id or "unknown",
+ error_message=task.result,
+ metadata={
+ 'failure_reason': 'dependency_failure',
+ 'failed_dependencies': (
+ permanently_failed_deps
+ ),
+ },
+ )
+ for cb in self._callbacks:
+ cb.log_task_failed(task_failed_event)
self._completed_tasks.append(task)
self._cleanup_task_tracking(task.id)
@@ -3435,17 +3506,18 @@ class Workforce(BaseNode):
f"{failure_reason}{Fore.RESET}"
)
- if self.metrics_logger:
- self.metrics_logger.log_task_failed(
- task_id=task.id,
- worker_id=worker_id,
- error_message=detailed_error,
- metadata={
- 'failure_count': task.failure_count,
- 'task_content': task.content,
- 'result_length': len(task.result) if task.result else 0,
- },
- )
+ task_failed_event = TaskFailedEvent(
+ task_id=task.id,
+ worker_id=worker_id,
+ error_message=detailed_error,
+ metadata={
+ 'failure_count': task.failure_count,
+ 'task_content': task.content,
+ 'result_length': len(task.result) if task.result else 0,
+ },
+ )
+ for cb in self._callbacks:
+ cb.log_task_failed(task_failed_event)
# Check for immediate halt conditions
if task.failure_count >= MAX_TASK_RETRIES:
@@ -3536,61 +3608,60 @@ class Workforce(BaseNode):
return False
async def _handle_completed_task(self, task: Task) -> None:
- if self.metrics_logger:
- worker_id = task.assigned_worker_id or "unknown"
- processing_time_seconds = None
- token_usage = None
-
- # Get processing time from task start time or additional info
- if task.id in self._task_start_times:
- processing_time_seconds = (
- time.time() - self._task_start_times[task.id]
- )
- self._cleanup_task_tracking(task.id)
- elif (
- task.additional_info is not None
- and 'processing_time_seconds' in task.additional_info
- ):
- processing_time_seconds = task.additional_info[
- 'processing_time_seconds'
- ]
+ worker_id = task.assigned_worker_id or "unknown"
+ processing_time_seconds = None
+ token_usage = None
- # Get token usage from task additional info (preferred - actual
- # usage)
- if (
- task.additional_info is not None
- and 'token_usage' in task.additional_info
- ):
- token_usage = task.additional_info['token_usage']
- else:
- # Fallback: Try to get token usage from SingleAgentWorker
- # memory
- assignee_node = next(
- (
- child
- for child in self._children
- if child.node_id == worker_id
- ),
- None,
- )
- if isinstance(assignee_node, SingleAgentWorker):
- try:
- _, total_tokens = (
- assignee_node.worker.memory.get_context()
- )
- token_usage = {'total_tokens': total_tokens}
- except Exception:
- token_usage = None
+ # Get processing time from task start time or additional info
+ if task.id in self._task_start_times:
+ processing_time_seconds = (
+ time.time() - self._task_start_times[task.id]
+ )
+ self._cleanup_task_tracking(task.id)
+ elif (
+ task.additional_info is not None
+ and 'processing_time_seconds' in task.additional_info
+ ):
+ processing_time_seconds = task.additional_info[
+ 'processing_time_seconds'
+ ]
- # Log the completed task
- self.metrics_logger.log_task_completed(
- task_id=task.id,
- worker_id=worker_id,
- result_summary=task.result if task.result else "Completed",
- processing_time_seconds=processing_time_seconds,
- token_usage=token_usage,
- metadata={'current_state': task.state.value},
+ # Get token usage from task additional info (preferred - actual
+ # usage)
+ if (
+ task.additional_info is not None
+ and 'token_usage' in task.additional_info
+ ):
+ token_usage = task.additional_info['token_usage']
+ else:
+ # Fallback: Try to get token usage from SingleAgentWorker
+ # memory
+ assignee_node = next(
+ (
+ child
+ for child in self._children
+ if child.node_id == worker_id
+ ),
+ None,
)
+ if isinstance(assignee_node, SingleAgentWorker):
+ try:
+ _, total_tokens = assignee_node.worker.memory.get_context()
+ token_usage = {'total_tokens': total_tokens}
+ except Exception:
+ token_usage = None
+
+ # Log the completed task
+ task_completed_event = TaskCompletedEvent(
+ task_id=task.id,
+ worker_id=worker_id,
+ result_summary=task.result if task.result else "Completed",
+ processing_time_seconds=processing_time_seconds,
+ token_usage=token_usage,
+ metadata={'current_state': task.state.value},
+ )
+ for cb in self._callbacks:
+ cb.log_task_completed(task_completed_event)
# Find and remove the completed task from pending tasks
tasks_list = list(self._pending_tasks)
@@ -3710,15 +3781,23 @@ class Workforce(BaseNode):
r"""Returns an ASCII tree representation of the task hierarchy and
worker status.
"""
- if not self.metrics_logger:
- return "Logger not initialized."
- return self.metrics_logger.get_ascii_tree_representation()
+ metrics_cb: List[WorkforceMetrics] = [
+ cb for cb in self._callbacks if isinstance(cb, WorkforceMetrics)
+ ]
+ if len(metrics_cb) == 0:
+ return "Metrics Callback not initialized."
+ else:
+ return metrics_cb[0].get_ascii_tree_representation()
def get_workforce_kpis(self) -> Dict[str, Any]:
r"""Returns a dictionary of key performance indicators."""
- if not self.metrics_logger:
- return {"error": "Logger not initialized."}
- return self.metrics_logger.get_kpis()
+ metrics_cb: List[WorkforceMetrics] = [
+ cb for cb in self._callbacks if isinstance(cb, WorkforceMetrics)
+ ]
+ if len(metrics_cb) == 0:
+ return {"error": "Metrics Callback not initialized."}
+ else:
+ return metrics_cb[0].get_kpis()
def dump_workforce_logs(self, file_path: str) -> None:
r"""Dumps all collected logs to a JSON file.
@@ -3726,10 +3805,13 @@ class Workforce(BaseNode):
Args:
file_path (str): The path to the JSON file.
"""
- if not self.metrics_logger:
+ metrics_cb: List[WorkforceMetrics] = [
+ cb for cb in self._callbacks if isinstance(cb, WorkforceMetrics)
+ ]
+ if len(metrics_cb) == 0:
print("Logger not initialized. Cannot dump logs.")
return
- self.metrics_logger.dump_to_json(file_path)
+ metrics_cb[0].dump_to_json(file_path)
# Use logger.info or print, consistent with existing style
logger.info(f"Workforce logs dumped to {file_path}")
@@ -4177,6 +4259,9 @@ class Workforce(BaseNode):
elif not self._pending_tasks and self._in_flight_tasks == 0:
self._state = WorkforceState.IDLE
logger.info("All tasks completed.")
+ all_tasks_completed_event = AllTasksCompletedEvent()
+ for cb in self._callbacks:
+ cb.log_all_tasks_completed(all_tasks_completed_event)
# shut down the whole workforce tree
self.stop()
diff --git a/camel/societies/workforce/workforce_callback.py b/camel/societies/workforce/workforce_callback.py
new file mode 100644
index 000000000..9dee1d559
--- /dev/null
+++ b/camel/societies/workforce/workforce_callback.py
@@ -0,0 +1,74 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+
+from .events import (
+ AllTasksCompletedEvent,
+ TaskAssignedEvent,
+ TaskCompletedEvent,
+ TaskCreatedEvent,
+ TaskDecomposedEvent,
+ TaskFailedEvent,
+ TaskStartedEvent,
+ WorkerCreatedEvent,
+ WorkerDeletedEvent,
+)
+
+
+class WorkforceCallback(ABC):
+ r"""Interface for recording workforce lifecycle events.
+
+ Implementations should persist or stream events as appropriate.
+ """
+
+ @abstractmethod
+ def log_task_created(
+ self,
+ event: TaskCreatedEvent,
+ ) -> None:
+ pass
+
+ @abstractmethod
+ def log_task_decomposed(self, event: TaskDecomposedEvent) -> None:
+ pass
+
+ @abstractmethod
+ def log_task_assigned(self, event: TaskAssignedEvent) -> None:
+ pass
+
+ @abstractmethod
+ def log_task_started(self, event: TaskStartedEvent) -> None:
+ pass
+
+ @abstractmethod
+ def log_task_completed(self, event: TaskCompletedEvent) -> None:
+ pass
+
+ @abstractmethod
+ def log_task_failed(self, event: TaskFailedEvent) -> None:
+ pass
+
+ @abstractmethod
+ def log_worker_created(self, event: WorkerCreatedEvent) -> None:
+ pass
+
+ @abstractmethod
+ def log_worker_deleted(self, event: WorkerDeletedEvent) -> None:
+ pass
+
+ @abstractmethod
+ def log_all_tasks_completed(self, event: AllTasksCompletedEvent) -> None:
+ pass
diff --git a/camel/societies/workforce/workforce_logger.py b/camel/societies/workforce/workforce_logger.py
index 049d53249..b04c13428 100644
--- a/camel/societies/workforce/workforce_logger.py
+++ b/camel/societies/workforce/workforce_logger.py
@@ -16,12 +16,26 @@ from datetime import datetime, timezone
from typing import Any, Dict, List, Optional
from camel.logger import get_logger
+from camel.societies.workforce.events import (
+ AllTasksCompletedEvent,
+ QueueStatusEvent,
+ TaskAssignedEvent,
+ TaskCompletedEvent,
+ TaskCreatedEvent,
+ TaskDecomposedEvent,
+ TaskFailedEvent,
+ TaskStartedEvent,
+ WorkerCreatedEvent,
+ WorkerDeletedEvent,
+)
+from camel.societies.workforce.workforce_callback import WorkforceCallback
+from camel.societies.workforce.workforce_metrics import WorkforceMetrics
from camel.types.agents import ToolCallingRecord
logger = get_logger(__name__)
-class WorkforceLogger:
+class WorkforceLogger(WorkforceCallback, WorkforceMetrics):
r"""Logs events and metrics for a Workforce instance."""
def __init__(self, workforce_id: str):
@@ -55,195 +69,201 @@ class WorkforceLogger:
def log_task_created(
self,
- task_id: str,
- description: str,
- parent_task_id: Optional[str] = None,
- task_type: Optional[str] = None,
- metadata: Optional[Dict[str, Any]] = None,
+ event: TaskCreatedEvent,
) -> None:
r"""Logs the creation of a new task."""
self._log_event(
- 'task_created',
- task_id=task_id,
- description=description,
- parent_task_id=parent_task_id,
- task_type=task_type,
- metadata=metadata or {},
+ event_type=event.event_type,
+ task_id=event.task_id,
+ description=event.description,
+ parent_task_id=event.parent_task_id,
+ task_type=event.task_type,
+ metadata=event.metadata or {},
)
- self._task_hierarchy[task_id] = {
- 'parent': parent_task_id,
+ self._task_hierarchy[event.task_id] = {
+ 'parent': event.parent_task_id,
'children': [],
'status': 'created',
- 'description': description,
+ 'description': event.description,
'assigned_to': None,
- **(metadata or {}),
+ **(event.metadata or {}),
}
- if parent_task_id and parent_task_id in self._task_hierarchy:
- self._task_hierarchy[parent_task_id]['children'].append(task_id)
+ if (
+ event.parent_task_id
+ and event.parent_task_id in self._task_hierarchy
+ ):
+ self._task_hierarchy[event.parent_task_id]['children'].append(
+ event.task_id
+ )
def log_task_decomposed(
self,
- parent_task_id: str,
- subtask_ids: List[str],
- metadata: Optional[Dict[str, Any]] = None,
+ event: TaskDecomposedEvent,
) -> None:
r"""Logs the decomposition of a task into subtasks."""
self._log_event(
- 'task_decomposed',
- parent_task_id=parent_task_id,
- subtask_ids=subtask_ids,
- metadata=metadata or {},
+ event_type=event.event_type,
+ parent_task_id=event.parent_task_id,
+ subtask_ids=event.subtask_ids,
+ metadata=event.metadata or {},
)
- if parent_task_id in self._task_hierarchy:
- self._task_hierarchy[parent_task_id]['status'] = "decomposed"
+ if event.parent_task_id in self._task_hierarchy:
+ self._task_hierarchy[event.parent_task_id]['status'] = "decomposed"
def log_task_assigned(
self,
- task_id: str,
- worker_id: str,
- queue_time_seconds: Optional[float] = None,
- dependencies: Optional[List[str]] = None,
- metadata: Optional[Dict[str, Any]] = None,
+ event: TaskAssignedEvent,
) -> None:
r"""Logs the assignment of a task to a worker."""
self._log_event(
- 'task_assigned',
- task_id=task_id,
- worker_id=worker_id,
- queue_time_seconds=queue_time_seconds,
- dependencies=dependencies or [],
- metadata=metadata or {},
+ event_type=event.event_type,
+ task_id=event.task_id,
+ worker_id=event.worker_id,
+ queue_time_seconds=event.queue_time_seconds,
+ dependencies=event.dependencies or [],
+ metadata=event.metadata or {},
)
- if task_id in self._task_hierarchy:
- self._task_hierarchy[task_id]['status'] = 'assigned'
- self._task_hierarchy[task_id]['assigned_to'] = worker_id
- self._task_hierarchy[task_id]['dependencies'] = dependencies or []
- if worker_id in self._worker_information:
- self._worker_information[worker_id]['current_task_id'] = task_id
- self._worker_information[worker_id]['status'] = 'busy'
+ if event.task_id in self._task_hierarchy:
+ self._task_hierarchy[event.task_id]['status'] = 'assigned'
+ self._task_hierarchy[event.task_id]['assigned_to'] = (
+ event.worker_id
+ )
+ self._task_hierarchy[event.task_id]['dependencies'] = (
+ event.dependencies or []
+ )
+ if event.worker_id in self._worker_information:
+ self._worker_information[event.worker_id]['current_task_id'] = (
+ event.task_id
+ )
+ self._worker_information[event.worker_id]['status'] = 'busy'
def log_task_started(
self,
- task_id: str,
- worker_id: str,
- metadata: Optional[Dict[str, Any]] = None,
+ event: TaskStartedEvent,
) -> None:
r"""Logs when a worker starts processing a task."""
self._log_event(
- 'task_started',
- task_id=task_id,
- worker_id=worker_id,
- metadata=metadata or {},
+ event_type=event.event_type,
+ task_id=event.task_id,
+ worker_id=event.worker_id,
+ metadata=event.metadata or {},
)
- if task_id in self._task_hierarchy:
- self._task_hierarchy[task_id]['status'] = 'processing'
+ if event.task_id in self._task_hierarchy:
+ self._task_hierarchy[event.task_id]['status'] = 'processing'
- def log_task_completed(
- self,
- task_id: str,
- worker_id: str,
- result_summary: Optional[str] = None,
- processing_time_seconds: Optional[float] = None,
- token_usage: Optional[Dict[str, int]] = None,
- metadata: Optional[Dict[str, Any]] = None,
- ) -> None:
+ def log_task_completed(self, event: TaskCompletedEvent) -> None:
r"""Logs the successful completion of a task."""
self._log_event(
- 'task_completed',
- task_id=task_id,
- worker_id=worker_id,
- result_summary=result_summary,
- processing_time_seconds=processing_time_seconds,
- token_usage=token_usage or {},
- metadata=metadata or {},
+ event_type=event.event_type,
+ task_id=event.task_id,
+ worker_id=event.worker_id,
+ result_summary=event.result_summary,
+ processing_time_seconds=event.processing_time_seconds,
+ token_usage=event.token_usage or {},
+ metadata=event.metadata or {},
)
- if task_id in self._task_hierarchy:
- self._task_hierarchy[task_id]['status'] = 'completed'
- self._task_hierarchy[task_id]['assigned_to'] = None
+ if event.task_id in self._task_hierarchy:
+ self._task_hierarchy[event.task_id]['status'] = 'completed'
+ self._task_hierarchy[event.task_id]['assigned_to'] = None
# Store processing time in task hierarchy for display in tree
- if processing_time_seconds is not None:
- self._task_hierarchy[task_id]['completion_time_seconds'] = (
- processing_time_seconds
- )
+ if event.processing_time_seconds is not None:
+ self._task_hierarchy[event.task_id][
+ 'completion_time_seconds'
+ ] = event.processing_time_seconds
# Store token usage in task hierarchy for display in tree
- if token_usage is not None:
- self._task_hierarchy[task_id]['token_usage'] = token_usage
- if worker_id in self._worker_information:
- self._worker_information[worker_id]['current_task_id'] = None
- self._worker_information[worker_id]['status'] = 'idle'
- self._worker_information[worker_id]['tasks_completed'] = (
- self._worker_information[worker_id].get('tasks_completed', 0)
+ if event.token_usage is not None:
+ self._task_hierarchy[event.task_id]['token_usage'] = (
+ event.token_usage
+ )
+ if event.worker_id in self._worker_information:
+ self._worker_information[event.worker_id]['current_task_id'] = None
+ self._worker_information[event.worker_id]['status'] = 'idle'
+ self._worker_information[event.worker_id]['tasks_completed'] = (
+ self._worker_information[event.worker_id].get(
+ 'tasks_completed', 0
+ )
+ 1
)
def log_task_failed(
self,
- task_id: str,
- error_message: str,
- worker_id: Optional[str] = None,
- metadata: Optional[Dict[str, Any]] = None,
+ event: TaskFailedEvent,
) -> None:
r"""Logs the failure of a task."""
self._log_event(
- 'task_failed',
- task_id=task_id,
- worker_id=worker_id,
- error_message=error_message,
- metadata=metadata or {},
+ event_type=event.event_type,
+ task_id=event.task_id,
+ worker_id=event.worker_id,
+ error_message=event.error_message,
+ metadata=event.metadata or {},
)
- if task_id in self._task_hierarchy:
- self._task_hierarchy[task_id]['status'] = 'failed'
- self._task_hierarchy[task_id]['error'] = error_message
- self._task_hierarchy[task_id]['assigned_to'] = None
- if worker_id and worker_id in self._worker_information:
- self._worker_information[worker_id]['current_task_id'] = None
- self._worker_information[worker_id]['status'] = 'idle'
- self._worker_information[worker_id]['tasks_failed'] = (
- self._worker_information[worker_id].get('tasks_failed', 0) + 1
+ if event.task_id in self._task_hierarchy:
+ self._task_hierarchy[event.task_id]['status'] = 'failed'
+ self._task_hierarchy[event.task_id]['error'] = event.error_message
+ self._task_hierarchy[event.task_id]['assigned_to'] = None
+ if event.worker_id and event.worker_id in self._worker_information:
+ self._worker_information[event.worker_id]['current_task_id'] = None
+ self._worker_information[event.worker_id]['status'] = 'idle'
+ self._worker_information[event.worker_id]['tasks_failed'] = (
+ self._worker_information[event.worker_id].get(
+ 'tasks_failed', 0
+ )
+ + 1
)
def log_worker_created(
self,
- worker_id: str,
- worker_type: str,
- role: str,
- metadata: Optional[Dict[str, Any]] = None,
+ event: WorkerCreatedEvent,
) -> None:
r"""Logs the creation of a new worker."""
self._log_event(
- 'worker_created',
- worker_id=worker_id,
- worker_type=worker_type,
- role=role,
- metadata=metadata or {},
+ event_type=event.event_type,
+ worker_id=event.worker_id,
+ worker_type=event.worker_type,
+ role=event.role,
+ metadata=event.metadata or {},
)
- self._worker_information[worker_id] = {
- 'type': worker_type,
- 'role': role,
+ self._worker_information[event.worker_id] = {
+ 'type': event.worker_type,
+ 'role': event.role,
'status': 'idle',
'current_task_id': None,
'tasks_completed': 0,
'tasks_failed': 0,
- **(metadata or {}),
+ **(event.metadata or {}),
}
def log_worker_deleted(
self,
- worker_id: str,
- reason: Optional[str] = None,
- metadata: Optional[Dict[str, Any]] = None,
+ event: WorkerDeletedEvent,
) -> None:
r"""Logs the deletion of a worker."""
self._log_event(
- 'worker_deleted',
- worker_id=worker_id,
- reason=reason,
- metadata=metadata or {},
+ event_type=event.event_type,
+ worker_id=event.worker_id,
+ reason=event.reason,
+ metadata=event.metadata or {},
)
- if worker_id in self._worker_information:
- self._worker_information[worker_id]['status'] = 'deleted'
+ if event.worker_id in self._worker_information:
+ self._worker_information[event.worker_id]['status'] = 'deleted'
# Or del self._worker_information[worker_id]
+ def log_queue_status(
+ self,
+ event: QueueStatusEvent,
+ ) -> None:
+ r"""Logs the status of a task queue."""
+ self._log_event(
+ event_type=event.event_type,
+ queue_name=event.queue_name,
+ length=event.length,
+ pending_task_ids=event.pending_task_ids or [],
+ metadata=event.metadata or {},
+ )
+
+ def log_all_tasks_completed(self, event: AllTasksCompletedEvent) -> None:
+ pass
+
def reset_task_data(self) -> None:
r"""Resets logs and data related to tasks, preserving worker
information.
@@ -263,22 +283,6 @@ class WorkforceLogger:
f"{self.workforce_id}"
)
- def log_queue_status(
- self,
- queue_name: str,
- length: int,
- pending_task_ids: Optional[List[str]] = None,
- metadata: Optional[Dict[str, Any]] = None,
- ) -> None:
- r"""Logs the status of a task queue."""
- self._log_event(
- 'queue_status',
- queue_name=queue_name,
- length=length,
- pending_task_ids=pending_task_ids or [],
- metadata=metadata or {},
- )
-
def dump_to_json(self, file_path: str) -> None:
r"""Dumps all log entries to a JSON file.
diff --git a/camel/societies/workforce/workforce_metrics.py b/camel/societies/workforce/workforce_metrics.py
new file mode 100644
index 000000000..06a7735c6
--- /dev/null
+++ b/camel/societies/workforce/workforce_metrics.py
@@ -0,0 +1,33 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+from abc import ABC, abstractmethod
+from typing import Any, Dict
+
+
+class WorkforceMetrics(ABC):
+ @abstractmethod
+ def reset_task_data(self) -> None:
+ pass
+
+ @abstractmethod
+ def dump_to_json(self, file_path: str) -> None:
+ pass
+
+ @abstractmethod
+ def get_ascii_tree_representation(self) -> str:
+ pass
+
+ @abstractmethod
+ def get_kpis(self) -> Dict[str, Any]:
+ pass
diff --git a/docs/key_modules/workforce.md b/docs/key_modules/workforce.md
index d2192203a..2e33e5a04 100644
--- a/docs/key_modules/workforce.md
+++ b/docs/key_modules/workforce.md
@@ -22,8 +22,10 @@ class Workforce(BaseNode):
task_agent: Optional[ChatAgent] = None,
new_worker_agent: Optional[ChatAgent] = None,
graceful_shutdown_timeout: float = 15.0,
+ task_timeout_seconds: Optional[float] = None,
share_memory: bool = False,
use_structured_output_handler: bool = True,
+ callbacks: Optional[List[WorkforceCallback]] = None,
) -> None:
# ...
```
@@ -35,8 +37,10 @@ class Workforce(BaseNode):
- `coordinator_agent`: A `ChatAgent` for assigning tasks.
- `task_agent`: A `ChatAgent` for decomposing tasks.
- `new_worker_agent`: A template `ChatAgent` for creating new workers.
+- `task_timeout_seconds`: Optional per-workforce task timeout in seconds.
- `share_memory`: If `True`, `SingleAgentWorker` instances will share memory.
- `use_structured_output_handler`: Defaults to `True`. Enables structured output handling so models without native JSON + tool-calling can still interop reliably.
+- `callbacks`: Optional A list of callback handlers to observe and record workforce lifecycle events and metrics.
</Card>
<Card title="Worker Types" icon="users-gear">
@@ -247,4 +251,4 @@ Understanding these models is key to interpreting the workforce's internal state
>
Full documentation for advanced usage and configuration.
</Card>
-</CardGroup>
\ No newline at end of file
+</CardGroup>
diff --git a/examples/workforce/workforce_callbacks_example.py b/examples/workforce/workforce_callbacks_example.py
new file mode 100644
index 000000000..01fdeda2a
--- /dev/null
+++ b/examples/workforce/workforce_callbacks_example.py
@@ -0,0 +1,148 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+"""
+Example: Subscribe to callbacks in Workforce
+
+Includes:
+- Custom callback `PrintCallback` that logs key lifecycle events.
+- Subscribe callbacks via the `callbacks` parameter when constructing a
+ `Workforce`.
+"""
+
+import asyncio
+
+from camel.agents import ChatAgent
+from camel.logger import get_logger
+from camel.models import ModelFactory
+from camel.societies.workforce.events import (
+ AllTasksCompletedEvent,
+ TaskAssignedEvent,
+ TaskCompletedEvent,
+ TaskCreatedEvent,
+ TaskDecomposedEvent,
+ TaskFailedEvent,
+ TaskStartedEvent,
+ WorkerCreatedEvent,
+ WorkerDeletedEvent,
+)
+from camel.societies.workforce.workforce import Workforce
+from camel.societies.workforce.workforce_callback import WorkforceCallback
+from camel.types import ModelPlatformType, ModelType
+
+logger = get_logger(__name__)
+
+
+class PrintCallback(WorkforceCallback):
+ r"""Simple callback printing events to logs to observe ordering."""
+
+ def log_task_created(self, event: TaskCreatedEvent) -> None:
+ print(
+ f"[PrintCallback] task_created: id={event.task_id}, "
+ f"desc={event.description!r}, parent={event.parent_task_id}"
+ )
+
+ def log_task_decomposed(self, event: TaskDecomposedEvent) -> None:
+ print(
+ f"[PrintCallback] task_decomposed: parent={event.parent_task_id},"
+ f" subtasks={event.subtask_ids}"
+ )
+
+ def log_task_assigned(self, event: TaskAssignedEvent) -> None:
+ print(
+ f"[PrintCallback] task_assigned: task={event.task_id}, "
+ f"worker={event.worker_id}"
+ )
+
+ def log_task_started(self, event: TaskStartedEvent) -> None:
+ print(
+ f"[PrintCallback] task_started: task={event.task_id}, "
+ f"worker={event.worker_id}"
+ )
+
+ def log_task_completed(self, event: TaskCompletedEvent) -> None:
+ print(
+ f"[PrintCallback] task_completed: task={event.task_id}, "
+ f"worker={event.worker_id}, took={event.processing_time_seconds}s"
+ )
+
+ def log_task_failed(self, event: TaskFailedEvent) -> None:
+ logger.warning(
+ f"[PrintCallback] task_failed: task={event.task_id}, "
+ f"err={event.error_message}"
+ )
+
+ def log_worker_created(self, event: WorkerCreatedEvent) -> None:
+ print(
+ f"[PrintCallback] worker_created: id={event.worker_id}, "
+ f"type={event.worker_type}, role={event.role}"
+ )
+
+ def log_worker_deleted(self, event: WorkerDeletedEvent) -> None:
+ print(
+ f"[PrintCallback] worker_deleted: id={event.worker_id}, "
+ f"reason={event.reason}"
+ )
+
+ def log_all_tasks_completed(self, event: AllTasksCompletedEvent) -> None:
+ print("[PrintCallback] all_tasks_completed")
+
+
+def build_teacher_agent() -> ChatAgent:
+ model = ModelFactory.create(
+ model_platform=ModelPlatformType.OPENAI,
+ model_type=ModelType.GPT_4O_MINI,
+ )
+ return ChatAgent(system_message="You are a teacher", model=model)
+
+
+def build_student_agent() -> ChatAgent:
+ model = ModelFactory.create(
+ model_platform=ModelPlatformType.OPENAI,
+ model_type=ModelType.GPT_4O_MINI,
+ )
+ return ChatAgent(system_message="You are a student", model=model)
+
+
+async def run_demo() -> None:
+ callbacks = [PrintCallback()]
+
+ workforce = Workforce(
+ "Workforce Callbacks Demo",
+ callbacks=callbacks,
+ use_structured_output_handler=True,
+ )
+
+ teacher = build_teacher_agent()
+ student = build_student_agent()
+ workforce.add_single_agent_worker("Teacher Worker", teacher)
+ workforce.add_single_agent_worker("Student Worker", student)
+ workforce.add_main_task(
+ "The teacher set an exam question and had the students answer it."
+ )
+
+ # Start Workforce and wait for completion (timeout to avoid hanging)
+ wf_task = asyncio.create_task(workforce.start())
+ try:
+ await asyncio.wait_for(wf_task, timeout=30.0)
+ except asyncio.TimeoutError:
+ logger.warning("Workforce run timed out; stopping...")
+ workforce.stop()
+
+ # Read KPIs and a simple "tree"
+ print(f"KPIs: {workforce.get_workforce_kpis()}")
+ print(f"Tree: {workforce.get_workforce_log_tree()}")
+
+
+if __name__ == "__main__":
+ asyncio.run(run_demo())
diff --git a/test/workforce/test_workforce_callbacks.py b/test/workforce/test_workforce_callbacks.py
new file mode 100644
index 000000000..58b439880
--- /dev/null
+++ b/test/workforce/test_workforce_callbacks.py
@@ -0,0 +1,366 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+from typing import Any, Dict
+
+import pytest
+
+from camel.agents import ChatAgent
+from camel.messages import BaseMessage
+from camel.models import ModelFactory
+from camel.societies.workforce.events import (
+ AllTasksCompletedEvent,
+ TaskAssignedEvent,
+ TaskCompletedEvent,
+ TaskCreatedEvent,
+ TaskDecomposedEvent,
+ TaskFailedEvent,
+ TaskStartedEvent,
+ WorkerCreatedEvent,
+ WorkerDeletedEvent,
+ WorkforceEvent,
+)
+from camel.societies.workforce.workforce import Workforce
+from camel.societies.workforce.workforce_callback import WorkforceCallback
+from camel.societies.workforce.workforce_logger import WorkforceLogger
+from camel.societies.workforce.workforce_metrics import WorkforceMetrics
+from camel.tasks import Task
+from camel.types import ModelPlatformType, ModelType
+
+
+class _NonMetricsCallback(WorkforceCallback):
+ """A minimal callback implementation without metrics, for testing."""
+
+ def __init__(self) -> None:
+ self.events: list[WorkforceEvent] = []
+
+ # Task events
+ def log_task_created(self, event: TaskCreatedEvent) -> None:
+ self.events.append(event)
+
+ def log_task_decomposed(self, event: TaskDecomposedEvent) -> None:
+ self.events.append(event)
+
+ def log_task_assigned(self, event: TaskAssignedEvent) -> None:
+ self.events.append(event)
+
+ def log_task_started(self, event: TaskStartedEvent) -> None:
+ self.events.append(event)
+
+ def log_task_completed(self, event: TaskCompletedEvent) -> None:
+ self.events.append(event)
+
+ def log_task_failed(self, event: TaskFailedEvent) -> None:
+ self.events.append(event)
+
+ # Worker events
+ def log_worker_created(self, event: WorkerCreatedEvent) -> None:
+ self.events.append(event)
+
+ def log_worker_deleted(self, event: WorkerDeletedEvent) -> None:
+ self.events.append(event)
+
+ # Terminal event
+ def log_all_tasks_completed(self, event: AllTasksCompletedEvent) -> None:
+ self.events.append(event)
+
+
+class _MetricsCallback(WorkforceCallback, WorkforceMetrics):
+ """A minimal metrics-capable callback for testing."""
+
+ def __init__(self) -> None:
+ self.events: list[WorkforceEvent] = []
+ self.reset_task_data()
+ self.dump_to_json_called = False
+ self.get_ascii_tree_called = False
+ self.get_kpis_called = False
+
+ # WorkforceMetrics interface
+ def reset_task_data(self) -> None:
+ self.dump_to_json_called = False
+ self.get_ascii_tree_called = False
+ self.get_kpis_called = False
+
+ def dump_to_json(self, file_path: str) -> None:
+ self.dump_to_json_called = True
+
+ def get_ascii_tree_representation(self) -> str:
+ self.get_ascii_tree_called = True
+ return "Stub ASCII Tree"
+
+ def get_kpis(self) -> Dict[str, Any]:
+ self.get_kpis_called = True
+ return {}
+
+ def log_task_created(self, event: TaskCreatedEvent) -> None:
+ self.events.append(event)
+
+ def log_task_decomposed(self, event: TaskDecomposedEvent) -> None:
+ self.events.append(event)
+
+ def log_task_assigned(self, event: TaskAssignedEvent) -> None:
+ self.events.append(event)
+
+ def log_task_started(self, event: TaskStartedEvent) -> None:
+ self.events.append(event)
+
+ def log_task_completed(self, event: TaskCompletedEvent) -> None:
+ self.events.append(event)
+
+ def log_task_failed(self, event: TaskFailedEvent) -> None:
+ self.events.append(event)
+
+ def log_worker_created(self, event: WorkerCreatedEvent) -> None:
+ self.events.append(event)
+
+ def log_worker_deleted(self, event: WorkerDeletedEvent) -> None:
+ self.events.append(event)
+
+ def log_all_tasks_completed(self, event: AllTasksCompletedEvent) -> None:
+ self.events.append(event)
+
+
+def _build_stub_agent() -> ChatAgent:
+ """Construct a stub-backed ChatAgent for offline tests."""
+ model = ModelFactory.create(
+ model_platform=ModelPlatformType.OPENAI,
+ model_type=ModelType.STUB,
+ )
+ return ChatAgent(model=model)
+
+
+def test_workforce_callback_registration_and_metrics_handling():
+ """Verify default logger addition and metrics-callback skip logic.
+
+ - When no metrics callback is provided, WorkforceLogger is added.
+ - When a metrics callback is provided, no WorkforceLogger is added.
+ - Invalid callback types raise ValueError.
+ - Worker creation events are propagated to provided callbacks.
+ """
+ # 1) No metrics callback -> default WorkforceLogger should be appended
+ cb = _NonMetricsCallback()
+ wf1 = Workforce("CB Test - No Metrics", callbacks=[cb])
+ callback_types = [type(c) for c in wf1._callbacks]
+ assert any(issubclass(t, WorkforceLogger) for t in callback_types)
+ # Ensure our custom callback is present
+ assert any(isinstance(c, _NonMetricsCallback) for c in wf1._callbacks)
+
+ # Add a worker and ensure our callback saw the event
+ agent = _build_stub_agent()
+ wf1.add_single_agent_worker("UnitTest Worker", agent)
+ assert any(isinstance(e, WorkerCreatedEvent) for e in cb.events)
+
+ # 2) Metrics-capable callback present -> no default WorkforceLogger
+ metrics_cb = _MetricsCallback()
+ wf2 = Workforce("CB Test - With Metrics", callbacks=[metrics_cb])
+ assert all(not isinstance(c, WorkforceLogger) for c in wf2._callbacks)
+ # Also confirm that the metrics callback remains
+ assert any(isinstance(c, _MetricsCallback) for c in wf2._callbacks)
+
+ # 3) Invalid callback type -> ValueError
+ with pytest.raises(ValueError, match="instances of WorkforceCallback"):
+ Workforce("CB Test - Invalid", callbacks=[object()])
+
+
+def assert_event_sequence(events: list[str], min_worker_count: int):
+ """
+ Validate that the given event sequence follows the expected logical order.
+ This version is flexible to handle:
+ - Task retries and dynamic worker creation
+ - Cases where tasks are not decomposed (e.g., when using stub models)
+ """
+ idx = 0
+ n = len(events)
+
+ # 1. Expect at least min_worker_count WorkerCreatedEvent events first
+ initial_worker_count = 0
+ while idx < n and events[idx] == "WorkerCreatedEvent":
+ initial_worker_count += 1
+ idx += 1
+ assert initial_worker_count >= min_worker_count, (
+ f"Expected at least {min_worker_count} initial "
+ f"WorkerCreatedEvents, got {initial_worker_count}"
+ )
+
+ # 2. Expect one main TaskCreatedEvent
+ assert idx < n and events[idx] == "TaskCreatedEvent", (
+ f"Event {idx} should be TaskCreatedEvent, got "
+ f"{events[idx] if idx < n else 'END'}"
+ )
+ idx += 1
+
+ # 3. TaskDecomposedEvent may or may not be present
+ # (depends on coordinator behavior)
+ # If the coordinator can't parse stub responses, it may skip
+ # decomposition
+ has_decomposition = idx < n and events[idx] == "TaskDecomposedEvent"
+ if has_decomposition:
+ idx += 1
+
+ # 4. Count all event types in the remaining events
+ all_events = events[idx:]
+ task_assigned_count = all_events.count("TaskAssignedEvent")
+ task_started_count = all_events.count("TaskStartedEvent")
+ task_completed_count = all_events.count("TaskCompletedEvent")
+ all_tasks_completed_count = all_events.count("AllTasksCompletedEvent")
+
+ # 5. Validate basic invariants
+ # At minimum, the main task should be assigned and processed
+ assert (
+ task_assigned_count >= 1
+ ), f"Expected at least 1 TaskAssignedEvent, got {task_assigned_count}"
+ assert (
+ task_started_count >= 1
+ ), f"Expected at least 1 TaskStartedEvent, got {task_started_count}"
+ assert (
+ task_completed_count >= 1
+ ), f"Expected at least 1 TaskCompletedEvent, got {task_completed_count}"
+
+ # 6. Expect exactly one AllTasksCompletedEvent at the end
+ assert all_tasks_completed_count == 1, (
+ f"Expected exactly 1 AllTasksCompletedEvent, got "
+ f"{all_tasks_completed_count}"
+ )
+ assert (
+ events[-1] == "AllTasksCompletedEvent"
+ ), "Last event should be AllTasksCompletedEvent"
+
+ # 7. All events should be of expected types
+ allowed_events = {
+ "WorkerCreatedEvent",
+ "WorkerDeletedEvent",
+ "TaskCreatedEvent",
+ "TaskDecomposedEvent",
+ "TaskAssignedEvent",
+ "TaskStartedEvent",
+ "TaskCompletedEvent",
+ "TaskFailedEvent",
+ "AllTasksCompletedEvent",
+ }
+ for i, e in enumerate(events):
+ assert e in allowed_events, f"Unexpected event type at {i}: {e}"
+
+
+def test_workforce_emits_expected_event_sequence():
+ # Use STUB model to avoid real API calls and ensure fast,
+ # deterministic execution
+ search_agent = ChatAgent(
+ system_message=BaseMessage.make_assistant_message(
+ role_name="Research Specialist",
+ content="You are a research specialist who excels at finding and "
+ "gathering information from the web.",
+ ),
+ model=ModelFactory.create(
+ model_platform=ModelPlatformType.OPENAI,
+ model_type=ModelType.STUB,
+ ),
+ )
+
+ analyst_agent = ChatAgent(
+ system_message=BaseMessage.make_assistant_message(
+ role_name="Business Analyst",
+ content="You are an expert business analyst. Your job is "
+ "to analyze research findings, identify key insights, "
+ "opportunities, and challenges.",
+ ),
+ model=ModelFactory.create(
+ model_platform=ModelPlatformType.OPENAI,
+ model_type=ModelType.STUB,
+ ),
+ )
+
+ writer_agent = ChatAgent(
+ system_message=BaseMessage.make_assistant_message(
+ role_name="Report Writer",
+ content="You are a professional report writer. You take "
+ "analytical insights and synthesize them into a clear, "
+ "concise, and well-structured final report.",
+ ),
+ model=ModelFactory.create(
+ model_platform=ModelPlatformType.OPENAI,
+ model_type=ModelType.STUB,
+ ),
+ )
+
+ cb = _MetricsCallback()
+
+ # Use STUB models for coordinator and task agents to avoid real API calls
+ coordinator_agent = ChatAgent(
+ model=ModelFactory.create(
+ model_platform=ModelPlatformType.OPENAI,
+ model_type=ModelType.STUB,
+ )
+ )
+ task_agent = ChatAgent(
+ model=ModelFactory.create(
+ model_platform=ModelPlatformType.OPENAI,
+ model_type=ModelType.STUB,
+ )
+ )
+
+ workforce = Workforce(
+ 'Business Analysis Team',
+ graceful_shutdown_timeout=30.0,
+ callbacks=[cb],
+ coordinator_agent=coordinator_agent,
+ task_agent=task_agent,
+ )
+
+ workforce.add_single_agent_worker(
+ "A researcher who can search online for information.",
+ worker=search_agent,
+ ).add_single_agent_worker(
+ "An analyst who can process research findings.", worker=analyst_agent
+ ).add_single_agent_worker(
+ "A writer who can create a final report from the analysis.",
+ worker=writer_agent,
+ )
+
+ # Use a simpler task to ensure fast and deterministic execution
+ human_task = Task(
+ content=(
+ "Create a simple report about electric scooters. "
+ "The report should have three sections: "
+ "1. Market overview "
+ "2. Target customers "
+ "3. Summary"
+ ),
+ id='0',
+ )
+
+ workforce.process_task(human_task)
+
+ # test that the event sequence is as expected
+ actual_events = [e.__class__.__name__ for e in cb.events]
+ assert_event_sequence(actual_events, min_worker_count=3)
+
+ # test that metrics callback methods work as expected
+ assert not cb.dump_to_json_called
+ assert not cb.get_ascii_tree_called
+ assert not cb.get_kpis_called
+ workforce.dump_workforce_logs("foo.log")
+ assert cb.dump_to_json_called
+
+ workforce.reset()
+ assert not cb.dump_to_json_called
+ assert not cb.get_ascii_tree_called
+ assert not cb.get_kpis_called
+ workforce.get_workforce_kpis()
+ assert cb.get_kpis_called
+
+ workforce.reset()
+ assert not cb.dump_to_json_called
+ assert not cb.get_ascii_tree_called
+ assert not cb.get_kpis_called
+ workforce.get_workforce_log_tree()
+ assert cb.get_ascii_tree_called