🐫 CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org
Python agentai-societiesartificial-intelligencecommunicative-aicooperative-aideep-learninglarge-language-modelsmulti-agent-systemsnatural-language-processing
feat: add callback support for workforce events and metrics logging (#3257)
Lei Zhang150c6f51fff4
diff --git a/camel/societies/workforce/events.py b/camel/societies/workforce/events.pynew file mode 100644index 000000000..280073931--- /dev/null+++ b/camel/societies/workforce/events.py@@ -0,0 +1,122 @@+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+from __future__ import annotations++from datetime import datetime, timezone+from typing import Any, Dict, List, Literal, Optional, Union++from pydantic import BaseModel, ConfigDict, Field+++class WorkforceEventBase(BaseModel):+ model_config = ConfigDict(frozen=True, extra='forbid')+ event_type: Literal[+ "task_decomposed",+ "task_created",+ "task_assigned",+ "task_started",+ "task_completed",+ "task_failed",+ "worker_created",+ "worker_deleted",+ "queue_status",+ "all_tasks_completed",+ ]+ metadata: Optional[Dict[str, Any]] = None+ timestamp: datetime = Field(+ default_factory=lambda: datetime.now(timezone.utc)+ )+++class WorkerCreatedEvent(WorkforceEventBase):+ event_type: Literal["worker_created"] = "worker_created"+ worker_id: str+ worker_type: str+ role: str+++class WorkerDeletedEvent(WorkforceEventBase):+ event_type: Literal["worker_deleted"] = "worker_deleted"+ worker_id: str+ reason: Optional[str] = None+++class TaskDecomposedEvent(WorkforceEventBase):+ event_type: Literal["task_decomposed"] = "task_decomposed"+ parent_task_id: str+ subtask_ids: List[str]+++class TaskCreatedEvent(WorkforceEventBase):+ event_type: Literal["task_created"] = "task_created"+ task_id: str+ description: str+ parent_task_id: Optional[str] = None+ task_type: Optional[str] = None+++class TaskAssignedEvent(WorkforceEventBase):+ event_type: Literal["task_assigned"] = "task_assigned"+ task_id: str+ worker_id: str+ queue_time_seconds: Optional[float] = None+ dependencies: Optional[List[str]] = None+++class TaskStartedEvent(WorkforceEventBase):+ event_type: Literal["task_started"] = "task_started"+ task_id: str+ worker_id: str+++class TaskCompletedEvent(WorkforceEventBase):+ event_type: Literal["task_completed"] = "task_completed"+ task_id: str+ worker_id: str+ result_summary: Optional[str] = None+ processing_time_seconds: Optional[float] = None+ token_usage: Optional[Dict[str, int]] = None+++class TaskFailedEvent(WorkforceEventBase):+ event_type: Literal["task_failed"] = "task_failed"+ task_id: str+ error_message: str+ worker_id: Optional[str] = None+++class AllTasksCompletedEvent(WorkforceEventBase):+ event_type: Literal["all_tasks_completed"] = "all_tasks_completed"+++class QueueStatusEvent(WorkforceEventBase):+ event_type: Literal["queue_status"] = "queue_status"+ queue_name: str+ length: int+ pending_task_ids: Optional[List[str]] = None+ metadata: Optional[Dict[str, Any]] = None+++WorkforceEvent = Union[+ TaskDecomposedEvent,+ TaskCreatedEvent,+ TaskAssignedEvent,+ TaskStartedEvent,+ TaskCompletedEvent,+ TaskFailedEvent,+ WorkerCreatedEvent,+ WorkerDeletedEvent,+ AllTasksCompletedEvent,+ QueueStatusEvent,+]diff --git a/camel/societies/workforce/workforce.py b/camel/societies/workforce/workforce.pyindex e5c468643..e9787b7d8 100644--- a/camel/societies/workforce/workforce.py+++ b/camel/societies/workforce/workforce.py@@ -37,6 +37,9 @@ from typing import (cast,)+from .workforce_callback import WorkforceCallback+from .workforce_metrics import WorkforceMetrics+if TYPE_CHECKING:from camel.utils.context_utils import ContextUtility@@ -89,6 +92,16 @@ from camel.toolkits import (from camel.types import ModelPlatformType, ModelTypefrom camel.utils import dependencies_required+from .events import (+ AllTasksCompletedEvent,+ TaskAssignedEvent,+ TaskCompletedEvent,+ TaskCreatedEvent,+ TaskDecomposedEvent,+ TaskFailedEvent,+ TaskStartedEvent,+ WorkerCreatedEvent,+)from .workforce_logger import WorkforceLoggerif os.environ.get("TRACEROOT_ENABLED", "False").lower() == "true":@@ -205,6 +218,17 @@ class Workforce(BaseNode):support native structured output. When disabled, the workforceuses the native response_format parameter.(default: :obj:`True`)+ callbacks (Optional[List[WorkforceCallback]], optional): A list of+ callback handlers to observe and record workforce lifecycle events+ and metrics (e.g., task creation/assignment/start/completion/+ failure, worker creation/deletion, all-tasks-completed). All+ items must be instances of :class:`WorkforceCallback`, otherwise+ a :class:`ValueError` is raised. If none of the provided+ callbacks implement :class:`WorkforceMetrics`, a built-in+ :class:`WorkforceLogger` (implements both callback and metrics)+ is added automatically. If at least one provided callback+ implements :class:`WorkforceMetrics`, no default logger is added.+ (default: :obj:`None`)Example:>>> import asyncio@@ -257,6 +281,7 @@ class Workforce(BaseNode):share_memory: bool = False,use_structured_output_handler: bool = True,task_timeout_seconds: Optional[float] = None,+ callbacks: Optional[List[WorkforceCallback]] = None,) -> None:super().__init__(description)self._child_listening_tasks: Deque[@@ -272,7 +297,6 @@ class Workforce(BaseNode):)if self.use_structured_output_handler:self.structured_handler = StructuredOutputHandler()- self.metrics_logger = WorkforceLogger(workforce_id=self.node_id)self._task: Optional[Task] = Noneself._pending_tasks: Deque[Task] = deque()self._task_dependencies: Dict[str, List[str]] = {}@@ -297,15 +321,7 @@ class Workforce(BaseNode):self.snapshot_interval: float = 30.0# Shared memory UUID tracking to prevent re-sharing duplicatesself._shared_memory_uuids: Set[str] = set()- if self.metrics_logger:- for child in self._children:- worker_type = type(child).__name__- role_or_desc = child.description- self.metrics_logger.log_worker_created(- worker_id=child.node_id,- worker_type=worker_type,- role=role_or_desc,- )+ self._initialize_callbacks(callbacks)# Set up coordinator agent with default system messagecoord_agent_sys_msg = BaseMessage.make_assistant_message(@@ -465,6 +481,54 @@ class Workforce(BaseNode):# Helper for propagating pause control to externally supplied agents# ------------------------------------------------------------------+ def _initialize_callbacks(+ self, callbacks: Optional[List[WorkforceCallback]]+ ) -> None:+ r"""Validate, register, and prime workforce callbacks."""+ self._callbacks: List[WorkforceCallback] = []++ if callbacks:+ for cb in callbacks:+ if isinstance(cb, WorkforceCallback):+ self._callbacks.append(cb)+ else:+ raise ValueError(+ "All callbacks must be instances of WorkforceCallback"+ )++ has_metrics_callback = any(+ isinstance(cb, WorkforceMetrics) for cb in self._callbacks+ )++ if not has_metrics_callback:+ self._callbacks.append(WorkforceLogger(workforce_id=self.node_id))+ else:+ logger.info(+ "WorkforceMetrics implementation detected. Skipping default "+ "WorkforceLogger addition."+ )++ for child in self._children:+ self._notify_worker_created(child)++ def _notify_worker_created(+ self,+ worker_node: BaseNode,+ *,+ worker_type: Optional[str] = None,+ role: Optional[str] = None,+ metadata: Optional[Dict[str, Any]] = None,+ ) -> None:+ r"""Emit a worker-created event to all registered callbacks."""+ event = WorkerCreatedEvent(+ worker_id=worker_node.node_id,+ worker_type=worker_type or type(worker_node).__name__,+ role=role or worker_node.description,+ metadata=metadata,+ )+ for cb in self._callbacks:+ cb.log_worker_created(event)+def _get_or_create_shared_context_utility(self,session_id: Optional[str] = None,@@ -1115,19 +1179,23 @@ class Workforce(BaseNode):else:subtasks = subtasks_result- if self.metrics_logger and subtasks:- self.metrics_logger.log_task_decomposed(+ if subtasks:+ task_decomposed_event = TaskDecomposedEvent(parent_task_id=task.id,subtask_ids=[st.id for st in subtasks],)+ for cb in self._callbacks:+ cb.log_task_decomposed(task_decomposed_event)for subtask in subtasks:- self.metrics_logger.log_task_created(+ task_created_event = TaskCreatedEvent(task_id=subtask.id,description=subtask.content,parent_task_id=task.id,task_type=subtask.type,metadata=subtask.additional_info,)+ for cb in self._callbacks:+ cb.log_task_created(task_created_event)# Insert subtasks at the head of the queueself._pending_tasks.extendleft(reversed(subtasks))@@ -1640,13 +1708,15 @@ class Workforce(BaseNode):self._task = tasktask.state = TaskState.FAILED- if self.metrics_logger:- self.metrics_logger.log_task_created(- task_id=task.id,- description=task.content,- task_type=task.type,- metadata=task.additional_info,- )+ task_created_event = TaskCreatedEvent(+ task_id=task.id,+ description=task.content,+ task_type=task.type,+ metadata=task.additional_info,+ )+ for cb in self._callbacks:+ cb.log_task_created(task_created_event)+# The agent tend to be overconfident on the whole task, so we# decompose the task into subtasks firstsubtasks_result = self._decompose_task(task)@@ -1660,18 +1730,23 @@ class Workforce(BaseNode):else:# This is a regular list (non-streaming mode)subtasks = subtasks_result- if self.metrics_logger and subtasks:- self.metrics_logger.log_task_decomposed(- parent_task_id=task.id, subtask_ids=[st.id for st in subtasks]+ if subtasks:+ task_decomposed_event = TaskDecomposedEvent(+ parent_task_id=task.id,+ subtask_ids=[st.id for st in subtasks],)+ for cb in self._callbacks:+ cb.log_task_decomposed(task_decomposed_event)for subtask in subtasks:- self.metrics_logger.log_task_created(+ task_created_event = TaskCreatedEvent(task_id=subtask.id,description=subtask.content,parent_task_id=task.id,task_type=subtask.type,metadata=subtask.additional_info,)+ for cb in self._callbacks:+ cb.log_task_created(task_created_event)if subtasks:# _pending_tasks will contain both undecomposed@@ -1990,12 +2065,10 @@ class Workforce(BaseNode):# If workforce is paused, start the worker's listening taskself._start_child_node_when_paused(worker_node.start())- if self.metrics_logger:- self.metrics_logger.log_worker_created(- worker_id=worker_node.node_id,- worker_type='SingleAgentWorker',- role=worker_node.description,- )+ self._notify_worker_created(+ worker_node,+ worker_type='SingleAgentWorker',+ )return selfdef add_role_playing_worker(@@ -2069,12 +2142,10 @@ class Workforce(BaseNode):# If workforce is paused, start the worker's listening taskself._start_child_node_when_paused(worker_node.start())- if self.metrics_logger:- self.metrics_logger.log_worker_created(- worker_id=worker_node.node_id,- worker_type='RolePlayingWorker',- role=worker_node.description,- )+ self._notify_worker_created(+ worker_node,+ worker_type='RolePlayingWorker',+ )return selfdef add_workforce(self, workforce: Workforce) -> Workforce:@@ -2151,10 +2222,9 @@ class Workforce(BaseNode):# No active loop, directly set the eventself._pause_event.set()- if hasattr(self, 'metrics_logger') and self.metrics_logger is not None:- self.metrics_logger.reset_task_data()- else:- self.metrics_logger = WorkforceLogger(workforce_id=self.node_id)+ for cb in self._callbacks:+ if isinstance(cb, WorkforceMetrics):+ cb.reset_task_data()def save_workflow_memories(self,@@ -2983,10 +3053,11 @@ class Workforce(BaseNode):task.assigned_worker_id = assignee_id- if self.metrics_logger:- self.metrics_logger.log_task_started(- task_id=task.id, worker_id=assignee_id- )+ task_started_event = TaskStartedEvent(+ task_id=task.id, worker_id=assignee_id+ )+ for cb in self._callbacks:+ cb.log_task_started(task_started_event)try:await self._channel.post_task(task, self.node_id, assignee_id)@@ -3130,13 +3201,13 @@ class Workforce(BaseNode):print(f"{Fore.CYAN}{new_node} created.{Fore.RESET}")self._children.append(new_node)- if self.metrics_logger:- self.metrics_logger.log_worker_created(- worker_id=new_node.node_id,- worker_type='SingleAgentWorker',- role=new_node_conf.role,- metadata={'description': new_node_conf.description},- )++ self._notify_worker_created(+ new_node,+ worker_type='SingleAgentWorker',+ role=new_node_conf.role,+ metadata={'description': new_node_conf.description},+ )self._child_listening_tasks.append(asyncio.create_task(new_node.start()))@@ -3237,22 +3308,24 @@ class Workforce(BaseNode):batch_result = await self._find_assignee(tasks_to_assign)logger.debug(f"Coordinator returned assignments:\n"- f"{json.dumps(batch_result.dict(), indent=2)}"+ f"{json.dumps(batch_result.model_dump(), indent=2)}")for assignment in batch_result.assignments:self._task_dependencies[assignment.task_id] = (assignment.dependencies)self._assignees[assignment.task_id] = assignment.assignee_id- if self.metrics_logger:++ task_assigned_event = TaskAssignedEvent(+ task_id=assignment.task_id,+ worker_id=assignment.assignee_id,+ dependencies=assignment.dependencies,+ queue_time_seconds=None,+ )+ for cb in self._callbacks:# queue_time_seconds can be derived by logger if task# creation time is logged- self.metrics_logger.log_task_assigned(- task_id=assignment.task_id,- worker_id=assignment.assignee_id,- dependencies=assignment.dependencies,- queue_time_seconds=None,- )+ cb.log_task_assigned(task_assigned_event)# Step 2: Iterate through all pending tasks and post those that are# ready@@ -3369,21 +3442,19 @@ class Workforce(BaseNode):)# Log the failure to metrics- if self.metrics_logger:- self.metrics_logger.log_task_failed(- task_id=task.id,- worker_id=task.assigned_worker_id- or "unknown",- error_message=task.result,- metadata={- 'failure_reason': (- 'dependency_failure'- ),- 'failed_dependencies': (- permanently_failed_deps- ),- },- )+ task_failed_event = TaskFailedEvent(+ task_id=task.id,+ worker_id=task.assigned_worker_id or "unknown",+ error_message=task.result,+ metadata={+ 'failure_reason': 'dependency_failure',+ 'failed_dependencies': (+ permanently_failed_deps+ ),+ },+ )+ for cb in self._callbacks:+ cb.log_task_failed(task_failed_event)self._completed_tasks.append(task)self._cleanup_task_tracking(task.id)@@ -3435,17 +3506,18 @@ class Workforce(BaseNode):f"{failure_reason}{Fore.RESET}")- if self.metrics_logger:- self.metrics_logger.log_task_failed(- task_id=task.id,- worker_id=worker_id,- error_message=detailed_error,- metadata={- 'failure_count': task.failure_count,- 'task_content': task.content,- 'result_length': len(task.result) if task.result else 0,- },- )+ task_failed_event = TaskFailedEvent(+ task_id=task.id,+ worker_id=worker_id,+ error_message=detailed_error,+ metadata={+ 'failure_count': task.failure_count,+ 'task_content': task.content,+ 'result_length': len(task.result) if task.result else 0,+ },+ )+ for cb in self._callbacks:+ cb.log_task_failed(task_failed_event)# Check for immediate halt conditionsif task.failure_count >= MAX_TASK_RETRIES:@@ -3536,61 +3608,60 @@ class Workforce(BaseNode):return Falseasync def _handle_completed_task(self, task: Task) -> None:- if self.metrics_logger:- worker_id = task.assigned_worker_id or "unknown"- processing_time_seconds = None- token_usage = None-- # Get processing time from task start time or additional info- if task.id in self._task_start_times:- processing_time_seconds = (- time.time() - self._task_start_times[task.id]- )- self._cleanup_task_tracking(task.id)- elif (- task.additional_info is not None- and 'processing_time_seconds' in task.additional_info- ):- processing_time_seconds = task.additional_info[- 'processing_time_seconds'- ]+ worker_id = task.assigned_worker_id or "unknown"+ processing_time_seconds = None+ token_usage = None- # Get token usage from task additional info (preferred - actual- # usage)- if (- task.additional_info is not None- and 'token_usage' in task.additional_info- ):- token_usage = task.additional_info['token_usage']- else:- # Fallback: Try to get token usage from SingleAgentWorker- # memory- assignee_node = next(- (- child- for child in self._children- if child.node_id == worker_id- ),- None,- )- if isinstance(assignee_node, SingleAgentWorker):- try:- _, total_tokens = (- assignee_node.worker.memory.get_context()- )- token_usage = {'total_tokens': total_tokens}- except Exception:- token_usage = None+ # Get processing time from task start time or additional info+ if task.id in self._task_start_times:+ processing_time_seconds = (+ time.time() - self._task_start_times[task.id]+ )+ self._cleanup_task_tracking(task.id)+ elif (+ task.additional_info is not None+ and 'processing_time_seconds' in task.additional_info+ ):+ processing_time_seconds = task.additional_info[+ 'processing_time_seconds'+ ]- # Log the completed task- self.metrics_logger.log_task_completed(- task_id=task.id,- worker_id=worker_id,- result_summary=task.result if task.result else "Completed",- processing_time_seconds=processing_time_seconds,- token_usage=token_usage,- metadata={'current_state': task.state.value},+ # Get token usage from task additional info (preferred - actual+ # usage)+ if (+ task.additional_info is not None+ and 'token_usage' in task.additional_info+ ):+ token_usage = task.additional_info['token_usage']+ else:+ # Fallback: Try to get token usage from SingleAgentWorker+ # memory+ assignee_node = next(+ (+ child+ for child in self._children+ if child.node_id == worker_id+ ),+ None,)+ if isinstance(assignee_node, SingleAgentWorker):+ try:+ _, total_tokens = assignee_node.worker.memory.get_context()+ token_usage = {'total_tokens': total_tokens}+ except Exception:+ token_usage = None++ # Log the completed task+ task_completed_event = TaskCompletedEvent(+ task_id=task.id,+ worker_id=worker_id,+ result_summary=task.result if task.result else "Completed",+ processing_time_seconds=processing_time_seconds,+ token_usage=token_usage,+ metadata={'current_state': task.state.value},+ )+ for cb in self._callbacks:+ cb.log_task_completed(task_completed_event)# Find and remove the completed task from pending taskstasks_list = list(self._pending_tasks)@@ -3710,15 +3781,23 @@ class Workforce(BaseNode):r"""Returns an ASCII tree representation of the task hierarchy andworker status."""- if not self.metrics_logger:- return "Logger not initialized."- return self.metrics_logger.get_ascii_tree_representation()+ metrics_cb: List[WorkforceMetrics] = [+ cb for cb in self._callbacks if isinstance(cb, WorkforceMetrics)+ ]+ if len(metrics_cb) == 0:+ return "Metrics Callback not initialized."+ else:+ return metrics_cb[0].get_ascii_tree_representation()def get_workforce_kpis(self) -> Dict[str, Any]:r"""Returns a dictionary of key performance indicators."""- if not self.metrics_logger:- return {"error": "Logger not initialized."}- return self.metrics_logger.get_kpis()+ metrics_cb: List[WorkforceMetrics] = [+ cb for cb in self._callbacks if isinstance(cb, WorkforceMetrics)+ ]+ if len(metrics_cb) == 0:+ return {"error": "Metrics Callback not initialized."}+ else:+ return metrics_cb[0].get_kpis()def dump_workforce_logs(self, file_path: str) -> None:r"""Dumps all collected logs to a JSON file.@@ -3726,10 +3805,13 @@ class Workforce(BaseNode):Args:file_path (str): The path to the JSON file."""- if not self.metrics_logger:+ metrics_cb: List[WorkforceMetrics] = [+ cb for cb in self._callbacks if isinstance(cb, WorkforceMetrics)+ ]+ if len(metrics_cb) == 0:print("Logger not initialized. Cannot dump logs.")return- self.metrics_logger.dump_to_json(file_path)+ metrics_cb[0].dump_to_json(file_path)# Use logger.info or print, consistent with existing stylelogger.info(f"Workforce logs dumped to {file_path}")@@ -4177,6 +4259,9 @@ class Workforce(BaseNode):elif not self._pending_tasks and self._in_flight_tasks == 0:self._state = WorkforceState.IDLElogger.info("All tasks completed.")+ all_tasks_completed_event = AllTasksCompletedEvent()+ for cb in self._callbacks:+ cb.log_all_tasks_completed(all_tasks_completed_event)# shut down the whole workforce treeself.stop()diff --git a/camel/societies/workforce/workforce_callback.py b/camel/societies/workforce/workforce_callback.pynew file mode 100644index 000000000..9dee1d559--- /dev/null+++ b/camel/societies/workforce/workforce_callback.py@@ -0,0 +1,74 @@+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+from __future__ import annotations++from abc import ABC, abstractmethod++from .events import (+ AllTasksCompletedEvent,+ TaskAssignedEvent,+ TaskCompletedEvent,+ TaskCreatedEvent,+ TaskDecomposedEvent,+ TaskFailedEvent,+ TaskStartedEvent,+ WorkerCreatedEvent,+ WorkerDeletedEvent,+)+++class WorkforceCallback(ABC):+ r"""Interface for recording workforce lifecycle events.++ Implementations should persist or stream events as appropriate.+ """++ @abstractmethod+ def log_task_created(+ self,+ event: TaskCreatedEvent,+ ) -> None:+ pass++ @abstractmethod+ def log_task_decomposed(self, event: TaskDecomposedEvent) -> None:+ pass++ @abstractmethod+ def log_task_assigned(self, event: TaskAssignedEvent) -> None:+ pass++ @abstractmethod+ def log_task_started(self, event: TaskStartedEvent) -> None:+ pass++ @abstractmethod+ def log_task_completed(self, event: TaskCompletedEvent) -> None:+ pass++ @abstractmethod+ def log_task_failed(self, event: TaskFailedEvent) -> None:+ pass++ @abstractmethod+ def log_worker_created(self, event: WorkerCreatedEvent) -> None:+ pass++ @abstractmethod+ def log_worker_deleted(self, event: WorkerDeletedEvent) -> None:+ pass++ @abstractmethod+ def log_all_tasks_completed(self, event: AllTasksCompletedEvent) -> None:+ passdiff --git a/camel/societies/workforce/workforce_logger.py b/camel/societies/workforce/workforce_logger.pyindex 049d53249..b04c13428 100644--- a/camel/societies/workforce/workforce_logger.py+++ b/camel/societies/workforce/workforce_logger.py@@ -16,12 +16,26 @@ from datetime import datetime, timezonefrom typing import Any, Dict, List, Optionalfrom camel.logger import get_logger+from camel.societies.workforce.events import (+ AllTasksCompletedEvent,+ QueueStatusEvent,+ TaskAssignedEvent,+ TaskCompletedEvent,+ TaskCreatedEvent,+ TaskDecomposedEvent,+ TaskFailedEvent,+ TaskStartedEvent,+ WorkerCreatedEvent,+ WorkerDeletedEvent,+)+from camel.societies.workforce.workforce_callback import WorkforceCallback+from camel.societies.workforce.workforce_metrics import WorkforceMetricsfrom camel.types.agents import ToolCallingRecordlogger = get_logger(__name__)-class WorkforceLogger:+class WorkforceLogger(WorkforceCallback, WorkforceMetrics):r"""Logs events and metrics for a Workforce instance."""def __init__(self, workforce_id: str):@@ -55,195 +69,201 @@ class WorkforceLogger:def log_task_created(self,- task_id: str,- description: str,- parent_task_id: Optional[str] = None,- task_type: Optional[str] = None,- metadata: Optional[Dict[str, Any]] = None,+ event: TaskCreatedEvent,) -> None:r"""Logs the creation of a new task."""self._log_event(- 'task_created',- task_id=task_id,- description=description,- parent_task_id=parent_task_id,- task_type=task_type,- metadata=metadata or {},+ event_type=event.event_type,+ task_id=event.task_id,+ description=event.description,+ parent_task_id=event.parent_task_id,+ task_type=event.task_type,+ metadata=event.metadata or {},)- self._task_hierarchy[task_id] = {- 'parent': parent_task_id,+ self._task_hierarchy[event.task_id] = {+ 'parent': event.parent_task_id,'children': [],'status': 'created',- 'description': description,+ 'description': event.description,'assigned_to': None,- **(metadata or {}),+ **(event.metadata or {}),}- if parent_task_id and parent_task_id in self._task_hierarchy:- self._task_hierarchy[parent_task_id]['children'].append(task_id)+ if (+ event.parent_task_id+ and event.parent_task_id in self._task_hierarchy+ ):+ self._task_hierarchy[event.parent_task_id]['children'].append(+ event.task_id+ )def log_task_decomposed(self,- parent_task_id: str,- subtask_ids: List[str],- metadata: Optional[Dict[str, Any]] = None,+ event: TaskDecomposedEvent,) -> None:r"""Logs the decomposition of a task into subtasks."""self._log_event(- 'task_decomposed',- parent_task_id=parent_task_id,- subtask_ids=subtask_ids,- metadata=metadata or {},+ event_type=event.event_type,+ parent_task_id=event.parent_task_id,+ subtask_ids=event.subtask_ids,+ metadata=event.metadata or {},)- if parent_task_id in self._task_hierarchy:- self._task_hierarchy[parent_task_id]['status'] = "decomposed"+ if event.parent_task_id in self._task_hierarchy:+ self._task_hierarchy[event.parent_task_id]['status'] = "decomposed"def log_task_assigned(self,- task_id: str,- worker_id: str,- queue_time_seconds: Optional[float] = None,- dependencies: Optional[List[str]] = None,- metadata: Optional[Dict[str, Any]] = None,+ event: TaskAssignedEvent,) -> None:r"""Logs the assignment of a task to a worker."""self._log_event(- 'task_assigned',- task_id=task_id,- worker_id=worker_id,- queue_time_seconds=queue_time_seconds,- dependencies=dependencies or [],- metadata=metadata or {},+ event_type=event.event_type,+ task_id=event.task_id,+ worker_id=event.worker_id,+ queue_time_seconds=event.queue_time_seconds,+ dependencies=event.dependencies or [],+ metadata=event.metadata or {},)- if task_id in self._task_hierarchy:- self._task_hierarchy[task_id]['status'] = 'assigned'- self._task_hierarchy[task_id]['assigned_to'] = worker_id- self._task_hierarchy[task_id]['dependencies'] = dependencies or []- if worker_id in self._worker_information:- self._worker_information[worker_id]['current_task_id'] = task_id- self._worker_information[worker_id]['status'] = 'busy'+ if event.task_id in self._task_hierarchy:+ self._task_hierarchy[event.task_id]['status'] = 'assigned'+ self._task_hierarchy[event.task_id]['assigned_to'] = (+ event.worker_id+ )+ self._task_hierarchy[event.task_id]['dependencies'] = (+ event.dependencies or []+ )+ if event.worker_id in self._worker_information:+ self._worker_information[event.worker_id]['current_task_id'] = (+ event.task_id+ )+ self._worker_information[event.worker_id]['status'] = 'busy'def log_task_started(self,- task_id: str,- worker_id: str,- metadata: Optional[Dict[str, Any]] = None,+ event: TaskStartedEvent,) -> None:r"""Logs when a worker starts processing a task."""self._log_event(- 'task_started',- task_id=task_id,- worker_id=worker_id,- metadata=metadata or {},+ event_type=event.event_type,+ task_id=event.task_id,+ worker_id=event.worker_id,+ metadata=event.metadata or {},)- if task_id in self._task_hierarchy:- self._task_hierarchy[task_id]['status'] = 'processing'+ if event.task_id in self._task_hierarchy:+ self._task_hierarchy[event.task_id]['status'] = 'processing'- def log_task_completed(- self,- task_id: str,- worker_id: str,- result_summary: Optional[str] = None,- processing_time_seconds: Optional[float] = None,- token_usage: Optional[Dict[str, int]] = None,- metadata: Optional[Dict[str, Any]] = None,- ) -> None:+ def log_task_completed(self, event: TaskCompletedEvent) -> None:r"""Logs the successful completion of a task."""self._log_event(- 'task_completed',- task_id=task_id,- worker_id=worker_id,- result_summary=result_summary,- processing_time_seconds=processing_time_seconds,- token_usage=token_usage or {},- metadata=metadata or {},+ event_type=event.event_type,+ task_id=event.task_id,+ worker_id=event.worker_id,+ result_summary=event.result_summary,+ processing_time_seconds=event.processing_time_seconds,+ token_usage=event.token_usage or {},+ metadata=event.metadata or {},)- if task_id in self._task_hierarchy:- self._task_hierarchy[task_id]['status'] = 'completed'- self._task_hierarchy[task_id]['assigned_to'] = None+ if event.task_id in self._task_hierarchy:+ self._task_hierarchy[event.task_id]['status'] = 'completed'+ self._task_hierarchy[event.task_id]['assigned_to'] = None# Store processing time in task hierarchy for display in tree- if processing_time_seconds is not None:- self._task_hierarchy[task_id]['completion_time_seconds'] = (- processing_time_seconds- )+ if event.processing_time_seconds is not None:+ self._task_hierarchy[event.task_id][+ 'completion_time_seconds'+ ] = event.processing_time_seconds# Store token usage in task hierarchy for display in tree- if token_usage is not None:- self._task_hierarchy[task_id]['token_usage'] = token_usage- if worker_id in self._worker_information:- self._worker_information[worker_id]['current_task_id'] = None- self._worker_information[worker_id]['status'] = 'idle'- self._worker_information[worker_id]['tasks_completed'] = (- self._worker_information[worker_id].get('tasks_completed', 0)+ if event.token_usage is not None:+ self._task_hierarchy[event.task_id]['token_usage'] = (+ event.token_usage+ )+ if event.worker_id in self._worker_information:+ self._worker_information[event.worker_id]['current_task_id'] = None+ self._worker_information[event.worker_id]['status'] = 'idle'+ self._worker_information[event.worker_id]['tasks_completed'] = (+ self._worker_information[event.worker_id].get(+ 'tasks_completed', 0+ )+ 1)def log_task_failed(self,- task_id: str,- error_message: str,- worker_id: Optional[str] = None,- metadata: Optional[Dict[str, Any]] = None,+ event: TaskFailedEvent,) -> None:r"""Logs the failure of a task."""self._log_event(- 'task_failed',- task_id=task_id,- worker_id=worker_id,- error_message=error_message,- metadata=metadata or {},+ event_type=event.event_type,+ task_id=event.task_id,+ worker_id=event.worker_id,+ error_message=event.error_message,+ metadata=event.metadata or {},)- if task_id in self._task_hierarchy:- self._task_hierarchy[task_id]['status'] = 'failed'- self._task_hierarchy[task_id]['error'] = error_message- self._task_hierarchy[task_id]['assigned_to'] = None- if worker_id and worker_id in self._worker_information:- self._worker_information[worker_id]['current_task_id'] = None- self._worker_information[worker_id]['status'] = 'idle'- self._worker_information[worker_id]['tasks_failed'] = (- self._worker_information[worker_id].get('tasks_failed', 0) + 1+ if event.task_id in self._task_hierarchy:+ self._task_hierarchy[event.task_id]['status'] = 'failed'+ self._task_hierarchy[event.task_id]['error'] = event.error_message+ self._task_hierarchy[event.task_id]['assigned_to'] = None+ if event.worker_id and event.worker_id in self._worker_information:+ self._worker_information[event.worker_id]['current_task_id'] = None+ self._worker_information[event.worker_id]['status'] = 'idle'+ self._worker_information[event.worker_id]['tasks_failed'] = (+ self._worker_information[event.worker_id].get(+ 'tasks_failed', 0+ )+ + 1)def log_worker_created(self,- worker_id: str,- worker_type: str,- role: str,- metadata: Optional[Dict[str, Any]] = None,+ event: WorkerCreatedEvent,) -> None:r"""Logs the creation of a new worker."""self._log_event(- 'worker_created',- worker_id=worker_id,- worker_type=worker_type,- role=role,- metadata=metadata or {},+ event_type=event.event_type,+ worker_id=event.worker_id,+ worker_type=event.worker_type,+ role=event.role,+ metadata=event.metadata or {},)- self._worker_information[worker_id] = {- 'type': worker_type,- 'role': role,+ self._worker_information[event.worker_id] = {+ 'type': event.worker_type,+ 'role': event.role,'status': 'idle','current_task_id': None,'tasks_completed': 0,'tasks_failed': 0,- **(metadata or {}),+ **(event.metadata or {}),}def log_worker_deleted(self,- worker_id: str,- reason: Optional[str] = None,- metadata: Optional[Dict[str, Any]] = None,+ event: WorkerDeletedEvent,) -> None:r"""Logs the deletion of a worker."""self._log_event(- 'worker_deleted',- worker_id=worker_id,- reason=reason,- metadata=metadata or {},+ event_type=event.event_type,+ worker_id=event.worker_id,+ reason=event.reason,+ metadata=event.metadata or {},)- if worker_id in self._worker_information:- self._worker_information[worker_id]['status'] = 'deleted'+ if event.worker_id in self._worker_information:+ self._worker_information[event.worker_id]['status'] = 'deleted'# Or del self._worker_information[worker_id]+ def log_queue_status(+ self,+ event: QueueStatusEvent,+ ) -> None:+ r"""Logs the status of a task queue."""+ self._log_event(+ event_type=event.event_type,+ queue_name=event.queue_name,+ length=event.length,+ pending_task_ids=event.pending_task_ids or [],+ metadata=event.metadata or {},+ )++ def log_all_tasks_completed(self, event: AllTasksCompletedEvent) -> None:+ pass+def reset_task_data(self) -> None:r"""Resets logs and data related to tasks, preserving workerinformation.@@ -263,22 +283,6 @@ class WorkforceLogger:f"{self.workforce_id}")- def log_queue_status(- self,- queue_name: str,- length: int,- pending_task_ids: Optional[List[str]] = None,- metadata: Optional[Dict[str, Any]] = None,- ) -> None:- r"""Logs the status of a task queue."""- self._log_event(- 'queue_status',- queue_name=queue_name,- length=length,- pending_task_ids=pending_task_ids or [],- metadata=metadata or {},- )-def dump_to_json(self, file_path: str) -> None:r"""Dumps all log entries to a JSON file.diff --git a/camel/societies/workforce/workforce_metrics.py b/camel/societies/workforce/workforce_metrics.pynew file mode 100644index 000000000..06a7735c6--- /dev/null+++ b/camel/societies/workforce/workforce_metrics.py@@ -0,0 +1,33 @@+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+from abc import ABC, abstractmethod+from typing import Any, Dict+++class WorkforceMetrics(ABC):+ @abstractmethod+ def reset_task_data(self) -> None:+ pass++ @abstractmethod+ def dump_to_json(self, file_path: str) -> None:+ pass++ @abstractmethod+ def get_ascii_tree_representation(self) -> str:+ pass++ @abstractmethod+ def get_kpis(self) -> Dict[str, Any]:+ passdiff --git a/docs/key_modules/workforce.md b/docs/key_modules/workforce.mdindex d2192203a..2e33e5a04 100644--- a/docs/key_modules/workforce.md+++ b/docs/key_modules/workforce.md@@ -22,8 +22,10 @@ class Workforce(BaseNode):task_agent: Optional[ChatAgent] = None,new_worker_agent: Optional[ChatAgent] = None,graceful_shutdown_timeout: float = 15.0,+ task_timeout_seconds: Optional[float] = None,share_memory: bool = False,use_structured_output_handler: bool = True,+ callbacks: Optional[List[WorkforceCallback]] = None,) -> None:# ...```@@ -35,8 +37,10 @@ class Workforce(BaseNode):- `coordinator_agent`: A `ChatAgent` for assigning tasks.- `task_agent`: A `ChatAgent` for decomposing tasks.- `new_worker_agent`: A template `ChatAgent` for creating new workers.+- `task_timeout_seconds`: Optional per-workforce task timeout in seconds.- `share_memory`: If `True`, `SingleAgentWorker` instances will share memory.- `use_structured_output_handler`: Defaults to `True`. Enables structured output handling so models without native JSON + tool-calling can still interop reliably.+- `callbacks`: Optional A list of callback handlers to observe and record workforce lifecycle events and metrics.</Card><Card title="Worker Types" icon="users-gear">@@ -247,4 +251,4 @@ Understanding these models is key to interpreting the workforce's internal state>Full documentation for advanced usage and configuration.</Card>-</CardGroup>\ No newline at end of file+</CardGroup>diff --git a/examples/workforce/workforce_callbacks_example.py b/examples/workforce/workforce_callbacks_example.pynew file mode 100644index 000000000..01fdeda2a--- /dev/null+++ b/examples/workforce/workforce_callbacks_example.py@@ -0,0 +1,148 @@+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+"""+Example: Subscribe to callbacks in Workforce++Includes:+- Custom callback `PrintCallback` that logs key lifecycle events.+- Subscribe callbacks via the `callbacks` parameter when constructing a+ `Workforce`.+"""++import asyncio++from camel.agents import ChatAgent+from camel.logger import get_logger+from camel.models import ModelFactory+from camel.societies.workforce.events import (+ AllTasksCompletedEvent,+ TaskAssignedEvent,+ TaskCompletedEvent,+ TaskCreatedEvent,+ TaskDecomposedEvent,+ TaskFailedEvent,+ TaskStartedEvent,+ WorkerCreatedEvent,+ WorkerDeletedEvent,+)+from camel.societies.workforce.workforce import Workforce+from camel.societies.workforce.workforce_callback import WorkforceCallback+from camel.types import ModelPlatformType, ModelType++logger = get_logger(__name__)+++class PrintCallback(WorkforceCallback):+ r"""Simple callback printing events to logs to observe ordering."""++ def log_task_created(self, event: TaskCreatedEvent) -> None:+ print(+ f"[PrintCallback] task_created: id={event.task_id}, "+ f"desc={event.description!r}, parent={event.parent_task_id}"+ )++ def log_task_decomposed(self, event: TaskDecomposedEvent) -> None:+ print(+ f"[PrintCallback] task_decomposed: parent={event.parent_task_id},"+ f" subtasks={event.subtask_ids}"+ )++ def log_task_assigned(self, event: TaskAssignedEvent) -> None:+ print(+ f"[PrintCallback] task_assigned: task={event.task_id}, "+ f"worker={event.worker_id}"+ )++ def log_task_started(self, event: TaskStartedEvent) -> None:+ print(+ f"[PrintCallback] task_started: task={event.task_id}, "+ f"worker={event.worker_id}"+ )++ def log_task_completed(self, event: TaskCompletedEvent) -> None:+ print(+ f"[PrintCallback] task_completed: task={event.task_id}, "+ f"worker={event.worker_id}, took={event.processing_time_seconds}s"+ )++ def log_task_failed(self, event: TaskFailedEvent) -> None:+ logger.warning(+ f"[PrintCallback] task_failed: task={event.task_id}, "+ f"err={event.error_message}"+ )++ def log_worker_created(self, event: WorkerCreatedEvent) -> None:+ print(+ f"[PrintCallback] worker_created: id={event.worker_id}, "+ f"type={event.worker_type}, role={event.role}"+ )++ def log_worker_deleted(self, event: WorkerDeletedEvent) -> None:+ print(+ f"[PrintCallback] worker_deleted: id={event.worker_id}, "+ f"reason={event.reason}"+ )++ def log_all_tasks_completed(self, event: AllTasksCompletedEvent) -> None:+ print("[PrintCallback] all_tasks_completed")+++def build_teacher_agent() -> ChatAgent:+ model = ModelFactory.create(+ model_platform=ModelPlatformType.OPENAI,+ model_type=ModelType.GPT_4O_MINI,+ )+ return ChatAgent(system_message="You are a teacher", model=model)+++def build_student_agent() -> ChatAgent:+ model = ModelFactory.create(+ model_platform=ModelPlatformType.OPENAI,+ model_type=ModelType.GPT_4O_MINI,+ )+ return ChatAgent(system_message="You are a student", model=model)+++async def run_demo() -> None:+ callbacks = [PrintCallback()]++ workforce = Workforce(+ "Workforce Callbacks Demo",+ callbacks=callbacks,+ use_structured_output_handler=True,+ )++ teacher = build_teacher_agent()+ student = build_student_agent()+ workforce.add_single_agent_worker("Teacher Worker", teacher)+ workforce.add_single_agent_worker("Student Worker", student)+ workforce.add_main_task(+ "The teacher set an exam question and had the students answer it."+ )++ # Start Workforce and wait for completion (timeout to avoid hanging)+ wf_task = asyncio.create_task(workforce.start())+ try:+ await asyncio.wait_for(wf_task, timeout=30.0)+ except asyncio.TimeoutError:+ logger.warning("Workforce run timed out; stopping...")+ workforce.stop()++ # Read KPIs and a simple "tree"+ print(f"KPIs: {workforce.get_workforce_kpis()}")+ print(f"Tree: {workforce.get_workforce_log_tree()}")+++if __name__ == "__main__":+ asyncio.run(run_demo())diff --git a/test/workforce/test_workforce_callbacks.py b/test/workforce/test_workforce_callbacks.pynew file mode 100644index 000000000..58b439880--- /dev/null+++ b/test/workforce/test_workforce_callbacks.py@@ -0,0 +1,366 @@+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+from typing import Any, Dict++import pytest++from camel.agents import ChatAgent+from camel.messages import BaseMessage+from camel.models import ModelFactory+from camel.societies.workforce.events import (+ AllTasksCompletedEvent,+ TaskAssignedEvent,+ TaskCompletedEvent,+ TaskCreatedEvent,+ TaskDecomposedEvent,+ TaskFailedEvent,+ TaskStartedEvent,+ WorkerCreatedEvent,+ WorkerDeletedEvent,+ WorkforceEvent,+)+from camel.societies.workforce.workforce import Workforce+from camel.societies.workforce.workforce_callback import WorkforceCallback+from camel.societies.workforce.workforce_logger import WorkforceLogger+from camel.societies.workforce.workforce_metrics import WorkforceMetrics+from camel.tasks import Task+from camel.types import ModelPlatformType, ModelType+++class _NonMetricsCallback(WorkforceCallback):+ """A minimal callback implementation without metrics, for testing."""++ def __init__(self) -> None:+ self.events: list[WorkforceEvent] = []++ # Task events+ def log_task_created(self, event: TaskCreatedEvent) -> None:+ self.events.append(event)++ def log_task_decomposed(self, event: TaskDecomposedEvent) -> None:+ self.events.append(event)++ def log_task_assigned(self, event: TaskAssignedEvent) -> None:+ self.events.append(event)++ def log_task_started(self, event: TaskStartedEvent) -> None:+ self.events.append(event)++ def log_task_completed(self, event: TaskCompletedEvent) -> None:+ self.events.append(event)++ def log_task_failed(self, event: TaskFailedEvent) -> None:+ self.events.append(event)++ # Worker events+ def log_worker_created(self, event: WorkerCreatedEvent) -> None:+ self.events.append(event)++ def log_worker_deleted(self, event: WorkerDeletedEvent) -> None:+ self.events.append(event)++ # Terminal event+ def log_all_tasks_completed(self, event: AllTasksCompletedEvent) -> None:+ self.events.append(event)+++class _MetricsCallback(WorkforceCallback, WorkforceMetrics):+ """A minimal metrics-capable callback for testing."""++ def __init__(self) -> None:+ self.events: list[WorkforceEvent] = []+ self.reset_task_data()+ self.dump_to_json_called = False+ self.get_ascii_tree_called = False+ self.get_kpis_called = False++ # WorkforceMetrics interface+ def reset_task_data(self) -> None:+ self.dump_to_json_called = False+ self.get_ascii_tree_called = False+ self.get_kpis_called = False++ def dump_to_json(self, file_path: str) -> None:+ self.dump_to_json_called = True++ def get_ascii_tree_representation(self) -> str:+ self.get_ascii_tree_called = True+ return "Stub ASCII Tree"++ def get_kpis(self) -> Dict[str, Any]:+ self.get_kpis_called = True+ return {}++ def log_task_created(self, event: TaskCreatedEvent) -> None:+ self.events.append(event)++ def log_task_decomposed(self, event: TaskDecomposedEvent) -> None:+ self.events.append(event)++ def log_task_assigned(self, event: TaskAssignedEvent) -> None:+ self.events.append(event)++ def log_task_started(self, event: TaskStartedEvent) -> None:+ self.events.append(event)++ def log_task_completed(self, event: TaskCompletedEvent) -> None:+ self.events.append(event)++ def log_task_failed(self, event: TaskFailedEvent) -> None:+ self.events.append(event)++ def log_worker_created(self, event: WorkerCreatedEvent) -> None:+ self.events.append(event)++ def log_worker_deleted(self, event: WorkerDeletedEvent) -> None:+ self.events.append(event)++ def log_all_tasks_completed(self, event: AllTasksCompletedEvent) -> None:+ self.events.append(event)+++def _build_stub_agent() -> ChatAgent:+ """Construct a stub-backed ChatAgent for offline tests."""+ model = ModelFactory.create(+ model_platform=ModelPlatformType.OPENAI,+ model_type=ModelType.STUB,+ )+ return ChatAgent(model=model)+++def test_workforce_callback_registration_and_metrics_handling():+ """Verify default logger addition and metrics-callback skip logic.++ - When no metrics callback is provided, WorkforceLogger is added.+ - When a metrics callback is provided, no WorkforceLogger is added.+ - Invalid callback types raise ValueError.+ - Worker creation events are propagated to provided callbacks.+ """+ # 1) No metrics callback -> default WorkforceLogger should be appended+ cb = _NonMetricsCallback()+ wf1 = Workforce("CB Test - No Metrics", callbacks=[cb])+ callback_types = [type(c) for c in wf1._callbacks]+ assert any(issubclass(t, WorkforceLogger) for t in callback_types)+ # Ensure our custom callback is present+ assert any(isinstance(c, _NonMetricsCallback) for c in wf1._callbacks)++ # Add a worker and ensure our callback saw the event+ agent = _build_stub_agent()+ wf1.add_single_agent_worker("UnitTest Worker", agent)+ assert any(isinstance(e, WorkerCreatedEvent) for e in cb.events)++ # 2) Metrics-capable callback present -> no default WorkforceLogger+ metrics_cb = _MetricsCallback()+ wf2 = Workforce("CB Test - With Metrics", callbacks=[metrics_cb])+ assert all(not isinstance(c, WorkforceLogger) for c in wf2._callbacks)+ # Also confirm that the metrics callback remains+ assert any(isinstance(c, _MetricsCallback) for c in wf2._callbacks)++ # 3) Invalid callback type -> ValueError+ with pytest.raises(ValueError, match="instances of WorkforceCallback"):+ Workforce("CB Test - Invalid", callbacks=[object()])+++def assert_event_sequence(events: list[str], min_worker_count: int):+ """+ Validate that the given event sequence follows the expected logical order.+ This version is flexible to handle:+ - Task retries and dynamic worker creation+ - Cases where tasks are not decomposed (e.g., when using stub models)+ """+ idx = 0+ n = len(events)++ # 1. Expect at least min_worker_count WorkerCreatedEvent events first+ initial_worker_count = 0+ while idx < n and events[idx] == "WorkerCreatedEvent":+ initial_worker_count += 1+ idx += 1+ assert initial_worker_count >= min_worker_count, (+ f"Expected at least {min_worker_count} initial "+ f"WorkerCreatedEvents, got {initial_worker_count}"+ )++ # 2. Expect one main TaskCreatedEvent+ assert idx < n and events[idx] == "TaskCreatedEvent", (+ f"Event {idx} should be TaskCreatedEvent, got "+ f"{events[idx] if idx < n else 'END'}"+ )+ idx += 1++ # 3. TaskDecomposedEvent may or may not be present+ # (depends on coordinator behavior)+ # If the coordinator can't parse stub responses, it may skip+ # decomposition+ has_decomposition = idx < n and events[idx] == "TaskDecomposedEvent"+ if has_decomposition:+ idx += 1++ # 4. Count all event types in the remaining events+ all_events = events[idx:]+ task_assigned_count = all_events.count("TaskAssignedEvent")+ task_started_count = all_events.count("TaskStartedEvent")+ task_completed_count = all_events.count("TaskCompletedEvent")+ all_tasks_completed_count = all_events.count("AllTasksCompletedEvent")++ # 5. Validate basic invariants+ # At minimum, the main task should be assigned and processed+ assert (+ task_assigned_count >= 1+ ), f"Expected at least 1 TaskAssignedEvent, got {task_assigned_count}"+ assert (+ task_started_count >= 1+ ), f"Expected at least 1 TaskStartedEvent, got {task_started_count}"+ assert (+ task_completed_count >= 1+ ), f"Expected at least 1 TaskCompletedEvent, got {task_completed_count}"++ # 6. Expect exactly one AllTasksCompletedEvent at the end+ assert all_tasks_completed_count == 1, (+ f"Expected exactly 1 AllTasksCompletedEvent, got "+ f"{all_tasks_completed_count}"+ )+ assert (+ events[-1] == "AllTasksCompletedEvent"+ ), "Last event should be AllTasksCompletedEvent"++ # 7. All events should be of expected types+ allowed_events = {+ "WorkerCreatedEvent",+ "WorkerDeletedEvent",+ "TaskCreatedEvent",+ "TaskDecomposedEvent",+ "TaskAssignedEvent",+ "TaskStartedEvent",+ "TaskCompletedEvent",+ "TaskFailedEvent",+ "AllTasksCompletedEvent",+ }+ for i, e in enumerate(events):+ assert e in allowed_events, f"Unexpected event type at {i}: {e}"+++def test_workforce_emits_expected_event_sequence():+ # Use STUB model to avoid real API calls and ensure fast,+ # deterministic execution+ search_agent = ChatAgent(+ system_message=BaseMessage.make_assistant_message(+ role_name="Research Specialist",+ content="You are a research specialist who excels at finding and "+ "gathering information from the web.",+ ),+ model=ModelFactory.create(+ model_platform=ModelPlatformType.OPENAI,+ model_type=ModelType.STUB,+ ),+ )++ analyst_agent = ChatAgent(+ system_message=BaseMessage.make_assistant_message(+ role_name="Business Analyst",+ content="You are an expert business analyst. Your job is "+ "to analyze research findings, identify key insights, "+ "opportunities, and challenges.",+ ),+ model=ModelFactory.create(+ model_platform=ModelPlatformType.OPENAI,+ model_type=ModelType.STUB,+ ),+ )++ writer_agent = ChatAgent(+ system_message=BaseMessage.make_assistant_message(+ role_name="Report Writer",+ content="You are a professional report writer. You take "+ "analytical insights and synthesize them into a clear, "+ "concise, and well-structured final report.",+ ),+ model=ModelFactory.create(+ model_platform=ModelPlatformType.OPENAI,+ model_type=ModelType.STUB,+ ),+ )++ cb = _MetricsCallback()++ # Use STUB models for coordinator and task agents to avoid real API calls+ coordinator_agent = ChatAgent(+ model=ModelFactory.create(+ model_platform=ModelPlatformType.OPENAI,+ model_type=ModelType.STUB,+ )+ )+ task_agent = ChatAgent(+ model=ModelFactory.create(+ model_platform=ModelPlatformType.OPENAI,+ model_type=ModelType.STUB,+ )+ )++ workforce = Workforce(+ 'Business Analysis Team',+ graceful_shutdown_timeout=30.0,+ callbacks=[cb],+ coordinator_agent=coordinator_agent,+ task_agent=task_agent,+ )++ workforce.add_single_agent_worker(+ "A researcher who can search online for information.",+ worker=search_agent,+ ).add_single_agent_worker(+ "An analyst who can process research findings.", worker=analyst_agent+ ).add_single_agent_worker(+ "A writer who can create a final report from the analysis.",+ worker=writer_agent,+ )++ # Use a simpler task to ensure fast and deterministic execution+ human_task = Task(+ content=(+ "Create a simple report about electric scooters. "+ "The report should have three sections: "+ "1. Market overview "+ "2. Target customers "+ "3. Summary"+ ),+ id='0',+ )++ workforce.process_task(human_task)++ # test that the event sequence is as expected+ actual_events = [e.__class__.__name__ for e in cb.events]+ assert_event_sequence(actual_events, min_worker_count=3)++ # test that metrics callback methods work as expected+ assert not cb.dump_to_json_called+ assert not cb.get_ascii_tree_called+ assert not cb.get_kpis_called+ workforce.dump_workforce_logs("foo.log")+ assert cb.dump_to_json_called++ workforce.reset()+ assert not cb.dump_to_json_called+ assert not cb.get_ascii_tree_called+ assert not cb.get_kpis_called+ workforce.get_workforce_kpis()+ assert cb.get_kpis_called++ workforce.reset()+ assert not cb.dump_to_json_called+ assert not cb.get_ascii_tree_called+ assert not cb.get_kpis_called+ workforce.get_workforce_log_tree()+ assert cb.get_ascii_tree_called