Framework for orchestrating role-playing, autonomous AI agents. By fostering collaborative intelligence, CrewAI empowers agents to work together seamlessly, tackling complex tasks.
Implement thread-safe AgentEvaluator (#3157)
Lucas Gomideb6d699f76412
diff --git a/src/crewai/experimental/evaluation/agent_evaluator.py b/src/crewai/experimental/evaluation/agent_evaluator.pyindex 3cf9583dc..5e6f490bf 100644--- a/src/crewai/experimental/evaluation/agent_evaluator.py+++ b/src/crewai/experimental/evaluation/agent_evaluator.py@@ -11,6 +11,16 @@ from crewai.crew import Crewfrom crewai.utilities.events.crewai_event_bus import crewai_event_busfrom crewai.utilities.events.utils.console_formatter import ConsoleFormatterfrom crewai.experimental.evaluation.evaluation_display import AgentAggregatedEvaluationResult+from contextlib import contextmanager+import threading++class ExecutionState:+ def __init__(self):+ self.traces: dict[str, Any] = {}+ self.current_agent_id: str | None = None+ self.current_task_id: str | None = None+ self.iteration: int = 1+ self.iterations_results: dict[int, dict[str, list[AgentEvaluationResult]]] = {}class AgentEvaluator:def __init__(@@ -21,24 +31,37 @@ class AgentEvaluator:self.crew: Crew | None = crewself.evaluators: Sequence[BaseEvaluator] | None = evaluators+ self.callback = create_evaluation_callbacks()+ self.console_formatter = ConsoleFormatter()+ self.display_formatter = EvaluationDisplayFormatter()++ self._thread_local: threading.local = threading.local()+self.agent_evaluators: dict[str, Sequence[BaseEvaluator] | None] = {}if crew is not None:assert crew and crew.agents is not Nonefor agent in crew.agents:self.agent_evaluators[str(agent.id)] = self.evaluators- self.callback = create_evaluation_callbacks()- self.console_formatter = ConsoleFormatter()- self.display_formatter = EvaluationDisplayFormatter()+ @contextmanager+ def execution_context(self):+ state = ExecutionState()+ try:+ yield state+ finally:+ pass- self.iteration = 1- self.iterations_results: dict[int, dict[str, list[AgentEvaluationResult]]] = {}+ @property+ def _execution_state(self) -> ExecutionState:+ if not hasattr(self._thread_local, 'execution_state'):+ self._thread_local.execution_state = ExecutionState()+ return self._thread_local.execution_statedef set_iteration(self, iteration: int) -> None:- self.iteration = iteration+ self._execution_state.iteration = iteration- def reset_iterations_results(self):- self.iterations_results = {}+ def reset_iterations_results(self) -> None:+ self._execution_state.iterations_results = {}def evaluate_current_iteration(self) -> dict[str, list[AgentEvaluationResult]]:if not self.crew:@@ -63,45 +86,50 @@ class AgentEvaluator:TextColumn("{task.percentage:.0f}% completed"),console=self.console_formatter.console) as progress:- eval_task = progress.add_task(f"Evaluating agents (iteration {self.iteration})...", total=total_evals)-- for agent in self.crew.agents:- evaluator = self.agent_evaluators.get(str(agent.id))- if not evaluator:- continue+ eval_task = progress.add_task(f"Evaluating agents (iteration {self._execution_state.iteration})...", total=total_evals)- for task in self.crew.tasks:+ with self.execution_context() as state:+ state.iteration = self._execution_state.iteration- if task.agent and str(task.agent.id) != str(agent.id):+ for agent in self.crew.agents:+ evaluator = self.agent_evaluators.get(str(agent.id))+ if not evaluator:continue- trace = self.callback.get_trace(str(agent.id), str(task.id))- if not trace:- self.console_formatter.print(f"[yellow]Warning: No trace found for agent {agent.role} on task {task.description[:30]}...[/yellow]")- progress.update(eval_task, advance=1)- continue-- with crewai_event_bus.scoped_handlers():- result = self.evaluate(- agent=agent,- task=task,- execution_trace=trace,- final_output=task.output- )- evaluation_results[agent.role].append(result)- progress.update(eval_task, advance=1)-- self.iterations_results[self.iteration] = evaluation_results+ for task in self.crew.tasks:+ if task.agent and str(task.agent.id) != str(agent.id):+ continue++ trace = self.callback.get_trace(str(agent.id), str(task.id))+ if not trace:+ self.console_formatter.print(f"[yellow]Warning: No trace found for agent {agent.role} on task {task.description[:30]}...[/yellow]")+ progress.update(eval_task, advance=1)+ continue++ state.current_agent_id = str(agent.id)+ state.current_task_id = str(task.id)++ with crewai_event_bus.scoped_handlers():+ result = self.evaluate(+ agent=agent,+ task=task,+ execution_trace=trace,+ final_output=task.output,+ state=state+ )+ evaluation_results[agent.role].append(result)+ progress.update(eval_task, advance=1)++ self._execution_state.iterations_results[self._execution_state.iteration] = evaluation_resultsreturn evaluation_results- def get_evaluation_results(self):- if self.iteration in self.iterations_results:- return self.iterations_results[self.iteration]-+ def get_evaluation_results(self) -> dict[str, list[AgentEvaluationResult]]:+ if self._execution_state.iteration in self._execution_state.iterations_results:+ return self._execution_state.iterations_results[self._execution_state.iteration]return self.evaluate_current_iteration()- def display_results_with_iterations(self):- self.display_formatter.display_summary_results(self.iterations_results)+ def display_results_with_iterations(self) -> None:+ self.display_formatter.display_summary_results(self._execution_state.iterations_results)def get_agent_evaluation(self, strategy: AggregationStrategy = AggregationStrategy.SIMPLE_AVERAGE, include_evaluation_feedback: bool = False) -> Dict[str, AgentAggregatedEvaluationResult]:agent_results = {}@@ -123,7 +151,7 @@ class AgentEvaluator:agent_results[agent_role] = aggregated_result- if self.iteration == max(self.iterations_results.keys()):+ if self._execution_state.iterations_results and self._execution_state.iteration == max(self._execution_state.iterations_results.keys(), default=0):self.display_results_with_iterations()if include_evaluation_feedback:@@ -131,20 +159,22 @@ class AgentEvaluator:return agent_results- def display_evaluation_with_feedback(self):- self.display_formatter.display_evaluation_with_feedback(self.iterations_results)+ def display_evaluation_with_feedback(self) -> None:+ self.display_formatter.display_evaluation_with_feedback(self._execution_state.iterations_results)def evaluate(self,agent: Agent,task: Task,- execution_trace: Dict[str, Any],- final_output: Any+ execution_trace: dict[str, Any],+ final_output: Any,+ state: ExecutionState) -> AgentEvaluationResult:result = AgentEvaluationResult(- agent_id=str(agent.id),- task_id=str(task.id)+ agent_id=state.current_agent_id or str(agent.id),+ task_id=state.current_task_id or str(task.id))+assert self.evaluators is not Nonefor evaluator in self.evaluators:try:diff --git a/src/crewai/experimental/evaluation/evaluation_display.py b/src/crewai/experimental/evaluation/evaluation_display.pyindex c25ea6db4..028d6d79e 100644--- a/src/crewai/experimental/evaluation/evaluation_display.py+++ b/src/crewai/experimental/evaluation/evaluation_display.py@@ -17,7 +17,6 @@ class EvaluationDisplayFormatter:self.console_formatter.print("[yellow]No evaluation results to display[/yellow]")return- # Get all agent roles across all iterationsall_agent_roles: set[str] = set()for iter_results in iterations_results.values():all_agent_roles.update(iter_results.keys())@@ -25,7 +24,6 @@ class EvaluationDisplayFormatter:for agent_role in sorted(all_agent_roles):self.console_formatter.print(f"\n[bold cyan]Agent: {agent_role}[/bold cyan]")- # Process each iterationfor iter_num, results in sorted(iterations_results.items()):if agent_role not in results or not results[agent_role]:continue@@ -33,23 +31,19 @@ class EvaluationDisplayFormatter:agent_results = results[agent_role]agent_id = agent_results[0].agent_id- # Aggregate results for this agent in this iterationaggregated_result = self._aggregate_agent_results(agent_id=agent_id,agent_role=agent_role,results=agent_results,)- # Display iteration headerself.console_formatter.print(f"\n[bold]Iteration {iter_num}[/bold]")- # Create table for this iterationtable = Table(box=ROUNDED)table.add_column("Metric", style="cyan")table.add_column("Score (1-10)", justify="center")table.add_column("Feedback", style="green")- # Add metrics to tableif aggregated_result.metrics:for metric, evaluation_score in aggregated_result.metrics.items():score = evaluation_score.score@@ -91,7 +85,6 @@ class EvaluationDisplayFormatter:"Overall agent evaluation score")- # Print the table for this iterationself.console_formatter.print(table)def display_summary_results(self, iterations_results: Dict[int, Dict[str, List[AgentAggregatedEvaluationResult]]]):@@ -248,7 +241,6 @@ class EvaluationDisplayFormatter:feedback_summary = Noneif feedbacks:if len(feedbacks) > 1:- # Use the summarization method for multiple feedbacksfeedback_summary = self._summarize_feedbacks(agent_role=agent_role,metric=category.title(),@@ -307,7 +299,7 @@ class EvaluationDisplayFormatter:strategy_guidance = "Focus on the highest-scoring aspects and strengths demonstrated."elif strategy == AggregationStrategy.WORST_PERFORMANCE:strategy_guidance = "Focus on areas that need improvement and common issues across tasks."- else: # Default/average strategies+ else:strategy_guidance = "Provide a balanced analysis of strengths and weaknesses across all tasks."prompt = [diff --git a/tests/experimental/evaluation/test_agent_evaluator.py b/tests/experimental/evaluation/test_agent_evaluator.pyindex 88f493908..3bde9c348 100644--- a/tests/experimental/evaluation/test_agent_evaluator.py+++ b/tests/experimental/evaluation/test_agent_evaluator.py@@ -42,7 +42,7 @@ class TestAgentEvaluator:agent_evaluator = AgentEvaluator()agent_evaluator.set_iteration(3)- assert agent_evaluator.iteration == 3+ assert agent_evaluator._execution_state.iteration == 3@pytest.mark.vcr(filter_headers=["authorization"])def test_evaluate_current_iteration(self, mock_crew):