Framework for orchestrating role-playing, autonomous AI agents. By fostering collaborative intelligence, CrewAI empowers agents to work together seamlessly, tackling complex tasks.
chore: remove evaluation folder (#3159)
Lucas Gomide53f674be600d
diff --git a/src/crewai/evaluation/__init__.py b/src/crewai/evaluation/__init__.pydeleted file mode 100644index 0c9f626b6..000000000--- a/src/crewai/evaluation/__init__.py+++ /dev/null@@ -1,53 +0,0 @@-from crewai.evaluation.base_evaluator import (- BaseEvaluator,- EvaluationScore,- MetricCategory,- AgentEvaluationResult-)--from crewai.evaluation.metrics.semantic_quality_metrics import (- SemanticQualityEvaluator-)--from crewai.evaluation.metrics.goal_metrics import (- GoalAlignmentEvaluator-)--from crewai.evaluation.metrics.reasoning_metrics import (- ReasoningEfficiencyEvaluator-)---from crewai.evaluation.metrics.tools_metrics import (- ToolSelectionEvaluator,- ParameterExtractionEvaluator,- ToolInvocationEvaluator-)--from crewai.evaluation.evaluation_listener import (- EvaluationTraceCallback,- create_evaluation_callbacks-)---from crewai.evaluation.agent_evaluator import (- AgentEvaluator,- create_default_evaluator-)--__all__ = [- "BaseEvaluator",- "EvaluationScore",- "MetricCategory",- "AgentEvaluationResult",- "SemanticQualityEvaluator",- "GoalAlignmentEvaluator",- "ReasoningEfficiencyEvaluator",- "ToolSelectionEvaluator",- "ParameterExtractionEvaluator",- "ToolInvocationEvaluator",- "EvaluationTraceCallback",- "create_evaluation_callbacks",- "AgentEvaluator",- "create_default_evaluator"-]\ No newline at end of filediff --git a/src/crewai/evaluation/agent_evaluator.py b/src/crewai/evaluation/agent_evaluator.pydeleted file mode 100644index 430196d14..000000000--- a/src/crewai/evaluation/agent_evaluator.py+++ /dev/null@@ -1,178 +0,0 @@-from crewai.evaluation.base_evaluator import AgentEvaluationResult, AggregationStrategy-from crewai.agent import Agent-from crewai.task import Task-from crewai.evaluation.evaluation_display import EvaluationDisplayFormatter--from typing import Any, Dict-from collections import defaultdict-from crewai.evaluation import BaseEvaluator, create_evaluation_callbacks-from collections.abc import Sequence-from crewai.crew import Crew-from crewai.utilities.events.crewai_event_bus import crewai_event_bus-from crewai.utilities.events.utils.console_formatter import ConsoleFormatter--class AgentEvaluator:- def __init__(- self,- evaluators: Sequence[BaseEvaluator] | None = None,- crew: Crew | None = None,- ):- self.crew: Crew | None = crew- self.evaluators: Sequence[BaseEvaluator] | None = evaluators-- self.agent_evaluators: dict[str, Sequence[BaseEvaluator] | None] = {}- if crew is not None:- assert crew and crew.agents is not None- for agent in crew.agents:- self.agent_evaluators[str(agent.id)] = self.evaluators-- self.callback = create_evaluation_callbacks()- self.console_formatter = ConsoleFormatter()- self.display_formatter = EvaluationDisplayFormatter()-- self.iteration = 1- self.iterations_results: dict[int, dict[str, list[AgentEvaluationResult]]] = {}-- def set_iteration(self, iteration: int) -> None:- self.iteration = iteration-- def evaluate_current_iteration(self) -> dict[str, list[AgentEvaluationResult]]:- if not self.crew:- raise ValueError("Cannot evaluate: no crew was provided to the evaluator.")-- if not self.callback:- raise ValueError("Cannot evaluate: no callback was set. Use set_callback() method first.")-- from rich.progress import Progress, SpinnerColumn, TextColumn, BarColumn- evaluation_results: defaultdict[str, list[AgentEvaluationResult]] = defaultdict(list)-- total_evals = 0- for agent in self.crew.agents:- for task in self.crew.tasks:- if task.agent and task.agent.id == agent.id and self.agent_evaluators.get(str(agent.id)):- total_evals += 1-- with Progress(- SpinnerColumn(),- TextColumn("[bold blue]{task.description}[/bold blue]"),- BarColumn(),- TextColumn("{task.percentage:.0f}% completed"),- console=self.console_formatter.console- ) as progress:- eval_task = progress.add_task(f"Evaluating agents (iteration {self.iteration})...", total=total_evals)-- for agent in self.crew.agents:- evaluator = self.agent_evaluators.get(str(agent.id))- if not evaluator:- continue-- for task in self.crew.tasks:-- if task.agent and str(task.agent.id) != str(agent.id):- continue-- trace = self.callback.get_trace(str(agent.id), str(task.id))- if not trace:- self.console_formatter.print(f"[yellow]Warning: No trace found for agent {agent.role} on task {task.description[:30]}...[/yellow]")- progress.update(eval_task, advance=1)- continue-- with crewai_event_bus.scoped_handlers():- result = self.evaluate(- agent=agent,- task=task,- execution_trace=trace,- final_output=task.output- )- evaluation_results[agent.role].append(result)- progress.update(eval_task, advance=1)-- self.iterations_results[self.iteration] = evaluation_results- return evaluation_results-- def get_evaluation_results(self):- if self.iteration in self.iterations_results:- return self.iterations_results[self.iteration]-- return self.evaluate_current_iteration()-- def display_results_with_iterations(self):- self.display_formatter.display_summary_results(self.iterations_results)-- def get_agent_evaluation(self, strategy: AggregationStrategy = AggregationStrategy.SIMPLE_AVERAGE, include_evaluation_feedback: bool = False):- agent_results = {}- with crewai_event_bus.scoped_handlers():- task_results = self.get_evaluation_results()- for agent_role, results in task_results.items():- if not results:- continue-- agent_id = results[0].agent_id-- aggregated_result = self.display_formatter._aggregate_agent_results(- agent_id=agent_id,- agent_role=agent_role,- results=results,- strategy=strategy- )-- agent_results[agent_role] = aggregated_result--- if self.iteration == max(self.iterations_results.keys()):- self.display_results_with_iterations()-- if include_evaluation_feedback:- self.display_evaluation_with_feedback()-- return agent_results-- def display_evaluation_with_feedback(self):- self.display_formatter.display_evaluation_with_feedback(self.iterations_results)-- def evaluate(- self,- agent: Agent,- task: Task,- execution_trace: Dict[str, Any],- final_output: Any- ) -> AgentEvaluationResult:- result = AgentEvaluationResult(- agent_id=str(agent.id),- task_id=str(task.id)- )- assert self.evaluators is not None- for evaluator in self.evaluators:- try:- score = evaluator.evaluate(- agent=agent,- task=task,- execution_trace=execution_trace,- final_output=final_output- )- result.metrics[evaluator.metric_category] = score- except Exception as e:- self.console_formatter.print(f"Error in {evaluator.metric_category.value} evaluator: {str(e)}")-- return result--def create_default_evaluator(crew, llm=None):- from crewai.evaluation import (- GoalAlignmentEvaluator,- SemanticQualityEvaluator,- ToolSelectionEvaluator,- ParameterExtractionEvaluator,- ToolInvocationEvaluator,- ReasoningEfficiencyEvaluator- )-- evaluators = [- GoalAlignmentEvaluator(llm=llm),- SemanticQualityEvaluator(llm=llm),- ToolSelectionEvaluator(llm=llm),- ParameterExtractionEvaluator(llm=llm),- ToolInvocationEvaluator(llm=llm),- ReasoningEfficiencyEvaluator(llm=llm),- ]-- return AgentEvaluator(evaluators=evaluators, crew=crew)diff --git a/src/crewai/evaluation/base_evaluator.py b/src/crewai/evaluation/base_evaluator.pydeleted file mode 100644index b11c61973..000000000--- a/src/crewai/evaluation/base_evaluator.py+++ /dev/null@@ -1,125 +0,0 @@-import abc-import enum-from enum import Enum-from typing import Any, Dict, List, Optional--from pydantic import BaseModel, Field--from crewai.agent import Agent-from crewai.task import Task-from crewai.llm import BaseLLM-from crewai.utilities.llm_utils import create_llm--class MetricCategory(enum.Enum):- GOAL_ALIGNMENT = "goal_alignment"- SEMANTIC_QUALITY = "semantic_quality"- REASONING_EFFICIENCY = "reasoning_efficiency"- TOOL_SELECTION = "tool_selection"- PARAMETER_EXTRACTION = "parameter_extraction"- TOOL_INVOCATION = "tool_invocation"-- def title(self):- return self.value.replace('_', ' ').title()---class EvaluationScore(BaseModel):- score: float | None = Field(- default=5.0,- description="Numeric score from 0-10 where 0 is worst and 10 is best, None if not applicable",- ge=0.0,- le=10.0- )- feedback: str = Field(- default="",- description="Detailed feedback explaining the evaluation score"- )- raw_response: str | None = Field(- default=None,- description="Raw response from the evaluator (e.g., LLM)"- )-- def __str__(self) -> str:- if self.score is None:- return f"Score: N/A - {self.feedback}"- return f"Score: {self.score:.1f}/10 - {self.feedback}"---class BaseEvaluator(abc.ABC):- def __init__(self, llm: BaseLLM | None = None):- self.llm: BaseLLM | None = create_llm(llm)-- @property- @abc.abstractmethod- def metric_category(self) -> MetricCategory:- pass-- @abc.abstractmethod- def evaluate(- self,- agent: Agent,- task: Task,- execution_trace: Dict[str, Any],- final_output: Any,- ) -> EvaluationScore:- pass---class AgentEvaluationResult(BaseModel):- agent_id: str = Field(description="ID of the evaluated agent")- task_id: str = Field(description="ID of the task that was executed")- metrics: Dict[MetricCategory, EvaluationScore] = Field(- default_factory=dict,- description="Evaluation scores for each metric category"- )---class AggregationStrategy(Enum):- SIMPLE_AVERAGE = "simple_average" # Equal weight to all tasks- WEIGHTED_BY_COMPLEXITY = "weighted_by_complexity" # Weight by task complexity- BEST_PERFORMANCE = "best_performance" # Use best scores across tasks- WORST_PERFORMANCE = "worst_performance" # Use worst scores across tasks---class AgentAggregatedEvaluationResult(BaseModel):- agent_id: str = Field(- default="",- description="ID of the agent"- )- agent_role: str = Field(- default="",- description="Role of the agent"- )- task_count: int = Field(- default=0,- description="Number of tasks included in this aggregation"- )- aggregation_strategy: AggregationStrategy = Field(- default=AggregationStrategy.SIMPLE_AVERAGE,- description="Strategy used for aggregation"- )- metrics: Dict[MetricCategory, EvaluationScore] = Field(- default_factory=dict,- description="Aggregated metrics across all tasks"- )- task_results: List[str] = Field(- default_factory=list,- description="IDs of tasks included in this aggregation"- )- overall_score: Optional[float] = Field(- default=None,- description="Overall score for this agent"- )-- def __str__(self) -> str:- result = f"Agent Evaluation: {self.agent_role}\n"- result += f"Strategy: {self.aggregation_strategy.value}\n"- result += f"Tasks evaluated: {self.task_count}\n"-- for category, score in self.metrics.items():- result += f"\n\n- {category.value.upper()}: {score.score}/10\n"-- if score.feedback:- detailed_feedback = "\n ".join(score.feedback.split('\n'))- result += f" {detailed_feedback}\n"-- return result\ No newline at end of filediff --git a/src/crewai/evaluation/evaluation_display.py b/src/crewai/evaluation/evaluation_display.pydeleted file mode 100644index 0e30c53f0..000000000--- a/src/crewai/evaluation/evaluation_display.py+++ /dev/null@@ -1,341 +0,0 @@-from collections import defaultdict-from typing import Dict, Any, List-from rich.table import Table-from rich.box import HEAVY_EDGE, ROUNDED-from collections.abc import Sequence-from crewai.evaluation.base_evaluator import AgentAggregatedEvaluationResult, AggregationStrategy, AgentEvaluationResult, MetricCategory-from crewai.evaluation import EvaluationScore-from crewai.utilities.events.utils.console_formatter import ConsoleFormatter-from crewai.utilities.llm_utils import create_llm--class EvaluationDisplayFormatter:- def __init__(self):- self.console_formatter = ConsoleFormatter()-- def display_evaluation_with_feedback(self, iterations_results: Dict[int, Dict[str, List[Any]]]):- if not iterations_results:- self.console_formatter.print("[yellow]No evaluation results to display[/yellow]")- return-- # Get all agent roles across all iterations- all_agent_roles: set[str] = set()- for iter_results in iterations_results.values():- all_agent_roles.update(iter_results.keys())-- for agent_role in sorted(all_agent_roles):- self.console_formatter.print(f"\n[bold cyan]Agent: {agent_role}[/bold cyan]")-- # Process each iteration- for iter_num, results in sorted(iterations_results.items()):- if agent_role not in results or not results[agent_role]:- continue-- agent_results = results[agent_role]- agent_id = agent_results[0].agent_id-- # Aggregate results for this agent in this iteration- aggregated_result = self._aggregate_agent_results(- agent_id=agent_id,- agent_role=agent_role,- results=agent_results,- )-- # Display iteration header- self.console_formatter.print(f"\n[bold]Iteration {iter_num}[/bold]")-- # Create table for this iteration- table = Table(box=ROUNDED)- table.add_column("Metric", style="cyan")- table.add_column("Score (1-10)", justify="center")- table.add_column("Feedback", style="green")-- # Add metrics to table- if aggregated_result.metrics:- for metric, evaluation_score in aggregated_result.metrics.items():- score = evaluation_score.score-- if isinstance(score, (int, float)):- if score >= 8.0:- score_text = f"[green]{score:.1f}[/green]"- elif score >= 6.0:- score_text = f"[cyan]{score:.1f}[/cyan]"- elif score >= 4.0:- score_text = f"[yellow]{score:.1f}[/yellow]"- else:- score_text = f"[red]{score:.1f}[/red]"- else:- score_text = "[dim]N/A[/dim]"-- table.add_section()- table.add_row(- metric.title(),- score_text,- evaluation_score.feedback or ""- )-- if aggregated_result.overall_score is not None:- overall_score = aggregated_result.overall_score- if overall_score >= 8.0:- overall_color = "green"- elif overall_score >= 6.0:- overall_color = "cyan"- elif overall_score >= 4.0:- overall_color = "yellow"- else:- overall_color = "red"-- table.add_section()- table.add_row(- "Overall Score",- f"[{overall_color}]{overall_score:.1f}[/]",- "Overall agent evaluation score"- )-- # Print the table for this iteration- self.console_formatter.print(table)-- def display_summary_results(self, iterations_results: Dict[int, Dict[str, List[AgentAggregatedEvaluationResult]]]):- if not iterations_results:- self.console_formatter.print("[yellow]No evaluation results to display[/yellow]")- return-- self.console_formatter.print("\n")-- table = Table(title="Agent Performance Scores \n (1-10 Higher is better)", box=HEAVY_EDGE)-- table.add_column("Agent/Metric", style="cyan")-- for iter_num in sorted(iterations_results.keys()):- run_label = f"Run {iter_num}"- table.add_column(run_label, justify="center")-- table.add_column("Avg. Total", justify="center")-- all_agent_roles: set[str] = set()- for results in iterations_results.values():- all_agent_roles.update(results.keys())-- for agent_role in sorted(all_agent_roles):- agent_scores_by_iteration = {}- agent_metrics_by_iteration = {}-- for iter_num, results in sorted(iterations_results.items()):- if agent_role not in results or not results[agent_role]:- continue-- agent_results = results[agent_role]- agent_id = agent_results[0].agent_id-- aggregated_result = self._aggregate_agent_results(- agent_id=agent_id,- agent_role=agent_role,- results=agent_results,- strategy=AggregationStrategy.SIMPLE_AVERAGE- )-- valid_scores = [score.score for score in aggregated_result.metrics.values()- if score.score is not None]- if valid_scores:- avg_score = sum(valid_scores) / len(valid_scores)- agent_scores_by_iteration[iter_num] = avg_score-- agent_metrics_by_iteration[iter_num] = aggregated_result.metrics-- if not agent_scores_by_iteration:- continue-- avg_across_iterations = sum(agent_scores_by_iteration.values()) / len(agent_scores_by_iteration)-- row = [f"[bold]{agent_role}[/bold]"]-- for iter_num in sorted(iterations_results.keys()):- if iter_num in agent_scores_by_iteration:- score = agent_scores_by_iteration[iter_num]- if score >= 8.0:- color = "green"- elif score >= 6.0:- color = "cyan"- elif score >= 4.0:- color = "yellow"- else:- color = "red"- row.append(f"[bold {color}]{score:.1f}[/]")- else:- row.append("-")-- if avg_across_iterations >= 8.0:- color = "green"- elif avg_across_iterations >= 6.0:- color = "cyan"- elif avg_across_iterations >= 4.0:- color = "yellow"- else:- color = "red"- row.append(f"[bold {color}]{avg_across_iterations:.1f}[/]")-- table.add_row(*row)-- all_metrics: set[Any] = set()- for metrics in agent_metrics_by_iteration.values():- all_metrics.update(metrics.keys())-- for metric in sorted(all_metrics, key=lambda x: x.value):- metric_scores = []-- row = [f" - {metric.title()}"]-- for iter_num in sorted(iterations_results.keys()):- if (iter_num in agent_metrics_by_iteration and- metric in agent_metrics_by_iteration[iter_num]):- metric_score = agent_metrics_by_iteration[iter_num][metric].score- if metric_score is not None:- metric_scores.append(metric_score)- if metric_score >= 8.0:- color = "green"- elif metric_score >= 6.0:- color = "cyan"- elif metric_score >= 4.0:- color = "yellow"- else:- color = "red"- row.append(f"[{color}]{metric_score:.1f}[/]")- else:- row.append("[dim]N/A[/dim]")- else:- row.append("-")-- if metric_scores:- avg = sum(metric_scores) / len(metric_scores)- if avg >= 8.0:- color = "green"- elif avg >= 6.0:- color = "cyan"- elif avg >= 4.0:- color = "yellow"- else:- color = "red"- row.append(f"[{color}]{avg:.1f}[/]")- else:- row.append("-")-- table.add_row(*row)-- table.add_row(*[""] * (len(sorted(iterations_results.keys())) + 2))-- self.console_formatter.print(table)- self.console_formatter.print("\n")-- def _aggregate_agent_results(- self,- agent_id: str,- agent_role: str,- results: Sequence[AgentEvaluationResult],- strategy: AggregationStrategy = AggregationStrategy.SIMPLE_AVERAGE,- ) -> AgentAggregatedEvaluationResult:- metrics_by_category: dict[MetricCategory, list[EvaluationScore]] = defaultdict(list)-- for result in results:- for metric_name, evaluation_score in result.metrics.items():- metrics_by_category[metric_name].append(evaluation_score)-- aggregated_metrics: dict[MetricCategory, EvaluationScore] = {}- for category, scores in metrics_by_category.items():- valid_scores = [s.score for s in scores if s.score is not None]- avg_score = sum(valid_scores) / len(valid_scores) if valid_scores else None-- feedbacks = [s.feedback for s in scores if s.feedback]-- feedback_summary = None- if feedbacks:- if len(feedbacks) > 1:- # Use the summarization method for multiple feedbacks- feedback_summary = self._summarize_feedbacks(- agent_role=agent_role,- metric=category.title(),- feedbacks=feedbacks,- scores=[s.score for s in scores],- strategy=strategy- )- else:- feedback_summary = feedbacks[0]-- aggregated_metrics[category] = EvaluationScore(- score=avg_score,- feedback=feedback_summary- )-- overall_score = None- if aggregated_metrics:- valid_scores = [m.score for m in aggregated_metrics.values() if m.score is not None]- if valid_scores:- overall_score = sum(valid_scores) / len(valid_scores)-- return AgentAggregatedEvaluationResult(- agent_id=agent_id,- agent_role=agent_role,- metrics=aggregated_metrics,- overall_score=overall_score,- task_count=len(results),- aggregation_strategy=strategy- )-- def _summarize_feedbacks(- self,- agent_role: str,- metric: str,- feedbacks: List[str],- scores: List[float | None],- strategy: AggregationStrategy- ) -> str:- if len(feedbacks) <= 2 and all(len(fb) < 200 for fb in feedbacks):- return "\n\n".join([f"Feedback {i+1}: {fb}" for i, fb in enumerate(feedbacks)])-- try:- llm = create_llm()-- formatted_feedbacks = []- for i, (feedback, score) in enumerate(zip(feedbacks, scores)):- if len(feedback) > 500:- feedback = feedback[:500] + "..."- score_text = f"{score:.1f}" if score is not None else "N/A"- formatted_feedbacks.append(f"Feedback #{i+1} (Score: {score_text}):\n{feedback}")-- all_feedbacks = "\n\n" + "\n\n---\n\n".join(formatted_feedbacks)-- strategy_guidance = ""- if strategy == AggregationStrategy.BEST_PERFORMANCE:- strategy_guidance = "Focus on the highest-scoring aspects and strengths demonstrated."- elif strategy == AggregationStrategy.WORST_PERFORMANCE:- strategy_guidance = "Focus on areas that need improvement and common issues across tasks."- else: # Default/average strategies- strategy_guidance = "Provide a balanced analysis of strengths and weaknesses across all tasks."-- prompt = [- {"role": "system", "content": f"""You are an expert evaluator creating a comprehensive summary of agent performance feedback.- Your job is to synthesize multiple feedback points about the same metric across different tasks.-- Create a concise, insightful summary that captures the key patterns and themes from all feedback.- {strategy_guidance}-- Your summary should be:- 1. Specific and concrete (not vague or general)- 2. Focused on actionable insights- 3. Highlighting patterns across tasks- 4. 150-250 words in length-- The summary should be directly usable as final feedback for the agent's performance on this metric."""},- {"role": "user", "content": f"""I need a synthesized summary of the following feedback for:-- Agent Role: {agent_role}- Metric: {metric.title()}-- {all_feedbacks}- """}- ]- assert llm is not None- response = llm.call(prompt)-- return response-- except Exception:- return "Synthesized from multiple tasks: " + "\n\n".join([f"- {fb[:500]}..." for fb in feedbacks])diff --git a/src/crewai/evaluation/evaluation_listener.py b/src/crewai/evaluation/evaluation_listener.pydeleted file mode 100644index 8fba03e14..000000000--- a/src/crewai/evaluation/evaluation_listener.py+++ /dev/null@@ -1,190 +0,0 @@-from datetime import datetime-from typing import Any, Dict, Optional--from collections.abc import Sequence--from crewai.agent import Agent-from crewai.task import Task-from crewai.utilities.events.base_event_listener import BaseEventListener-from crewai.utilities.events.crewai_event_bus import CrewAIEventsBus-from crewai.utilities.events.agent_events import (- AgentExecutionStartedEvent,- AgentExecutionCompletedEvent-)-from crewai.utilities.events.tool_usage_events import (- ToolUsageFinishedEvent,- ToolUsageErrorEvent,- ToolExecutionErrorEvent,- ToolSelectionErrorEvent,- ToolValidateInputErrorEvent-)-from crewai.utilities.events.llm_events import (- LLMCallStartedEvent,- LLMCallCompletedEvent-)--class EvaluationTraceCallback(BaseEventListener):- """Event listener for collecting execution traces for evaluation.-- This listener attaches to the event bus to collect detailed information- about the execution process, including agent steps, tool uses, knowledge- retrievals, and final output - all for use in agent evaluation.- """-- _instance = None-- def __new__(cls):- if cls._instance is None:- cls._instance = super().__new__(cls)- cls._instance._initialized = False- return cls._instance-- def __init__(self):- if not hasattr(self, "_initialized") or not self._initialized:- super().__init__()- self.traces = {}- self.current_agent_id = None- self.current_task_id = None- self._initialized = True-- def setup_listeners(self, event_bus: CrewAIEventsBus):- @event_bus.on(AgentExecutionStartedEvent)- def on_agent_started(source, event: AgentExecutionStartedEvent):- self.on_agent_start(event.agent, event.task)-- @event_bus.on(AgentExecutionCompletedEvent)- def on_agent_completed(source, event: AgentExecutionCompletedEvent):- self.on_agent_finish(event.agent, event.task, event.output)-- @event_bus.on(ToolUsageFinishedEvent)- def on_tool_completed(source, event: ToolUsageFinishedEvent):- self.on_tool_use(event.tool_name, event.tool_args, event.output, success=True)-- @event_bus.on(ToolUsageErrorEvent)- def on_tool_usage_error(source, event: ToolUsageErrorEvent):- self.on_tool_use(event.tool_name, event.tool_args, event.error,- success=False, error_type="usage_error")-- @event_bus.on(ToolExecutionErrorEvent)- def on_tool_execution_error(source, event: ToolExecutionErrorEvent):- self.on_tool_use(event.tool_name, event.tool_args, event.error,- success=False, error_type="execution_error")-- @event_bus.on(ToolSelectionErrorEvent)- def on_tool_selection_error(source, event: ToolSelectionErrorEvent):- self.on_tool_use(event.tool_name, event.tool_args, event.error,- success=False, error_type="selection_error")-- @event_bus.on(ToolValidateInputErrorEvent)- def on_tool_validate_input_error(source, event: ToolValidateInputErrorEvent):- self.on_tool_use(event.tool_name, event.tool_args, event.error,- success=False, error_type="validation_error")-- @event_bus.on(LLMCallStartedEvent)- def on_llm_call_started(source, event: LLMCallStartedEvent):- self.on_llm_call_start(event.messages, event.tools)-- @event_bus.on(LLMCallCompletedEvent)- def on_llm_call_completed(source, event: LLMCallCompletedEvent):- self.on_llm_call_end(event.messages, event.response)-- def on_agent_start(self, agent: Agent, task: Task):- self.current_agent_id = agent.id- self.current_task_id = task.id-- trace_key = f"{agent.id}_{task.id}"- self.traces[trace_key] = {- "agent_id": agent.id,- "task_id": task.id,- "tool_uses": [],- "llm_calls": [],- "start_time": datetime.now(),- "final_output": None- }-- def on_agent_finish(self, agent: Agent, task: Task, output: Any):- trace_key = f"{agent.id}_{task.id}"- if trace_key in self.traces:- self.traces[trace_key]["final_output"] = output- self.traces[trace_key]["end_time"] = datetime.now()-- self.current_agent_id = None- self.current_task_id = None-- def on_tool_use(self, tool_name: str, tool_args: dict[str, Any] | str, result: Any,- success: bool = True, error_type: str | None = None):- if not self.current_agent_id or not self.current_task_id:- return-- trace_key = f"{self.current_agent_id}_{self.current_task_id}"- if trace_key in self.traces:- tool_use = {- "tool": tool_name,- "args": tool_args,- "result": result,- "success": success,- "timestamp": datetime.now()- }-- # Add error information if applicable- if not success and error_type:- tool_use["error"] = True- tool_use["error_type"] = error_type-- self.traces[trace_key]["tool_uses"].append(tool_use)-- def on_llm_call_start(self, messages: str | Sequence[dict[str, Any]] | None, tools: Sequence[dict[str, Any]] | None = None):- if not self.current_agent_id or not self.current_task_id:- return-- trace_key = f"{self.current_agent_id}_{self.current_task_id}"- if trace_key not in self.traces:- return-- self.current_llm_call = {- "messages": messages,- "tools": tools,- "start_time": datetime.now(),- "response": None,- "end_time": None- }-- def on_llm_call_end(self, messages: str | list[dict[str, Any]] | None, response: Any):- if not self.current_agent_id or not self.current_task_id:- return-- trace_key = f"{self.current_agent_id}_{self.current_task_id}"- if trace_key not in self.traces:- return-- total_tokens = 0- if hasattr(response, "usage") and hasattr(response.usage, "total_tokens"):- total_tokens = response.usage.total_tokens-- current_time = datetime.now()- start_time = None- if hasattr(self, "current_llm_call") and self.current_llm_call:- start_time = self.current_llm_call.get("start_time")-- if not start_time:- start_time = current_time- llm_call = {- "messages": messages,- "response": response,- "start_time": start_time,- "end_time": current_time,- "total_tokens": total_tokens- }-- self.traces[trace_key]["llm_calls"].append(llm_call)-- if hasattr(self, "current_llm_call"):- self.current_llm_call = {}-- def get_trace(self, agent_id: str, task_id: str) -> Optional[Dict[str, Any]]:- trace_key = f"{agent_id}_{task_id}"- return self.traces.get(trace_key)---def create_evaluation_callbacks() -> EvaluationTraceCallback:- return EvaluationTraceCallback()\ No newline at end of filediff --git a/src/crewai/evaluation/experiment/testing.py b/src/crewai/evaluation/experiment/testing.pydeleted file mode 100644index 11de59a80..000000000--- a/src/crewai/evaluation/experiment/testing.py+++ /dev/null@@ -1,49 +0,0 @@-import warnings-from crewai.experimental.evaluation import ExperimentResults--def assert_experiment_successfully(experiment_results: ExperimentResults) -> None:- """- Assert that all experiment results passed successfully.-- Args:- experiment_results: The experiment results to check-- Raises:- AssertionError: If any test case failed- """- failed_tests = [result for result in experiment_results.results if not result.passed]-- if failed_tests:- detailed_failures: list[str] = []-- for result in failed_tests:- expected = result.expected_score- actual = result.score- detailed_failures.append(f"- {result.identifier}: expected {expected}, got {actual}")-- failure_details = "\n".join(detailed_failures)- raise AssertionError(f"The following test cases failed:\n{failure_details}")--def assert_experiment_no_regression(comparison_result: dict[str, list[str]]) -> None:- """- Assert that there are no regressions in the experiment results compared to baseline.- Also warns if there are missing tests.-- Args:- comparison_result: The result from compare_with_baseline()-- Raises:- AssertionError: If there are regressions- """- # Check for regressions- regressed = comparison_result.get("regressed", [])- if regressed:- raise AssertionError(f"Regression detected! The following tests that previously passed now fail: {regressed}")-- # Check for missing tests and warn- missing_tests = comparison_result.get("missing_tests", [])- if missing_tests:- warnings.warn(- f"Warning: {len(missing_tests)} tests from the baseline are missing in the current run: {missing_tests}",- UserWarning- )\ No newline at end of filediff --git a/src/crewai/evaluation/json_parser.py b/src/crewai/evaluation/json_parser.pydeleted file mode 100644index ce7303cde..000000000--- a/src/crewai/evaluation/json_parser.py+++ /dev/null@@ -1,30 +0,0 @@-"""Robust JSON parsing utilities for evaluation responses."""--import json-import re-from typing import Any---def extract_json_from_llm_response(text: str) -> dict[str, Any]:- try:- return json.loads(text)- except json.JSONDecodeError:- pass-- json_patterns = [- # Standard markdown code blocks with json- r'```json\s*([\s\S]*?)\s*```',- # Code blocks without language specifier- r'```\s*([\s\S]*?)\s*```',- # Inline code with JSON- r'`([{\\[].*[}\]])`',- ]-- for pattern in json_patterns:- matches = re.findall(pattern, text, re.IGNORECASE | re.DOTALL)- for match in matches:- try:- return json.loads(match.strip())- except json.JSONDecodeError:- continue- raise ValueError("No valid JSON found in the response")diff --git a/src/crewai/evaluation/metrics/__init__.py b/src/crewai/evaluation/metrics/__init__.pydeleted file mode 100644index e69de29bb..000000000diff --git a/src/crewai/evaluation/metrics/goal_metrics.py b/src/crewai/evaluation/metrics/goal_metrics.pydeleted file mode 100644index bc6c63801..000000000--- a/src/crewai/evaluation/metrics/goal_metrics.py+++ /dev/null@@ -1,66 +0,0 @@-from typing import Any, Dict--from crewai.agent import Agent-from crewai.task import Task--from crewai.evaluation.base_evaluator import BaseEvaluator, EvaluationScore, MetricCategory-from crewai.evaluation.json_parser import extract_json_from_llm_response--class GoalAlignmentEvaluator(BaseEvaluator):- @property- def metric_category(self) -> MetricCategory:- return MetricCategory.GOAL_ALIGNMENT-- def evaluate(- self,- agent: Agent,- task: Task,- execution_trace: Dict[str, Any],- final_output: Any,- ) -> EvaluationScore:- prompt = [- {"role": "system", "content": """You are an expert evaluator assessing how well an AI agent's output aligns with its assigned task goal.--Score the agent's goal alignment on a scale from 0-10 where:-- 0: Complete misalignment, agent did not understand or attempt the task goal-- 5: Partial alignment, agent attempted the task but missed key requirements-- 10: Perfect alignment, agent fully satisfied all task requirements--Consider:-1. Did the agent correctly interpret the task goal?-2. Did the final output directly address the requirements?-3. Did the agent focus on relevant aspects of the task?-4. Did the agent provide all requested information or deliverables?--Return your evaluation as JSON with fields 'score' (number) and 'feedback' (string).-"""},- {"role": "user", "content": f"""-Agent role: {agent.role}-Agent goal: {agent.goal}-Task description: {task.description}-Expected output: {task.expected_output}--Agent's final output:-{final_output}--Evaluate how well the agent's output aligns with the assigned task goal.-"""}- ]- assert self.llm is not None- response = self.llm.call(prompt)-- try:- evaluation_data: dict[str, Any] = extract_json_from_llm_response(response)- assert evaluation_data is not None-- return EvaluationScore(- score=evaluation_data.get("score", 0),- feedback=evaluation_data.get("feedback", response),- raw_response=response- )- except Exception:- return EvaluationScore(- score=None,- feedback=f"Failed to parse evaluation. Raw response: {response}",- raw_response=response- )diff --git a/src/crewai/evaluation/metrics/reasoning_metrics.py b/src/crewai/evaluation/metrics/reasoning_metrics.pydeleted file mode 100644index e1ce06c23..000000000--- a/src/crewai/evaluation/metrics/reasoning_metrics.py+++ /dev/null@@ -1,355 +0,0 @@-"""Agent reasoning efficiency evaluators.--This module provides evaluator implementations for:-- Reasoning efficiency-- Loop detection-- Thinking-to-action ratio-"""--import logging-import re-from enum import Enum-from typing import Any, Dict, List, Tuple-import numpy as np-from collections.abc import Sequence--from crewai.agent import Agent-from crewai.task import Task--from crewai.evaluation.base_evaluator import BaseEvaluator, EvaluationScore, MetricCategory-from crewai.evaluation.json_parser import extract_json_from_llm_response-from crewai.tasks.task_output import TaskOutput--class ReasoningPatternType(Enum):- EFFICIENT = "efficient" # Good reasoning flow- LOOP = "loop" # Agent is stuck in a loop- VERBOSE = "verbose" # Agent is unnecessarily verbose- INDECISIVE = "indecisive" # Agent struggles to make decisions- SCATTERED = "scattered" # Agent jumps between topics without focus---class ReasoningEfficiencyEvaluator(BaseEvaluator):- @property- def metric_category(self) -> MetricCategory:- return MetricCategory.REASONING_EFFICIENCY-- def evaluate(- self,- agent: Agent,- task: Task,- execution_trace: Dict[str, Any],- final_output: TaskOutput,- ) -> EvaluationScore:- llm_calls = execution_trace.get("llm_calls", [])-- if not llm_calls or len(llm_calls) < 2:- return EvaluationScore(- score=None,- feedback="Insufficient LLM calls to evaluate reasoning efficiency."- )-- total_calls = len(llm_calls)- total_tokens = sum(call.get("total_tokens", 0) for call in llm_calls)- avg_tokens_per_call = total_tokens / total_calls if total_calls > 0 else 0- time_intervals = []- has_reliable_timing = True- for i in range(1, len(llm_calls)):- start_time = llm_calls[i-1].get("end_time")- end_time = llm_calls[i].get("start_time")- if start_time and end_time and start_time != end_time:- try:- interval = end_time - start_time- time_intervals.append(interval.total_seconds() if hasattr(interval, 'total_seconds') else 0)- except Exception:- has_reliable_timing = False- else:- has_reliable_timing = False-- loop_detected, loop_details = self._detect_loops(llm_calls)- pattern_analysis = self._analyze_reasoning_patterns(llm_calls)-- efficiency_metrics = {- "total_llm_calls": total_calls,- "total_tokens": total_tokens,- "avg_tokens_per_call": avg_tokens_per_call,- "reasoning_pattern": pattern_analysis["primary_pattern"].value,- "loops_detected": loop_detected,- }-- if has_reliable_timing and time_intervals:- efficiency_metrics["avg_time_between_calls"] = np.mean(time_intervals)-- loop_info = f"Detected {len(loop_details)} potential reasoning loops." if loop_detected else "No significant reasoning loops detected."-- call_samples = self._get_call_samples(llm_calls)-- prompt = [- {"role": "system", "content": """You are an expert evaluator assessing the reasoning efficiency of an AI agent's thought process.--Evaluate the agent's reasoning efficiency across these five key subcategories:--1. Focus (0-10): How well the agent stays on topic and avoids unnecessary tangents-2. Progression (0-10): How effectively the agent builds on previous thoughts rather than repeating or circling-3. Decision Quality (0-10): How decisively and appropriately the agent makes decisions-4. Conciseness (0-10): How efficiently the agent communicates without unnecessary verbosity-5. Loop Avoidance (0-10): How well the agent avoids getting stuck in repetitive thinking patterns--For each subcategory, provide a score from 0-10 where:-- 0: Completely inefficient-- 5: Moderately efficient-- 10: Highly efficient--The overall score should be a weighted average of these subcategories.--Return your evaluation as JSON with the following structure:-{- "overall_score": float,- "scores": {- "focus": float,- "progression": float,- "decision_quality": float,- "conciseness": float,- "loop_avoidance": float- },- "feedback": string (general feedback about overall reasoning efficiency),- "optimization_suggestions": string (concrete suggestions for improving reasoning efficiency),- "detected_patterns": string (describe any inefficient reasoning patterns you observe)-}"""},- {"role": "user", "content": f"""-Agent role: {agent.role}-Task description: {task.description}--Reasoning efficiency metrics:-- Total LLM calls: {efficiency_metrics["total_llm_calls"]}-- Average tokens per call: {efficiency_metrics["avg_tokens_per_call"]:.1f}-- Primary reasoning pattern: {efficiency_metrics["reasoning_pattern"]}-- {loop_info}-{"- Average time between calls: {:.2f} seconds".format(efficiency_metrics.get("avg_time_between_calls", 0)) if "avg_time_between_calls" in efficiency_metrics else ""}--Sample of agent reasoning flow (chronological sequence):-{call_samples}--Agent's final output:-{final_output.raw[:500]}... (truncated)--Evaluate the reasoning efficiency of this agent based on these interaction patterns.-Identify any inefficient reasoning patterns and provide specific suggestions for optimization.-"""}- ]-- assert self.llm is not None- response = self.llm.call(prompt)-- try:- evaluation_data = extract_json_from_llm_response(response)-- scores = evaluation_data.get("scores", {})- focus = scores.get("focus", 5.0)- progression = scores.get("progression", 5.0)- decision_quality = scores.get("decision_quality", 5.0)- conciseness = scores.get("conciseness", 5.0)- loop_avoidance = scores.get("loop_avoidance", 5.0)-- overall_score = evaluation_data.get("overall_score", evaluation_data.get("score", 5.0))- feedback = evaluation_data.get("feedback", "No detailed feedback provided.")- optimization_suggestions = evaluation_data.get("optimization_suggestions", "No specific suggestions provided.")-- detailed_feedback = "Reasoning Efficiency Evaluation:\n"- detailed_feedback += f"• Focus: {focus}/10 - Staying on topic without tangents\n"- detailed_feedback += f"• Progression: {progression}/10 - Building on previous thinking\n"- detailed_feedback += f"• Decision Quality: {decision_quality}/10 - Making appropriate decisions\n"- detailed_feedback += f"• Conciseness: {conciseness}/10 - Communicating efficiently\n"- detailed_feedback += f"• Loop Avoidance: {loop_avoidance}/10 - Avoiding repetitive patterns\n\n"-- detailed_feedback += f"Feedback:\n{feedback}\n\n"- detailed_feedback += f"Optimization Suggestions:\n{optimization_suggestions}"-- return EvaluationScore(- score=float(overall_score),- feedback=detailed_feedback,- raw_response=response- )- except Exception as e:- logging.warning(f"Failed to parse reasoning efficiency evaluation: {e}")- return EvaluationScore(- score=None,- feedback=f"Failed to parse reasoning efficiency evaluation. Raw response: {response[:200]}...",- raw_response=response- )-- def _detect_loops(self, llm_calls: List[Dict]) -> Tuple[bool, List[Dict]]:- loop_details = []-- messages = []- for call in llm_calls:- content = call.get("response", "")- if isinstance(content, str):- messages.append(content)- elif isinstance(content, list) and len(content) > 0:- # Handle message list format- for msg in content:- if isinstance(msg, dict) and "content" in msg:- messages.append(msg["content"])-- # Simple n-gram based similarity detection- # For a more robust implementation, consider using embedding-based similarity- for i in range(len(messages) - 2):- for j in range(i + 1, len(messages) - 1):- # Check for repeated patterns (simplistic approach)- # A more sophisticated approach would use semantic similarity- similarity = self._calculate_text_similarity(messages[i], messages[j])- if similarity > 0.7: # Arbitrary threshold- loop_details.append({- "first_occurrence": i,- "second_occurrence": j,- "similarity": similarity,- "snippet": messages[i][:100] + "..."- })-- return len(loop_details) > 0, loop_details-- def _calculate_text_similarity(self, text1: str, text2: str) -> float:- text1 = re.sub(r'\s+', ' ', text1.lower()).strip()- text2 = re.sub(r'\s+', ' ', text2.lower()).strip()-- # Simple Jaccard similarity on word sets- words1 = set(text1.split())- words2 = set(text2.split())-- intersection = len(words1.intersection(words2))- union = len(words1.union(words2))-- return intersection / union if union > 0 else 0.0-- def _analyze_reasoning_patterns(self, llm_calls: List[Dict]) -> Dict[str, Any]:- call_lengths = []- response_times = []-- for call in llm_calls:- content = call.get("response", "")- if isinstance(content, str):- call_lengths.append(len(content))- elif isinstance(content, list) and len(content) > 0:- # Handle message list format- total_length = 0- for msg in content:- if isinstance(msg, dict) and "content" in msg:- total_length += len(msg["content"])- call_lengths.append(total_length)-- start_time = call.get("start_time")- end_time = call.get("end_time")- if start_time and end_time:- try:- response_times.append(end_time - start_time)- except Exception:- pass-- avg_length = np.mean(call_lengths) if call_lengths else 0- std_length = np.std(call_lengths) if call_lengths else 0- length_trend = self._calculate_trend(call_lengths)-- primary_pattern = ReasoningPatternType.EFFICIENT- details = "Agent demonstrates efficient reasoning patterns."-- loop_score = self._calculate_loop_likelihood(call_lengths, response_times)- if loop_score > 0.7:- primary_pattern = ReasoningPatternType.LOOP- details = "Agent appears to be stuck in repetitive thinking patterns."- elif avg_length > 1000 and std_length / avg_length < 0.3:- primary_pattern = ReasoningPatternType.VERBOSE- details = "Agent is consistently verbose across interactions."- elif len(llm_calls) > 10 and length_trend > 0.5:- primary_pattern = ReasoningPatternType.INDECISIVE- details = "Agent shows signs of indecisiveness with increasing message lengths."- elif std_length / avg_length > 0.8:- primary_pattern = ReasoningPatternType.SCATTERED- details = "Agent shows inconsistent reasoning flow with highly variable responses."-- return {- "primary_pattern": primary_pattern,- "details": details,- "metrics": {- "avg_length": avg_length,- "std_length": std_length,- "length_trend": length_trend,- "loop_score": loop_score- }- }-- def _calculate_trend(self, values: Sequence[float | int]) -> float:- if not values or len(values) < 2:- return 0.0-- try:- x = np.arange(len(values))- y = np.array(values)-- # Simple linear regression- slope = np.polyfit(x, y, 1)[0]-- # Normalize slope to -1 to 1 range- max_possible_slope = max(values) - min(values)- if max_possible_slope > 0:- normalized_slope = slope / max_possible_slope- return max(min(normalized_slope, 1.0), -1.0)- return 0.0- except Exception:- return 0.0-- def _calculate_loop_likelihood(self, call_lengths: Sequence[float], response_times: Sequence[float]) -> float:- if not call_lengths or len(call_lengths) < 3:- return 0.0-- indicators = []-- if len(call_lengths) >= 4:- repeated_lengths = 0- for i in range(len(call_lengths) - 2):- ratio = call_lengths[i] / call_lengths[i + 2] if call_lengths[i + 2] > 0 else 0- if 0.85 <= ratio <= 1.15:- repeated_lengths += 1-- length_repetition_score = repeated_lengths / (len(call_lengths) - 2)- indicators.append(length_repetition_score)-- if response_times and len(response_times) >= 3:- try:- std_time = np.std(response_times)- mean_time = np.mean(response_times)- if mean_time > 0:- time_consistency = 1.0 - (std_time / mean_time)- indicators.append(max(0, time_consistency - 0.3) * 1.5)- except Exception:- pass-- return np.mean(indicators) if indicators else 0.0-- def _get_call_samples(self, llm_calls: List[Dict]) -> str:- samples = []-- if len(llm_calls) <= 6:- sample_indices = list(range(len(llm_calls)))- else:- sample_indices = [0, 1, len(llm_calls) // 2 - 1, len(llm_calls) // 2,- len(llm_calls) - 2, len(llm_calls) - 1]-- for idx in sample_indices:- call = llm_calls[idx]- content = call.get("response", "")-- if isinstance(content, str):- sample = content- elif isinstance(content, list) and len(content) > 0:- sample_parts = []- for msg in content:- if isinstance(msg, dict) and "content" in msg:- sample_parts.append(msg["content"])- sample = "\n".join(sample_parts)- else:- sample = str(content)-- truncated = sample[:200] + "..." if len(sample) > 200 else sample- samples.append(f"Call {idx + 1}:\n{truncated}\n")-- return "\n".join(samples)diff --git a/src/crewai/evaluation/metrics/semantic_quality_metrics.py b/src/crewai/evaluation/metrics/semantic_quality_metrics.pydeleted file mode 100644index a12c62ae3..000000000--- a/src/crewai/evaluation/metrics/semantic_quality_metrics.py+++ /dev/null@@ -1,65 +0,0 @@-from typing import Any, Dict--from crewai.agent import Agent-from crewai.task import Task--from crewai.evaluation.base_evaluator import BaseEvaluator, EvaluationScore, MetricCategory-from crewai.evaluation.json_parser import extract_json_from_llm_response--class SemanticQualityEvaluator(BaseEvaluator):- @property- def metric_category(self) -> MetricCategory:- return MetricCategory.SEMANTIC_QUALITY-- def evaluate(- self,- agent: Agent,- task: Task,- execution_trace: Dict[str, Any],- final_output: Any,- ) -> EvaluationScore:- prompt = [- {"role": "system", "content": """You are an expert evaluator assessing the semantic quality of an AI agent's output.--Score the semantic quality on a scale from 0-10 where:-- 0: Completely incoherent, confusing, or logically flawed output-- 5: Moderately clear and logical output with some issues-- 10: Exceptionally clear, coherent, and logically sound output--Consider:-1. Is the output well-structured and organized?-2. Is the reasoning logical and well-supported?-3. Is the language clear, precise, and appropriate for the task?-4. Are claims supported by evidence when appropriate?-5. Is the output free from contradictions and logical fallacies?--Return your evaluation as JSON with fields 'score' (number) and 'feedback' (string).-"""},- {"role": "user", "content": f"""-Agent role: {agent.role}-Task description: {task.description}--Agent's final output:-{final_output}--Evaluate the semantic quality and reasoning of this output.-"""}- ]-- assert self.llm is not None- response = self.llm.call(prompt)-- try:- evaluation_data: dict[str, Any] = extract_json_from_llm_response(response)- assert evaluation_data is not None- return EvaluationScore(- score=float(evaluation_data["score"]) if evaluation_data.get("score") is not None else None,- feedback=evaluation_data.get("feedback", response),- raw_response=response- )- except Exception:- return EvaluationScore(- score=None,- feedback=f"Failed to parse evaluation. Raw response: {response}",- raw_response=response- )\ No newline at end of filediff --git a/src/crewai/evaluation/metrics/tools_metrics.py b/src/crewai/evaluation/metrics/tools_metrics.pydeleted file mode 100644index 00762fc76..000000000--- a/src/crewai/evaluation/metrics/tools_metrics.py+++ /dev/null@@ -1,400 +0,0 @@-import json-from typing import Dict, Any--from crewai.evaluation.base_evaluator import BaseEvaluator, EvaluationScore, MetricCategory-from crewai.evaluation.json_parser import extract_json_from_llm_response-from crewai.agent import Agent-from crewai.task import Task---class ToolSelectionEvaluator(BaseEvaluator):-- @property- def metric_category(self) -> MetricCategory:- return MetricCategory.TOOL_SELECTION-- def evaluate(- self,- agent: Agent,- task: Task,- execution_trace: Dict[str, Any],- final_output: str,- ) -> EvaluationScore:- tool_uses = execution_trace.get("tool_uses", [])- tool_count = len(tool_uses)- unique_tool_types = set([tool.get("tool", "Unknown tool") for tool in tool_uses])-- if tool_count == 0:- if not agent.tools:- return EvaluationScore(- score=None,- feedback="Agent had no tools available to use."- )- else:- return EvaluationScore(- score=None,- feedback="Agent had tools available but didn't use any."- )-- available_tools_info = ""- if agent.tools:- for tool in agent.tools:- available_tools_info += f"- {tool.name}: {tool.description}\n"- else:- available_tools_info = "No tools available"-- tool_types_summary = "Tools selected by the agent:\n"- for tool_type in sorted(unique_tool_types):- tool_types_summary += f"- {tool_type}\n"-- prompt = [- {"role": "system", "content": """You are an expert evaluator assessing if an AI agent selected the most appropriate tools for a given task.--You must evaluate based on these 2 criteria:-1. Relevance (0-10): Were the tools chosen directly aligned with the task's goals?-2. Coverage (0-10): Did the agent select ALL appropriate tools from the AVAILABLE tools?--IMPORTANT:-- ONLY consider tools that are listed as available to the agent-- DO NOT suggest tools that aren't in the 'Available tools' list-- DO NOT evaluate the quality or accuracy of tool outputs/results-- DO NOT evaluate how many times each tool was used-- DO NOT evaluate how the agent used the parameters-- DO NOT evaluate whether the agent interpreted the task correctly--Focus ONLY on whether the correct CATEGORIES of tools were selected from what was available.--Return your evaluation as JSON with these fields:-- scores: {"relevance": number, "coverage": number}-- overall_score: number (average of all scores, 0-10)-- feedback: string (focused ONLY on tool selection decisions from available tools)-- improvement_suggestions: string (ONLY suggest better selection from the AVAILABLE tools list, NOT new tools)-"""},- {"role": "user", "content": f"""-Agent role: {agent.role}-Task description: {task.description}--Available tools for this agent:-{available_tools_info}--{tool_types_summary}--Based ONLY on the task description and comparing the AVAILABLE tools with those that were selected (listed above), evaluate if the agent selected the appropriate tool types for this task.--IMPORTANT:-- ONLY evaluate selection from tools listed as available-- DO NOT suggest new tools that aren't in the available tools list-- DO NOT evaluate tool usage or results-"""}- ]- assert self.llm is not None- response = self.llm.call(prompt)-- try:- evaluation_data = extract_json_from_llm_response(response)- assert evaluation_data is not None-- scores = evaluation_data.get("scores", {})- relevance = scores.get("relevance", 5.0)- coverage = scores.get("coverage", 5.0)- overall_score = float(evaluation_data.get("overall_score", 5.0))-- feedback = "Tool Selection Evaluation:\n"- feedback += f"• Relevance: {relevance}/10 - Selection of appropriate tool types for the task\n"- feedback += f"• Coverage: {coverage}/10 - Selection of all necessary tool types\n"- if "improvement_suggestions" in evaluation_data:- feedback += f"Improvement Suggestions:\n{evaluation_data['improvement_suggestions']}"- else:- feedback += evaluation_data.get("feedback", "No detailed feedback available.")-- return EvaluationScore(- score=overall_score,- feedback=feedback,- raw_response=response- )- except Exception as e:- return EvaluationScore(- score=None,- feedback=f"Error evaluating tool selection: {e}",- raw_response=response- )---class ParameterExtractionEvaluator(BaseEvaluator):- @property- def metric_category(self) -> MetricCategory:- return MetricCategory.PARAMETER_EXTRACTION-- def evaluate(- self,- agent: Agent,- task: Task,- execution_trace: Dict[str, Any],- final_output: str,- ) -> EvaluationScore:- tool_uses = execution_trace.get("tool_uses", [])- tool_count = len(tool_uses)-- if tool_count == 0:- return EvaluationScore(- score=None,- feedback="No tool usage detected. Cannot evaluate parameter extraction."- )-- validation_errors = []- for tool_use in tool_uses:- if not tool_use.get("success", True) and tool_use.get("error_type") == "validation_error":- validation_errors.append({- "tool": tool_use.get("tool", "Unknown tool"),- "error": tool_use.get("result"),- "args": tool_use.get("args", {})- })-- validation_error_rate = len(validation_errors) / tool_count if tool_count > 0 else 0-- param_samples = []- for i, tool_use in enumerate(tool_uses[:5]):- tool_name = tool_use.get("tool", "Unknown tool")- tool_args = tool_use.get("args", {})- success = tool_use.get("success", True) and not tool_use.get("error", False)- error_type = tool_use.get("error_type", "") if not success else ""-- is_validation_error = error_type == "validation_error"-- sample = f"Tool use #{i+1} - {tool_name}:\n"- sample += f"- Parameters: {json.dumps(tool_args, indent=2)}\n"- sample += f"- Success: {'No' if not success else 'Yes'}"-- if is_validation_error:- sample += " (PARAMETER VALIDATION ERROR)\n"- sample += f"- Error: {tool_use.get('result', 'Unknown error')}"- elif not success:- sample += f" (Other error: {error_type})\n"-- param_samples.append(sample)-- validation_errors_info = ""- if validation_errors:- validation_errors_info = f"\nParameter validation errors detected: {len(validation_errors)} ({validation_error_rate:.1%} of tool uses)\n"- for i, err in enumerate(validation_errors[:3]):- tool_name = err.get("tool", "Unknown tool")- error_msg = err.get("error", "Unknown error")- args = err.get("args", {})- validation_errors_info += f"\nValidation Error #{i+1}:\n- Tool: {tool_name}\n- Args: {json.dumps(args, indent=2)}\n- Error: {error_msg}"-- if len(validation_errors) > 3:- validation_errors_info += f"\n...and {len(validation_errors) - 3} more validation errors."- param_samples_text = "\n\n".join(param_samples)- prompt = [- {"role": "system", "content": """You are an expert evaluator assessing how well an AI agent extracts and formats PARAMETER VALUES for tool calls.--Your job is to evaluate ONLY whether the agent used the correct parameter VALUES, not whether the right tools were selected or how the tools were invoked.--Evaluate parameter extraction based on these criteria:-1. Accuracy (0-10): Are parameter values correctly identified from the context/task?-2. Formatting (0-10): Are values formatted correctly for each tool's requirements?-3. Completeness (0-10): Are all required parameter values provided, with no missing information?--IMPORTANT: DO NOT evaluate:-- Whether the right tool was chosen (that's the ToolSelectionEvaluator's job)-- How the tools were structurally invoked (that's the ToolInvocationEvaluator's job)-- The quality of results from tools--Focus ONLY on the PARAMETER VALUES - whether they were correctly extracted from the context, properly formatted, and complete.--Validation errors are important signals that parameter values weren't properly extracted or formatted.--Return your evaluation as JSON with these fields:-- scores: {"accuracy": number, "formatting": number, "completeness": number}-- overall_score: number (average of all scores, 0-10)-- feedback: string (focused ONLY on parameter value extraction quality)-- improvement_suggestions: string (concrete suggestions for better parameter VALUE extraction)-"""},- {"role": "user", "content": f"""-Agent role: {agent.role}-Task description: {task.description}--Parameter extraction examples:-{param_samples_text}-{validation_errors_info}--Evaluate the quality of the agent's parameter extraction for this task.-"""}- ]-- assert self.llm is not None- response = self.llm.call(prompt)-- try:- evaluation_data = extract_json_from_llm_response(response)- assert evaluation_data is not None-- scores = evaluation_data.get("scores", {})- accuracy = scores.get("accuracy", 5.0)- formatting = scores.get("formatting", 5.0)- completeness = scores.get("completeness", 5.0)-- overall_score = float(evaluation_data.get("overall_score", 5.0))-- feedback = "Parameter Extraction Evaluation:\n"- feedback += f"• Accuracy: {accuracy}/10 - Correctly identifying required parameters\n"- feedback += f"• Formatting: {formatting}/10 - Properly formatting parameters for tools\n"- feedback += f"• Completeness: {completeness}/10 - Including all necessary information\n\n"-- if "improvement_suggestions" in evaluation_data:- feedback += f"Improvement Suggestions:\n{evaluation_data['improvement_suggestions']}"- else:- feedback += evaluation_data.get("feedback", "No detailed feedback available.")-- return EvaluationScore(- score=overall_score,- feedback=feedback,- raw_response=response- )- except Exception as e:- return EvaluationScore(- score=None,- feedback=f"Error evaluating parameter extraction: {e}",- raw_response=response- )---class ToolInvocationEvaluator(BaseEvaluator):- @property- def metric_category(self) -> MetricCategory:- return MetricCategory.TOOL_INVOCATION-- def evaluate(- self,- agent: Agent,- task: Task,- execution_trace: Dict[str, Any],- final_output: str,- ) -> EvaluationScore:- tool_uses = execution_trace.get("tool_uses", [])- tool_errors = []- tool_count = len(tool_uses)-- if tool_count == 0:- return EvaluationScore(- score=None,- feedback="No tool usage detected. Cannot evaluate tool invocation."- )-- for tool_use in tool_uses:- if not tool_use.get("success", True) or tool_use.get("error", False):- error_info = {- "tool": tool_use.get("tool", "Unknown tool"),- "error": tool_use.get("result"),- "error_type": tool_use.get("error_type", "unknown_error")- }- tool_errors.append(error_info)-- error_rate = len(tool_errors) / tool_count if tool_count > 0 else 0-- error_types = {}- for error in tool_errors:- error_type = error.get("error_type", "unknown_error")- if error_type not in error_types:- error_types[error_type] = 0- error_types[error_type] += 1-- invocation_samples = []- for i, tool_use in enumerate(tool_uses[:5]):- tool_name = tool_use.get("tool", "Unknown tool")- tool_args = tool_use.get("args", {})- success = tool_use.get("success", True) and not tool_use.get("error", False)- error_type = tool_use.get("error_type", "") if not success else ""- error_msg = tool_use.get("result", "No error") if not success else "No error"-- sample = f"Tool invocation #{i+1}:\n"- sample += f"- Tool: {tool_name}\n"- sample += f"- Parameters: {json.dumps(tool_args, indent=2)}\n"- sample += f"- Success: {'No' if not success else 'Yes'}\n"- if not success:- sample += f"- Error type: {error_type}\n"- sample += f"- Error: {error_msg}"- invocation_samples.append(sample)-- error_type_summary = ""- if error_types:- error_type_summary = "Error type breakdown:\n"- for error_type, count in error_types.items():- error_type_summary += f"- {error_type}: {count} occurrences ({(count/tool_count):.1%})\n"-- invocation_samples_text = "\n\n".join(invocation_samples)- prompt = [- {"role": "system", "content": """You are an expert evaluator assessing how correctly an AI agent's tool invocations are STRUCTURED.--Your job is to evaluate ONLY the structural and syntactical aspects of how the agent called tools, NOT which tools were selected or what parameter values were used.--Evaluate the agent's tool invocation based on these criteria:-1. Structure (0-10): Does the tool call follow the expected syntax and format?-2. Error Handling (0-10): Does the agent handle tool errors appropriately?-3. Invocation Patterns (0-10): Are tool calls properly sequenced, batched, or managed?--Error types that indicate invocation issues:-- execution_error: The tool was called correctly but failed during execution-- usage_error: General errors in how the tool was used structurally--IMPORTANT: DO NOT evaluate:-- Whether the right tool was chosen (that's the ToolSelectionEvaluator's job)-- Whether the parameter values are correct (that's the ParameterExtractionEvaluator's job)-- The quality of results from tools--Focus ONLY on HOW tools were invoked - the structure, format, and handling of the invocation process.--Return your evaluation as JSON with these fields:-- scores: {"structure": number, "error_handling": number, "invocation_patterns": number}-- overall_score: number (average of all scores, 0-10)-- feedback: string (focused ONLY on structural aspects of tool invocation)-- improvement_suggestions: string (concrete suggestions for better structuring of tool calls)-"""},- {"role": "user", "content": f"""-Agent role: {agent.role}-Task description: {task.description}--Tool invocation examples:-{invocation_samples_text}--Tool error rate: {error_rate:.2%} ({len(tool_errors)} errors out of {tool_count} invocations)-{error_type_summary}--Evaluate the quality of the agent's tool invocation structure during this task.-"""}- ]-- assert self.llm is not None- response = self.llm.call(prompt)-- try:- evaluation_data = extract_json_from_llm_response(response)- assert evaluation_data is not None- scores = evaluation_data.get("scores", {})- structure = scores.get("structure", 5.0)- error_handling = scores.get("error_handling", 5.0)- invocation_patterns = scores.get("invocation_patterns", 5.0)-- overall_score = float(evaluation_data.get("overall_score", 5.0))-- feedback = "Tool Invocation Evaluation:\n"- feedback += f"• Structure: {structure}/10 - Following proper syntax and format\n"- feedback += f"• Error Handling: {error_handling}/10 - Appropriately handling tool errors\n"- feedback += f"• Invocation Patterns: {invocation_patterns}/10 - Proper sequencing and management of calls\n\n"-- if "improvement_suggestions" in evaluation_data:- feedback += f"Improvement Suggestions:\n{evaluation_data['improvement_suggestions']}"- else:- feedback += evaluation_data.get("feedback", "No detailed feedback available.")-- return EvaluationScore(- score=overall_score,- feedback=feedback,- raw_response=response- )- except Exception as e:- return EvaluationScore(- score=None,- feedback=f"Error evaluating tool invocation: {e}",- raw_response=response- )