Skip to content

Framework for orchestrating role-playing, autonomous AI agents. By fostering collaborative intelligence, CrewAI empowers agents to work together seamlessly, tackling complex tasks.

Make evaluator regression test collectible; repair rotted tests (#8031)

SIDDARTHA REDDY753d19236602
diff --git a/lib/crewai/src/crewai/utilities/evaluators/crew_evaluator_handler.py b/lib/crewai/src/crewai/utilities/evaluators/crew_evaluator_handler.py
index 72dcd2c0d..de0fb7492 100644
--- a/lib/crewai/src/crewai/utilities/evaluators/crew_evaluator_handler.py
+++ b/lib/crewai/src/crewai/utilities/evaluators/crew_evaluator_handler.py
@@ -144,11 +144,14 @@ class CrewEvaluator:
f"- {agents[0]}" if agents else "",
)
+ # Continuation and separator rows must pad to the same column count
+ # declared by the header (3 + number of runs), not a hard-coded 6.
+ empty_score_columns = [""] * (len(self.tasks_scores) + 1)
for agent in agents[1:]:
- table.add_row("", "", "", "", "", f"- {agent}")
+ table.add_row("", *empty_score_columns, f"- {agent}")
if task_index < len(self.crew.tasks) - 1:
- table.add_row("", "", "", "", "", "")
+ table.add_row("", *empty_score_columns, "")
crew_scores = [
sum(self.tasks_scores[run]) / len(self.tasks_scores[run])
diff --git a/lib/crewai/tests/utilities/evaluators/test_crew_evaluator_handler.py b/lib/crewai/tests/utilities/evaluators/test_crew_evaluator_handler.py
index 72ef2e3ba..87fe311a9 100644
--- a/lib/crewai/tests/utilities/evaluators/test_crew_evaluator_handler.py
+++ b/lib/crewai/tests/utilities/evaluators/test_crew_evaluator_handler.py
@@ -1,3 +1,4 @@
+from datetime import datetime, timedelta, timezone
from unittest import mock
import pytest
@@ -11,7 +12,7 @@ from crewai.utilities.evaluators.crew_evaluator_handler import (
)
-class InternalCrewEvaluator:
+class TestInternalCrewEvaluator:
@pytest.fixture
def crew_planner(self):
agent = Agent(role="Agent 1", goal="Goal 1", backstory="Backstory 1")
@@ -22,7 +23,7 @@ class InternalCrewEvaluator:
)
crew = Crew(agents=[agent], tasks=[task])
- return CrewEvaluator(crew, openai_model_name="gpt-4o-mini")
+ return CrewEvaluator(crew, eval_llm="gpt-4o-mini")
def test_setup_for_evaluating(self, crew_planner):
crew_planner._setup_for_evaluating()
@@ -114,7 +115,7 @@ class InternalCrewEvaluator:
mock.call().add_column("Avg. Total", justify="center"),
mock.call().add_column("Agents", style="green"),
mock.call().add_row("Task 1", "10.0", "9.0", "9.5", "- Agent 1"),
- mock.call().add_row("", "", "", "", "", ""), # Blank row between tasks
+ mock.call().add_row("", "", "", "", ""), # Blank row between tasks
mock.call().add_row("Task 2", "9.0", "8.0", "8.5", "- Agent 2"),
mock.call().add_row("Crew", "9.00", "8.00", "8.5", ""),
mock.call().add_row("Execution Time (s)", "135", "155", "145", ""),
@@ -122,12 +123,71 @@ class InternalCrewEvaluator:
)
# Ensure the console prints the table
- console.assert_has_calls([mock.call(), mock.call().print(table())])
+ console.assert_has_calls(
+ [mock.call(), mock.call().print("\n"), mock.call().print(table())]
+ )
+
+ @mock.patch("crewai.utilities.evaluators.crew_evaluator_handler.Console")
+ @mock.patch("crewai.utilities.evaluators.crew_evaluator_handler.Table")
+ def test_print_crew_evaluation_result_row_cell_count(
+ self, table, console, crew_planner
+ ):
+ """Continuation and separator rows must match the dynamic column count.
+
+ Regression test for https://github.com/crewAIInc/crewAI/issues/8024:
+ with N != 3 evaluation runs, the continuation-agent and blank
+ separator rows must emit exactly 3 + N cells.
+ """
+ for num_runs in (1, 2, 5):
+ table.reset_mock()
+ crew_planner.tasks_scores = {
+ run: [8.0, 9.0] for run in range(1, num_runs + 1)
+ }
+ crew_planner.run_execution_times = {
+ run: [24, 45] for run in range(1, num_runs + 1)
+ }
+
+ crew_planner.crew.tasks = [
+ mock.Mock(
+ agent=mock.Mock(),
+ processed_by_agents=["Agent 1", "Agent 2"],
+ ),
+ mock.Mock(
+ agent=mock.Mock(),
+ processed_by_agents=["Agent 3"],
+ ),
+ ]
+
+ crew_planner.print_crew_evaluation_result()
+
+ expected_columns = 3 + num_runs
+ expected_padding = [""] * (num_runs + 1)
+ add_row_calls = [
+ call for call in table.mock_calls if call[0] == "().add_row"
+ ]
+ assert add_row_calls, "no add_row calls recorded"
+ for call in add_row_calls:
+ assert len(call.args) == expected_columns, (
+ f"{len(call.args)} cells for {num_runs} runs: {call.args}"
+ )
+ assert (
+ "",
+ *expected_padding,
+ "- Agent 2",
+ ) in [call.args for call in add_row_calls]
+ assert (
+ "",
+ *expected_padding,
+ "",
+ ) in [call.args for call in add_row_calls]
def test_evaluate(self, crew_planner):
task_output = TaskOutput(
description="Task 1", agent=str(crew_planner.crew.agents[0])
)
+ task = crew_planner.crew.tasks[0]
+ task.end_time = datetime.now(timezone.utc)
+ task.start_time = task.end_time - timedelta(seconds=5)
with mock.patch.object(Task, "execute_sync") as execute:
execute().pydantic = TaskEvaluationPydanticOutput(quality=9.5)