Framework for orchestrating role-playing, autonomous AI agents. By fostering collaborative intelligence, CrewAI empowers agents to work together seamlessly, tackling complex tasks.
fix: Crew.test emits its start event before setup so a failed setup pairs (#8420)
Wilson_Jiang2a5548a233d3
diff --git a/lib/crewai/src/crewai/crew.py b/lib/crewai/src/crewai/crew.pyindex d9fbcc114..fe33471fe 100644--- a/lib/crewai/src/crewai/crew.py+++ b/lib/crewai/src/crewai/crew.py@@ -2344,20 +2344,23 @@ class Crew(FlowTrackable, BaseModel):Uses concurrent.futures for concurrent execution."""+ # Emit the start event before setup: crew_test_failed is+ # scope-ending, so a failure during create_llm must find its opener+ # already on the stack instead of popping the caller's scope.+ crewai_event_bus.emit(+ self,+ CrewTestStartedEvent(+ crew_name=self.name,+ n_iterations=n_iterations,+ eval_llm=eval_llm,+ inputs=inputs,+ ),+ )try:llm_instance = create_llm(eval_llm)if not llm_instance:raise ValueError("Failed to create LLM instance.")- crewai_event_bus.emit(- self,- CrewTestStartedEvent(- crew_name=self.name,- n_iterations=n_iterations,- eval_llm=llm_instance,- inputs=inputs,- ),- )test_crew = self.copy()evaluator = CrewEvaluator(test_crew, llm_instance)diff --git a/lib/crewai/tests/test_crew.py b/lib/crewai/tests/test_crew.pyindex 89491b9b4..f20b180ed 100644--- a/lib/crewai/tests/test_crew.py+++ b/lib/crewai/tests/test_crew.py@@ -22,6 +22,7 @@ from crewai.events.types.crew_events import (CrewKickoffFailedEvent,CrewKickoffStartedEvent,CrewTestCompletedEvent,+ CrewTestFailedEvent,CrewTestStartedEvent,CrewTrainCompletedEvent,CrewTrainStartedEvent,@@ -4276,6 +4277,50 @@ def test_crew_testing_function(researcher):assert isinstance(received_events[1], CrewTestCompletedEvent)+def test_crew_test_setup_failure_pairs_events(researcher):+ """A failure during test() setup must still pair Started with Failed.++ crew_test_failed is scope-ending: the start event used to be emitted+ only after create_llm succeeded, so a create_llm failure emitted an+ unpaired Failed that popped the caller's enclosing event scope.+ """+ task = Task(+ description="Summarize the topic.",+ expected_output="A summary.",+ agent=researcher,+ )+ crew = Crew(agents=[researcher], tasks=[task])++ started: list[CrewTestStartedEvent] = []+ failed: list[CrewTestFailedEvent] = []++ @crewai_event_bus.on(CrewTestStartedEvent)+ def on_started(source, event: CrewTestStartedEvent):+ started.append(event)++ @crewai_event_bus.on(CrewTestFailedEvent)+ def on_failed(source, event: CrewTestFailedEvent):+ failed.append(event)++ crewai_event_bus.register_handler(CrewTestStartedEvent, on_started)+ crewai_event_bus.register_handler(CrewTestFailedEvent, on_failed)+ try:+ with patch("crewai.crew.create_llm", side_effect=RuntimeError("no key")):+ with pytest.raises(RuntimeError, match="no key"):+ crew.test(2, "gpt-4o-mini")+ crewai_event_bus.flush()+ finally:+ crewai_event_bus.off(CrewTestStartedEvent, on_started)+ crewai_event_bus.off(CrewTestFailedEvent, on_failed)++ assert started, "test() must emit CrewTestStartedEvent before setup runs"+ assert failed, "test() setup failure must emit CrewTestFailedEvent"+ assert failed[-1].started_event_id == started[-1].event_id, (+ "the Failed event must pair with the test's own start event, not pop "+ "the caller's enclosing scope"+ )++def test_hierarchical_verbose_manager_agent(researcher, writer):task = Task(description="Come up with a list of 5 interesting ideas to explore for an article, then write one amazing paragraph highlight for each idea that showcases how good an article about this topic could be. Return the list of ideas with their paragraph and your notes.",