Framework for orchestrating role-playing, autonomous AI agents. By fostering collaborative intelligence, CrewAI empowers agents to work together seamlessly, tackling complex tasks.
feat(events): record how a crew run ended, for every user (#7118)
João Mouracba6c036469c
diff --git a/lib/crewai/src/crewai/crew.py b/lib/crewai/src/crewai/crew.pyindex d3c54f513..0f77b2d22 100644--- a/lib/crewai/src/crewai/crew.py+++ b/lib/crewai/src/crewai/crew.py@@ -207,6 +207,9 @@ class Crew(FlowTrackable, BaseModel):__hash__ = object.__hash___execution_span: Span | None = PrivateAttr()+ # Monotonic stamp for the ungated "Crew Completed" span. A PrivateAttr+ # because Crew is a BaseModel, unlike Flow which takes a plain attribute.+ _telemetry_started_at: float | None = PrivateAttr(default=None)_rpm_controller: RPMController = PrivateAttr()_logger: Logger = PrivateAttr()_file_handler: FileHandler = PrivateAttr()diff --git a/lib/crewai/src/crewai/events/event_listener.py b/lib/crewai/src/crewai/events/event_listener.pyindex 95ec236a9..fd0d77e34 100644--- a/lib/crewai/src/crewai/events/event_listener.py+++ b/lib/crewai/src/crewai/events/event_listener.py@@ -182,17 +182,36 @@ class EventListener(BaseEventListener):def on_default_env(_: Any, event: DefaultEnvEvent) -> None:self._telemetry.env_context_span(event.type)+ def _report_crew_duration(source: Any, outcome: str) -> None:+ """Emit the elapsed time for a crew that reached a terminal state.++ A missing stamp means "no duration to report", not an error: a+ listener attached mid-run, or a completion re-emitted for a run this+ listener never saw start, both land here legitimately. The stamp is+ cleared so a second terminal event on the same instance cannot+ report a duration twice.+ """+ started_at = getattr(source, "_telemetry_started_at", None)+ if started_at is None:+ return+ source._telemetry_started_at = None+ self._telemetry.crew_completed_span(+ source, (time.monotonic() - started_at) * 1000, outcome+ )+@crewai_event_bus.on(CrewKickoffStartedEvent)def on_crew_started(source: Any, event: CrewKickoffStartedEvent) -> None:self.formatter.handle_crew_started(event.crew_name or "Crew", source.id)source._execution_span = self._telemetry.crew_execution_span(source, event.inputs)+ source._telemetry_started_at = time.monotonic()@crewai_event_bus.on(CrewKickoffCompletedEvent)def on_crew_completed(source: Any, event: CrewKickoffCompletedEvent) -> None:final_string_output = event.output.rawself._telemetry.end_crew(source, final_string_output)+ _report_crew_duration(source, "completed")self.formatter.handle_crew_status(event.crew_name or "Crew",@@ -203,6 +222,10 @@ class EventListener(BaseEventListener):@crewai_event_bus.on(CrewKickoffFailedEvent)def on_crew_failed(source: Any, event: CrewKickoffFailedEvent) -> None:+ # end_crew is deliberately not called here: it writes onto the+ # share_crew-gated execution span, which a failed run may never have+ # opened. This span is the only terminal record for a failed crew.+ _report_crew_duration(source, "failed")self.formatter.handle_crew_status(event.crew_name or "Crew",source.id,diff --git a/lib/crewai/src/crewai/telemetry/telemetry.py b/lib/crewai/src/crewai/telemetry/telemetry.pyindex f765fbfdb..749fc5496 100644--- a/lib/crewai/src/crewai/telemetry/telemetry.py+++ b/lib/crewai/src/crewai/telemetry/telemetry.py@@ -1120,6 +1120,45 @@ class Telemetry:return error_type.__name__return None+ def crew_completed_span(self, crew: Crew, duration_ms: float, outcome: str) -> None:+ """Records how long a crew ran and how it ended, for every user.++ The crew analogue of :meth:`flow_completed_span`, and it exists because+ crew was the one level with no ungated terminal record: ``Crew+ Execution`` and :meth:`end_crew` are both behind ``share_crew``, which+ defaults False, so for essentially every run there is no end-of-crew+ span at all -- not one with fields missing. Task and flow outcomes+ already ship ungated, which made crew the odd one out rather than a+ deliberate exclusion.++ A separate span rather than ``Crew Execution`` held open to completion,+ for the same reason flows use one: that span is emitted and closed at+ start, so holding it would drop every run that is killed or crashes.++ Carries no token count. ``crew.token_usage`` sums each agent's LLM+ counters, and two agents sharing one ``LLM`` object share one counter,+ so that total double-counts today -- putting it on a span would+ propagate a known-wrong number into a metric. Models are already on the+ ungated ``Crew Created`` span, which this joins by ``crew_id``.++ Args:+ crew: The crew that finished, for its key and id.+ duration_ms: Wall-clock milliseconds from kickoff, measured on a+ monotonic clock.+ outcome: Either ``"completed"`` or ``"failed"``.+ """++ def _operation() -> None:+ tracer = self.provider.get_tracer(TRACER_NAME)+ span = tracer.start_span("Crew Completed")+ self._add_attribute(span, "crewai_version", version("crewai"))+ add_crew_attributes(span, crew, self._add_attribute)+ self._add_attribute(span, "duration_ms", duration_ms)+ self._add_attribute(span, "outcome", outcome)+ close_span(span)++ self._safe_telemetry_operation(_operation)+def flow_completed_span(self,flow_name: str,diff --git a/lib/crewai/tests/telemetry/test_crew_completed_span.py b/lib/crewai/tests/telemetry/test_crew_completed_span.pynew file mode 100644index 000000000..34466b1ad--- /dev/null+++ b/lib/crewai/tests/telemetry/test_crew_completed_span.py@@ -0,0 +1,173 @@+"""The ungated end-of-run crew span.++Crew was the one level with no terminal record for most users: `Crew Execution`+and `end_crew` are both behind `share_crew`, which defaults False, so a normal+run produced no end-of-crew span at all -- not one with fields missing. Task and+flow outcomes already ship ungated.+"""++from __future__ import annotations++from unittest.mock import MagicMock, patch++from crewai import Agent, Crew, Task+from crewai.events import crewai_event_bus+from crewai.events.event_listener import EventListener+from crewai.events.types.crew_events import (+ CrewKickoffCompletedEvent,+ CrewKickoffFailedEvent,+ CrewKickoffStartedEvent,+)+from crewai.telemetry.telemetry import Telemetry+from opentelemetry.sdk.trace import TracerProvider+from opentelemetry.sdk.trace.export import SimpleSpanProcessor+from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter+import pytest++[email protected](autouse=True)+def enable_otel_sdk(monkeypatch: pytest.MonkeyPatch) -> None:+ """The suite otherwise runs with OTEL_SDK_DISABLED, which makes every+ assertion here pass vacuously against non-recording spans."""+ monkeypatch.delenv("OTEL_SDK_DISABLED", raising=False)+ monkeypatch.delenv("CREWAI_DISABLE_TELEMETRY", raising=False)+ monkeypatch.delenv("CREWAI_DISABLE_TRACKING", raising=False)+++def telemetry():+ """Telemetry whose spans land in memory rather than being exported."""+ exporter = InMemorySpanExporter()+ provider = TracerProvider()+ provider.add_span_processor(SimpleSpanProcessor(exporter))+ instance = Telemetry()+ with (+ patch.object(instance, "provider", provider, create=True),+ patch.object(instance, "_should_execute_telemetry", return_value=True),+ ):+ # yield, not return: the frame keeps `provider` alive, and a collected+ # provider shuts down its processor and silently loses spans.+ yield instance, exporter+++def crew() -> Crew:+ """A minimal one-agent, one-task crew for the span assertions below."""+ agent = Agent(role="r", goal="g", backstory="b")+ return Crew(+ agents=[agent],+ tasks=[Task(description="d", expected_output="e", agent=agent)],+ )+++def _span(exporter: InMemorySpanExporter, name: str):+ """The single exported span with ``name``, failing loudly if it is not unique."""+ matches = [s for s in exporter.get_finished_spans() if s.name == name]+ assert len(matches) == 1, [s.name for s in exporter.get_finished_spans()]+ return matches[0]+++class TestTheSpanItself:+ def test_it_records_the_outcome_and_an_explicit_duration(+ self, telemetry, crew: Crew+ ) -> None:+ """A completed run reports outcome and an explicit duration_ms."""+ instance, exporter = telemetry++ instance.crew_completed_span(crew, 1234.5, "completed")++ span = _span(exporter, "Crew Completed")+ assert span.attributes["outcome"] == "completed"+ assert span.attributes["duration_ms"] == 1234.5++ def test_a_failed_run_is_distinguishable_from_a_successful_one(+ self, telemetry, crew: Crew+ ) -> None:+ """The gap this closes: crew failure had no ungated record at all."""+ instance, exporter = telemetry++ instance.crew_completed_span(crew, 1.0, "completed")+ instance.crew_completed_span(crew, 2.0, "failed")++ outcomes = [+ s.attributes["outcome"]+ for s in exporter.get_finished_spans()+ if s.name == "Crew Completed"+ ]+ assert outcomes == ["completed", "failed"]++ def test_it_joins_to_crew_created_by_key_and_id(+ self, telemetry, crew: Crew+ ) -> None:+ """Models and shape live on Crew Created; this must be joinable to them."""+ instance, exporter = telemetry++ instance.crew_completed_span(crew, 1.0, "completed")++ span = _span(exporter, "Crew Completed")+ assert span.attributes["crew_key"] == crew.key+ assert span.attributes["crew_id"] == str(crew.id)++ def test_it_is_not_gated_by_share_crew(self, telemetry, crew: Crew) -> None:+ """The whole point. `Crew Execution` and `end_crew` are both gated."""+ instance, exporter = telemetry+ assert crew.share_crew is False++ instance.crew_completed_span(crew, 1.0, "completed")++ assert _span(exporter, "Crew Completed") is not None++ def test_it_carries_no_token_count(self, telemetry, crew: Crew) -> None:+ """`crew.token_usage` double-counts agents that share one LLM object, so+ putting it here would propagate a known-wrong number into a metric."""+ instance, exporter = telemetry++ instance.crew_completed_span(crew, 1.0, "completed")++ attributes = _span(exporter, "Crew Completed").attributes+ assert not [k for k in attributes if "token" in k.lower()]+++class TestTheListenerWiring:+ """Asserts on the stamp the listener manages, not on a mock of the shared+ singleton `EventListener._telemetry` -- swapping that leaks across tests."""++ def _run(self, crew: Crew, *events) -> None:+ """Emit ``events`` through a scoped bus with a freshly wired listener."""+ with crewai_event_bus.scoped_handlers():+ EventListener().setup_listeners(crewai_event_bus)+ for event in events:+ crewai_event_bus.emit(crew, event)+ assert crewai_event_bus.flush(), "event bus did not drain"++ def test_kickoff_stamps_a_start_time(self, crew: Crew) -> None:+ """Kickoff records a start time; a terminal event consumes and clears it."""+ assert crew._telemetry_started_at is None++ self._run(crew, CrewKickoffStartedEvent(crew_name="c", inputs=None))++ assert crew._telemetry_started_at is not None++ @pytest.mark.parametrize("terminal", ["completed", "failed"])+ def test_either_terminal_event_consumes_the_stamp(+ self, crew: Crew, terminal: str+ ) -> None:+ """Both paths must report, and both must clear -- a stamp left behind+ would let a later event report a duration measured from the wrong run."""+ output = MagicMock()+ output.raw = "done"+ end = (+ CrewKickoffCompletedEvent(crew_name="c", output=output, total_tokens=0)+ if terminal == "completed"+ else CrewKickoffFailedEvent(crew_name="c", error="boom")+ )++ self._run(crew, CrewKickoffStartedEvent(crew_name="c", inputs=None), end)++ assert crew._telemetry_started_at is None++ def test_a_terminal_event_without_a_start_is_inert(self, crew: Crew) -> None:+ """A listener attached mid-run has no stamp; that is not an error."""+ self._run(crew, CrewKickoffFailedEvent(crew_name="c", error="boom"))++ assert crew._telemetry_started_at is None