Skip to content

Framework for orchestrating role-playing, autonomous AI agents. By fostering collaborative intelligence, CrewAI empowers agents to work together seamlessly, tackling complex tasks.

feat(events): record how a crew run ended, for every user (#7118)

João Mouracba6c036469c
diff --git a/lib/crewai/src/crewai/crew.py b/lib/crewai/src/crewai/crew.py
index d3c54f513..0f77b2d22 100644
--- a/lib/crewai/src/crewai/crew.py
+++ b/lib/crewai/src/crewai/crew.py
@@ -207,6 +207,9 @@ class Crew(FlowTrackable, BaseModel):
__hash__ = object.__hash__
_execution_span: Span | None = PrivateAttr()
+ # Monotonic stamp for the ungated "Crew Completed" span. A PrivateAttr
+ # because Crew is a BaseModel, unlike Flow which takes a plain attribute.
+ _telemetry_started_at: float | None = PrivateAttr(default=None)
_rpm_controller: RPMController = PrivateAttr()
_logger: Logger = PrivateAttr()
_file_handler: FileHandler = PrivateAttr()
diff --git a/lib/crewai/src/crewai/events/event_listener.py b/lib/crewai/src/crewai/events/event_listener.py
index 95ec236a9..fd0d77e34 100644
--- a/lib/crewai/src/crewai/events/event_listener.py
+++ b/lib/crewai/src/crewai/events/event_listener.py
@@ -182,17 +182,36 @@ class EventListener(BaseEventListener):
def on_default_env(_: Any, event: DefaultEnvEvent) -> None:
self._telemetry.env_context_span(event.type)
+ def _report_crew_duration(source: Any, outcome: str) -> None:
+ """Emit the elapsed time for a crew that reached a terminal state.
+
+ A missing stamp means "no duration to report", not an error: a
+ listener attached mid-run, or a completion re-emitted for a run this
+ listener never saw start, both land here legitimately. The stamp is
+ cleared so a second terminal event on the same instance cannot
+ report a duration twice.
+ """
+ started_at = getattr(source, "_telemetry_started_at", None)
+ if started_at is None:
+ return
+ source._telemetry_started_at = None
+ self._telemetry.crew_completed_span(
+ source, (time.monotonic() - started_at) * 1000, outcome
+ )
+
@crewai_event_bus.on(CrewKickoffStartedEvent)
def on_crew_started(source: Any, event: CrewKickoffStartedEvent) -> None:
self.formatter.handle_crew_started(event.crew_name or "Crew", source.id)
source._execution_span = self._telemetry.crew_execution_span(
source, event.inputs
)
+ source._telemetry_started_at = time.monotonic()
@crewai_event_bus.on(CrewKickoffCompletedEvent)
def on_crew_completed(source: Any, event: CrewKickoffCompletedEvent) -> None:
final_string_output = event.output.raw
self._telemetry.end_crew(source, final_string_output)
+ _report_crew_duration(source, "completed")
self.formatter.handle_crew_status(
event.crew_name or "Crew",
@@ -203,6 +222,10 @@ class EventListener(BaseEventListener):
@crewai_event_bus.on(CrewKickoffFailedEvent)
def on_crew_failed(source: Any, event: CrewKickoffFailedEvent) -> None:
+ # end_crew is deliberately not called here: it writes onto the
+ # share_crew-gated execution span, which a failed run may never have
+ # opened. This span is the only terminal record for a failed crew.
+ _report_crew_duration(source, "failed")
self.formatter.handle_crew_status(
event.crew_name or "Crew",
source.id,
diff --git a/lib/crewai/src/crewai/telemetry/telemetry.py b/lib/crewai/src/crewai/telemetry/telemetry.py
index f765fbfdb..749fc5496 100644
--- a/lib/crewai/src/crewai/telemetry/telemetry.py
+++ b/lib/crewai/src/crewai/telemetry/telemetry.py
@@ -1120,6 +1120,45 @@ class Telemetry:
return error_type.__name__
return None
+ def crew_completed_span(self, crew: Crew, duration_ms: float, outcome: str) -> None:
+ """Records how long a crew ran and how it ended, for every user.
+
+ The crew analogue of :meth:`flow_completed_span`, and it exists because
+ crew was the one level with no ungated terminal record: ``Crew
+ Execution`` and :meth:`end_crew` are both behind ``share_crew``, which
+ defaults False, so for essentially every run there is no end-of-crew
+ span at all -- not one with fields missing. Task and flow outcomes
+ already ship ungated, which made crew the odd one out rather than a
+ deliberate exclusion.
+
+ A separate span rather than ``Crew Execution`` held open to completion,
+ for the same reason flows use one: that span is emitted and closed at
+ start, so holding it would drop every run that is killed or crashes.
+
+ Carries no token count. ``crew.token_usage`` sums each agent's LLM
+ counters, and two agents sharing one ``LLM`` object share one counter,
+ so that total double-counts today -- putting it on a span would
+ propagate a known-wrong number into a metric. Models are already on the
+ ungated ``Crew Created`` span, which this joins by ``crew_id``.
+
+ Args:
+ crew: The crew that finished, for its key and id.
+ duration_ms: Wall-clock milliseconds from kickoff, measured on a
+ monotonic clock.
+ outcome: Either ``"completed"`` or ``"failed"``.
+ """
+
+ def _operation() -> None:
+ tracer = self.provider.get_tracer(TRACER_NAME)
+ span = tracer.start_span("Crew Completed")
+ self._add_attribute(span, "crewai_version", version("crewai"))
+ add_crew_attributes(span, crew, self._add_attribute)
+ self._add_attribute(span, "duration_ms", duration_ms)
+ self._add_attribute(span, "outcome", outcome)
+ close_span(span)
+
+ self._safe_telemetry_operation(_operation)
+
def flow_completed_span(
self,
flow_name: str,
diff --git a/lib/crewai/tests/telemetry/test_crew_completed_span.py b/lib/crewai/tests/telemetry/test_crew_completed_span.py
new file mode 100644
index 000000000..34466b1ad
--- /dev/null
+++ b/lib/crewai/tests/telemetry/test_crew_completed_span.py
@@ -0,0 +1,173 @@
+"""The ungated end-of-run crew span.
+
+Crew was the one level with no terminal record for most users: `Crew Execution`
+and `end_crew` are both behind `share_crew`, which defaults False, so a normal
+run produced no end-of-crew span at all -- not one with fields missing. Task and
+flow outcomes already ship ungated.
+"""
+
+from __future__ import annotations
+
+from unittest.mock import MagicMock, patch
+
+from crewai import Agent, Crew, Task
+from crewai.events import crewai_event_bus
+from crewai.events.event_listener import EventListener
+from crewai.events.types.crew_events import (
+ CrewKickoffCompletedEvent,
+ CrewKickoffFailedEvent,
+ CrewKickoffStartedEvent,
+)
+from crewai.telemetry.telemetry import Telemetry
+from opentelemetry.sdk.trace import TracerProvider
+from opentelemetry.sdk.trace.export import SimpleSpanProcessor
+from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter
+import pytest
+
+
[email protected](autouse=True)
+def enable_otel_sdk(monkeypatch: pytest.MonkeyPatch) -> None:
+ """The suite otherwise runs with OTEL_SDK_DISABLED, which makes every
+ assertion here pass vacuously against non-recording spans."""
+ monkeypatch.delenv("OTEL_SDK_DISABLED", raising=False)
+ monkeypatch.delenv("CREWAI_DISABLE_TELEMETRY", raising=False)
+ monkeypatch.delenv("CREWAI_DISABLE_TRACKING", raising=False)
+
+
+def telemetry():
+ """Telemetry whose spans land in memory rather than being exported."""
+ exporter = InMemorySpanExporter()
+ provider = TracerProvider()
+ provider.add_span_processor(SimpleSpanProcessor(exporter))
+ instance = Telemetry()
+ with (
+ patch.object(instance, "provider", provider, create=True),
+ patch.object(instance, "_should_execute_telemetry", return_value=True),
+ ):
+ # yield, not return: the frame keeps `provider` alive, and a collected
+ # provider shuts down its processor and silently loses spans.
+ yield instance, exporter
+
+
+def crew() -> Crew:
+ """A minimal one-agent, one-task crew for the span assertions below."""
+ agent = Agent(role="r", goal="g", backstory="b")
+ return Crew(
+ agents=[agent],
+ tasks=[Task(description="d", expected_output="e", agent=agent)],
+ )
+
+
+def _span(exporter: InMemorySpanExporter, name: str):
+ """The single exported span with ``name``, failing loudly if it is not unique."""
+ matches = [s for s in exporter.get_finished_spans() if s.name == name]
+ assert len(matches) == 1, [s.name for s in exporter.get_finished_spans()]
+ return matches[0]
+
+
+class TestTheSpanItself:
+ def test_it_records_the_outcome_and_an_explicit_duration(
+ self, telemetry, crew: Crew
+ ) -> None:
+ """A completed run reports outcome and an explicit duration_ms."""
+ instance, exporter = telemetry
+
+ instance.crew_completed_span(crew, 1234.5, "completed")
+
+ span = _span(exporter, "Crew Completed")
+ assert span.attributes["outcome"] == "completed"
+ assert span.attributes["duration_ms"] == 1234.5
+
+ def test_a_failed_run_is_distinguishable_from_a_successful_one(
+ self, telemetry, crew: Crew
+ ) -> None:
+ """The gap this closes: crew failure had no ungated record at all."""
+ instance, exporter = telemetry
+
+ instance.crew_completed_span(crew, 1.0, "completed")
+ instance.crew_completed_span(crew, 2.0, "failed")
+
+ outcomes = [
+ s.attributes["outcome"]
+ for s in exporter.get_finished_spans()
+ if s.name == "Crew Completed"
+ ]
+ assert outcomes == ["completed", "failed"]
+
+ def test_it_joins_to_crew_created_by_key_and_id(
+ self, telemetry, crew: Crew
+ ) -> None:
+ """Models and shape live on Crew Created; this must be joinable to them."""
+ instance, exporter = telemetry
+
+ instance.crew_completed_span(crew, 1.0, "completed")
+
+ span = _span(exporter, "Crew Completed")
+ assert span.attributes["crew_key"] == crew.key
+ assert span.attributes["crew_id"] == str(crew.id)
+
+ def test_it_is_not_gated_by_share_crew(self, telemetry, crew: Crew) -> None:
+ """The whole point. `Crew Execution` and `end_crew` are both gated."""
+ instance, exporter = telemetry
+ assert crew.share_crew is False
+
+ instance.crew_completed_span(crew, 1.0, "completed")
+
+ assert _span(exporter, "Crew Completed") is not None
+
+ def test_it_carries_no_token_count(self, telemetry, crew: Crew) -> None:
+ """`crew.token_usage` double-counts agents that share one LLM object, so
+ putting it here would propagate a known-wrong number into a metric."""
+ instance, exporter = telemetry
+
+ instance.crew_completed_span(crew, 1.0, "completed")
+
+ attributes = _span(exporter, "Crew Completed").attributes
+ assert not [k for k in attributes if "token" in k.lower()]
+
+
+class TestTheListenerWiring:
+ """Asserts on the stamp the listener manages, not on a mock of the shared
+ singleton `EventListener._telemetry` -- swapping that leaks across tests."""
+
+ def _run(self, crew: Crew, *events) -> None:
+ """Emit ``events`` through a scoped bus with a freshly wired listener."""
+ with crewai_event_bus.scoped_handlers():
+ EventListener().setup_listeners(crewai_event_bus)
+ for event in events:
+ crewai_event_bus.emit(crew, event)
+ assert crewai_event_bus.flush(), "event bus did not drain"
+
+ def test_kickoff_stamps_a_start_time(self, crew: Crew) -> None:
+ """Kickoff records a start time; a terminal event consumes and clears it."""
+ assert crew._telemetry_started_at is None
+
+ self._run(crew, CrewKickoffStartedEvent(crew_name="c", inputs=None))
+
+ assert crew._telemetry_started_at is not None
+
+ @pytest.mark.parametrize("terminal", ["completed", "failed"])
+ def test_either_terminal_event_consumes_the_stamp(
+ self, crew: Crew, terminal: str
+ ) -> None:
+ """Both paths must report, and both must clear -- a stamp left behind
+ would let a later event report a duration measured from the wrong run."""
+ output = MagicMock()
+ output.raw = "done"
+ end = (
+ CrewKickoffCompletedEvent(crew_name="c", output=output, total_tokens=0)
+ if terminal == "completed"
+ else CrewKickoffFailedEvent(crew_name="c", error="boom")
+ )
+
+ self._run(crew, CrewKickoffStartedEvent(crew_name="c", inputs=None), end)
+
+ assert crew._telemetry_started_at is None
+
+ def test_a_terminal_event_without_a_start_is_inert(self, crew: Crew) -> None:
+ """A listener attached mid-run has no stamp; that is not an error."""
+ self._run(crew, CrewKickoffFailedEvent(crew_name="c", error="boom"))
+
+ assert crew._telemetry_started_at is None