Framework for orchestrating role-playing, autonomous AI agents. By fostering collaborative intelligence, CrewAI empowers agents to work together seamlessly, tackling complex tasks.
feat(tracing): collect human feedback and pause events in the trace (#7499)
João Moura32a9d2ae7b44
diff --git a/lib/crewai/src/crewai/events/listeners/tracing/trace_listener.py b/lib/crewai/src/crewai/events/listeners/tracing/trace_listener.pyindex ba4a94313..ce3badb28 100644--- a/lib/crewai/src/crewai/events/listeners/tracing/trace_listener.py+++ b/lib/crewai/src/crewai/events/listeners/tracing/trace_listener.py@@ -70,10 +70,14 @@ from crewai.events.types.flow_events import (FlowCreatedEvent,FlowFailedEvent,FlowFinishedEvent,+ FlowPausedEvent,FlowPlotEvent,FlowStartedEvent,+ HumanFeedbackReceivedEvent,+ HumanFeedbackRequestedEvent,MethodExecutionFailedEvent,MethodExecutionFinishedEvent,+ MethodExecutionPausedEvent,MethodExecutionStartedEvent,)from crewai.events.types.knowledge_events import (@@ -280,6 +284,34 @@ class TraceCollectionListener(BaseEventListener):def on_method_failed(source: Any, event: MethodExecutionFailedEvent) -> None:self._handle_trace_event("method_execution_failed", source, event)+ # Registered through `_on`, like every other handler here: while a kickoff+ # owns an execution uuid the OTEL session records these events (see+ # `telemetry/tracing/handlers.py`) and this legacy collector stays idle.++ @self._on(event_bus, MethodExecutionPausedEvent)+ def on_method_paused(source: Any, event: MethodExecutionPausedEvent) -> None:+ """Collect a method pausing for human feedback, whole."""+ self._handle_action_event("method_execution_paused", source, event)++ @self._on(event_bus, HumanFeedbackRequestedEvent)+ def on_human_feedback_requested(+ source: Any, event: HumanFeedbackRequestedEvent+ ) -> None:+ """Collect what the reviewer was shown when a gate asked for feedback."""+ self._handle_action_event("human_feedback_requested", source, event)++ @self._on(event_bus, HumanFeedbackReceivedEvent)+ def on_human_feedback_received(+ source: Any, event: HumanFeedbackReceivedEvent+ ) -> None:+ """Collect the reviewer's answer and where the flow routed on it."""+ self._handle_action_event("human_feedback_received", source, event)++ @self._on(event_bus, FlowPausedEvent)+ def on_flow_paused(source: Any, event: FlowPausedEvent) -> None:+ """Collect a flow pausing (a gate or an explicit pause), whole."""+ self._handle_action_event("flow_paused", source, event)+@self._on(event_bus, ConversationMessageAddedEvent)def on_conversation_message_added(source: Any, event: ConversationMessageAddedEventdiff --git a/lib/crewai/tests/tracing/test_trace_human_feedback_events.py b/lib/crewai/tests/tracing/test_trace_human_feedback_events.pynew file mode 100644index 000000000..bb7e8a78d--- /dev/null+++ b/lib/crewai/tests/tracing/test_trace_human_feedback_events.py@@ -0,0 +1,365 @@+"""A `@human_feedback` gate must reach the trace as more than method start/finish.++The listener subscribed to method and conversation events but not to the+review-gate events (`HumanFeedbackRequestedEvent`, `HumanFeedbackReceivedEvent`)+or the pause events (`MethodExecutionPausedEvent`, `FlowPausedEvent`). A trace+could therefore not say that a run stopped for review, what the reviewer was+shown, or what they answered. Each is collected as a whole-event payload, like+every other non-complex type.+"""++from __future__ import annotations++from collections.abc import Iterator+from datetime import datetime, timedelta, timezone+import os+from typing import Any+from unittest.mock import patch++from crewai.events.event_bus import crewai_event_bus+from crewai.events.listeners.tracing.trace_listener import TraceCollectionListener+from crewai.execution import clear_execution_uuid, set_execution_uuid+from crewai.flow.async_feedback import HumanFeedbackPending, PendingFeedbackContext+from crewai.flow.flow import Flow, listen, start+from crewai.flow.human_feedback import human_feedback+from crewai.flow.persistence.base import FlowPersistence+from crewai.telemetry.tracing.grants import GrantSpanExporter, TraceGrant, TraceGrantClient+from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter+from crewai.events.listeners.tracing.types import TraceEvent+from crewai.events.types.flow_events import (+ FlowPausedEvent,+ HumanFeedbackReceivedEvent,+ HumanFeedbackRequestedEvent,+ MethodExecutionPausedEvent,+)+import pytest+++def listener() -> Iterator[TraceCollectionListener]:+ """A listener with tracing enabled and every handler registered.++ Enabled through the environment like the other tracing tests. Only the+ env var is set, not the tracing context var, so batch initialization+ never reaches the backend: `_initialize_backend_batch` returns before+ any request when tracing is not enabled in context.++ `scoped_handlers` is required: the bus is a process-wide singleton, so+ handlers registered here would otherwise fire in whatever runs next.+ """+ with (+ crewai_event_bus.scoped_handlers(),+ patch.dict(os.environ, {"CREWAI_TRACING_ENABLED": "true"}),+ ):+ trace_listener = TraceCollectionListener()+ trace_listener.setup_listeners(crewai_event_bus)+ yield trace_listener+++def _collected(listener: TraceCollectionListener, event_type: str) -> TraceEvent:+ """The single collected trace event of `event_type`."""+ crewai_event_bus.flush()+ matches = [e for e in listener.batch_manager.event_buffer if e.type == event_type]+ assert len(matches) == 1, (+ f"expected one {event_type!r} trace event, got {len(matches)}: "+ f"{[e.type for e in listener.batch_manager.event_buffer]}"+ )+ return matches[0]+++def test_human_feedback_requested_is_collected_whole(listener) -> None:+ crewai_event_bus.emit(+ None,+ HumanFeedbackRequestedEvent(+ flow_name="review_flow",+ method_name="draft",+ output="the draft shown to the reviewer",+ message="Approve this draft?",+ emit=["approved", "rejected"],+ request_id="req-1",+ ),+ )++ data = _collected(listener, "human_feedback_requested").event_data++ assert data["flow_name"] == "review_flow"+ assert data["method_name"] == "draft"+ assert data["output"] == "the draft shown to the reviewer"+ assert data["message"] == "Approve this draft?"+ assert data["emit"] == ["approved", "rejected"]+ assert data["request_id"] == "req-1"+++def test_human_feedback_received_is_collected_whole(listener) -> None:+ crewai_event_bus.emit(+ None,+ HumanFeedbackReceivedEvent(+ flow_name="review_flow",+ method_name="draft",+ feedback="Looks good, ship it.",+ outcome="approved",+ request_id="req-1",+ ),+ )++ data = _collected(listener, "human_feedback_received").event_data++ assert data["flow_name"] == "review_flow"+ assert data["method_name"] == "draft"+ assert data["feedback"] == "Looks good, ship it."+ assert data["outcome"] == "approved"+ assert data["request_id"] == "req-1"+++def test_method_execution_paused_is_collected_whole(listener) -> None:+ crewai_event_bus.emit(+ None,+ MethodExecutionPausedEvent(+ flow_name="review_flow",+ method_name="draft",+ state={"draft": "v1"},+ flow_id="flow-1",+ message="Approve this draft?",+ emit=["approved", "rejected"],+ ),+ )++ data = _collected(listener, "method_execution_paused").event_data++ assert data["flow_name"] == "review_flow"+ assert data["method_name"] == "draft"+ assert data["flow_id"] == "flow-1"+ assert data["state"] == {"draft": "v1"}+ assert data["message"] == "Approve this draft?"+ assert data["emit"] == ["approved", "rejected"]+++def test_flow_paused_is_collected_whole(listener) -> None:+ crewai_event_bus.emit(+ None,+ FlowPausedEvent(+ flow_name="review_flow",+ flow_id="flow-1",+ method_name="draft",+ state={"draft": "v1"},+ message="Approve this draft?",+ emit=["approved", "rejected"],+ ),+ )++ data = _collected(listener, "flow_paused").event_data++ assert data["flow_name"] == "review_flow"+ assert data["flow_id"] == "flow-1"+ assert data["method_name"] == "draft"+ assert data["state"] == {"draft": "v1"}+ assert data["message"] == "Approve this draft?"+ assert data["emit"] == ["approved", "rejected"]+++def test_the_new_handlers_stay_idle_while_a_kickoff_owns_an_execution_uuid(listener) -> None:+ """The legacy collector must not run beside the OTEL session.++ Every handler in the listener registers through `_on`, which skips the+ event when an execution uuid is bound (the kickoff owns the new session and+ `telemetry/tracing/handlers.py` records these events there). The four gate+ and pause handlers, and the conversation handler, keep that gate: a run+ under an execution uuid collects none of them into a legacy batch.+ """+ token = set_execution_uuid("exec-owned-by-the-otel-session")+ try:+ crewai_event_bus.emit(+ object(),+ HumanFeedbackRequestedEvent(+ flow_name="review_flow",+ method_name="draft",+ output="the draft shown to the reviewer",+ message="Approve this draft?",+ emit=["approved", "rejected"],+ request_id="req-2",+ ),+ )+ crewai_event_bus.emit(+ object(),+ HumanFeedbackReceivedEvent(+ flow_name="review_flow",+ method_name="draft",+ feedback="Looks good, ship it.",+ outcome="approved",+ request_id="req-2",+ ),+ )+ crewai_event_bus.emit(+ object(),+ MethodExecutionPausedEvent(+ flow_name="review_flow",+ method_name="draft",+ state={"draft": "v1"},+ flow_id="flow-2",+ message="Approve this draft?",+ emit=["approved", "rejected"],+ ),+ )+ crewai_event_bus.emit(+ object(),+ FlowPausedEvent(+ flow_name="review_flow",+ flow_id="flow-2",+ method_name="draft",+ state={"draft": "v1"},+ message="Approve this draft?",+ emit=["approved", "rejected"],+ ),+ )+ crewai_event_bus.flush()+ finally:+ clear_execution_uuid(token)++ collected = [e.type for e in listener.batch_manager.event_buffer]+ assert not any(+ t in collected+ for t in (+ "human_feedback_requested",+ "human_feedback_received",+ "method_execution_paused",+ "flow_paused",+ )+ ), collected+++# ---------------------------------------------------------------------------+# Under a tracing kickoff the OTEL session records the four events and the+# legacy collector stays idle — the two halves of the gate, on real flows.+# ---------------------------------------------------------------------------++LEGACY_TYPES = (+ "human_feedback_requested",+ "human_feedback_received",+ "method_execution_paused",+ "flow_paused",+)+++def session_recorders(monkeypatch) -> dict[str, InMemorySpanExporter]:+ """A tracing session that exports to memory, as in tests/telemetry/test_trace_lifecycle.py.++ No socket is opened: the grant is synthetic and the exporter records+ spans per execution uuid; anonymous consent is granted without a prompt.+ """+ monkeypatch.delenv("OTEL_SDK_DISABLED", raising=False)+ monkeypatch.delenv("CREWAI_USER_PAT", raising=False)+ monkeypatch.delenv("CREWAI_PLATFORM_INTEGRATION_TOKEN", raising=False)+ monkeypatch.setenv("CREWAI_TRACING_ENABLED", "true")+ monkeypatch.setenv("CREWAI_DISABLE_TELEMETRY", "true")+ monkeypatch.setattr("crewai.telemetry.tracing.grants.get_auth_token", lambda: None)+ recorders: dict[str, InMemorySpanExporter] = {}++ def create(client: Any, execution_uuid: str) -> TraceGrant:+ return TraceGrant(+ token="synthetic-grant",+ collector_url="https://collector.invalid/v1/traces",+ execution_uuid=execution_uuid,+ expires_at=datetime.now(timezone.utc) + timedelta(minutes=15),+ )++ def exporter(grant: TraceGrant) -> InMemorySpanExporter:+ recorder = InMemorySpanExporter()+ recorders[grant.execution_uuid] = recorder+ return recorder++ monkeypatch.setattr(TraceGrantClient, "create", create)+ monkeypatch.setattr(GrantSpanExporter, "_exporter", staticmethod(exporter))+ return recorders+++def _session_event_names(recorders: dict[str, InMemorySpanExporter]) -> set[str]:+ """Every `crewai.event_name` the session recorded, across executions."""+ return {+ str(span.attributes.get("crewai.event_name"))+ for recorder in recorders.values()+ for span in recorder.get_finished_spans()+ if span.attributes and span.attributes.get("crewai.event_name")+ }+++def _legacy_types(listener: TraceCollectionListener) -> list[str]:+ crewai_event_bus.flush()+ return [e.type for e in listener.batch_manager.event_buffer if e.type in LEGACY_TYPES]+++def test_a_gate_answered_in_place_is_recorded_by_the_session_not_the_legacy_batch(+ listener, session_recorders+) -> None:+ """A `@human_feedback` gate answered at the console, under a tracing kickoff.++ The kickoff owns an execution uuid, so the OTEL session records+ `human_feedback_requested` and `human_feedback_received` as spans and the+ legacy collector — gated by `_on` — collects neither.+ """++ class ReviewFlow(Flow):+ @start()+ @human_feedback(message="Approve this draft?")+ def draft(self) -> str:+ return "the draft"++ @listen(draft)+ def finish(self, result) -> str:+ return f"done: {result.feedback}"++ with (+ patch("builtins.input", return_value="looks good"),+ patch(+ "crewai.telemetry.tracing.ephemeral.prompt_user_for_trace_viewing",+ return_value=True,+ ),+ ):+ result = ReviewFlow(tracing=True).kickoff()++ assert result == "done: looks good"+ recorded = _session_event_names(session_recorders)+ assert {"human_feedback_requested", "human_feedback_received"} <= recorded, recorded+ assert _legacy_types(listener) == []+++def test_a_gate_that_pauses_the_flow_is_recorded_by_the_session_not_the_legacy_batch(+ listener, session_recorders+) -> None:+ """An async provider parks the flow: the session records the two pause+ events; the legacy collector, gated by `_on`, collects neither."""++ class MemoryPersistence(FlowPersistence):+ def init_db(self) -> None:+ pass++ def save_state(self, flow_uuid, method_name, state_data) -> None:+ pass++ def load_state(self, flow_uuid):+ return None++ class AsyncProvider:+ def request_feedback(self, context: PendingFeedbackContext, flow: Flow) -> str:+ raise HumanFeedbackPending(context=context)++ class PausingFlow(Flow):+ @start()+ @human_feedback(message="Approve this draft?", provider=AsyncProvider())+ def draft(self) -> str:+ return "the draft"++ @listen(draft)+ def finish(self, result) -> str:+ return f"done: {result.feedback}"++ with patch(+ "crewai.telemetry.tracing.ephemeral.prompt_user_for_trace_viewing",+ return_value=True,+ ):+ result = PausingFlow(persistence=MemoryPersistence(), tracing=True).kickoff()++ assert isinstance(result, HumanFeedbackPending)+ recorded = _session_event_names(session_recorders)+ assert {"method_execution_paused", "flow_paused"} <= recorded, recorded+ assert _legacy_types(listener) == []