Framework for orchestrating role-playing, autonomous AI agents. By fostering collaborative intelligence, CrewAI empowers agents to work together seamlessly, tackling complex tasks.
from datetime import datetime, timedelta, timezone
import json
from unittest.mock import Mock
from crewai.events.event_bus import crewai_event_bus
from crewai.execution import get_execution_uuid
from crewai.flow.async_feedback.types import (
HumanFeedbackPending,
PendingFeedbackContext,
)
from crewai.flow.flow import Flow, listen, start
from crewai.flow.persistence.base import FlowPersistence
from crewai.telemetry.tracing.context import get_trace_session
from crewai.telemetry.tracing.ephemeral import EphemeralSpanBuffer, trace_consent
from crewai.telemetry.tracing.grants import (
GrantSpanExporter,
TraceGrant,
TraceGrantClient,
)
from opentelemetry import trace
from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter
from pydantic import Field
import pytest
@pytest.fixture
def traces(monkeypatch):
for name in (
"OTEL_SDK_DISABLED",
"CREWAI_USER_PAT",
"CREWAI_ENTERPRISE_ACTION_AUTH_TOKEN",
):
monkeypatch.delenv(name, raising=False)
monkeypatch.setenv("CREWAI_TRACING_ENABLED", "true")
monkeypatch.setenv("CREWAI_DISABLE_TELEMETRY", "true")
monkeypatch.setattr("crewai.telemetry.tracing.grants.get_auth_token", lambda: None)
monkeypatch.setattr(
"crewai.events.listeners.tracing.utils.should_auto_collect_first_time_traces",
lambda: False,
)
buffers, grants, recorders = [], [], []
def buffer():
result = EphemeralSpanBuffer()
buffers.append(result)
return result
def create(client, execution_uuid):
grant = TraceGrant(
token="synthetic-grant",
collector_url="https://collector.invalid/v1/traces",
execution_uuid=execution_uuid,
expires_at=datetime.now(timezone.utc) + timedelta(minutes=15),
)
grants.append(grant)
return grant
def exporter(grant):
result = InMemorySpanExporter()
recorders.append(result)
return result
monkeypatch.setattr(
"crewai.telemetry.tracing.ephemeral.EphemeralSpanBuffer", buffer
)
monkeypatch.setattr(TraceGrantClient, "create", create)
monkeypatch.setattr(GrantSpanExporter, "_exporter", staticmethod(exporter))
prompt = Mock(return_value=True)
monkeypatch.setattr(
"crewai.telemetry.tracing.ephemeral.prompt_user_for_trace_viewing", prompt
)
return buffers, grants, recorders, prompt
# `OTEL_SDK_DISABLED` is not among them: it is the telemetry opt-out
# (test_the_telemetry_opt_out_does_not_turn_tracing_off).
@pytest.mark.parametrize("disabled", ["flag", "env"])
@pytest.mark.parametrize("finalize_only", [False, True])
def test_disabling_deferred_trace_discards_without_consent(
traces, monkeypatch, disabled, finalize_only
):
buffers, grants, recorders, prompt = traces
sessions = []
class Conversation(Flow):
@start()
def turn(self):
sessions.append(get_trace_session())
return "private draft"
flow = Conversation(
tracing=None if disabled == "env" else True, defer_trace_finalization=True
)
consent = Mock(return_value=True)
with trace_consent(consent):
assert flow.kickoff() == "private draft"
assert crewai_event_bus.flush()
old_lifetime = flow._deferred_execution_trace
assert old_lifetime is not None and buffers[0]._spans
if disabled == "flag":
flow.tracing = False
else:
monkeypatch.setenv("CREWAI_TRACING_ENABLED", "false")
if not finalize_only:
assert flow.kickoff() == "private draft"
assert sessions[1] is None
flow.finalize_session_traces()
assert sessions[0] is old_lifetime.session
assert old_lifetime.closed and flow._deferred_execution_trace is None
assert buffers[0]._closed and not buffers[0]._spans
assert len(buffers) == 1 and not grants and not recorders
consent.assert_not_called()
prompt.assert_not_called()
assert get_trace_session() is None and get_execution_uuid() is None
@pytest.mark.parametrize("how", ["env", "flag"])
def test_tracing_asked_for_is_the_answer_and_the_run_is_not_asked_again(
traces, monkeypatch, how
):
"""Turning tracing on IS consent. The prompt at the end of a run is for the
first-time collection nobody asked for — not for a user who said collect it."""
buffers, grants, recorders, prompt = traces
# the rule only holds where a prompt could have been shown, and a suite is
# one of the places it could not — so say a person is here
utils = "crewai.events.listeners.tracing.utils"
monkeypatch.setattr(f"{utils}._is_interactive_terminal", lambda: True)
monkeypatch.setattr(f"{utils}._is_test_environment", lambda: False)
if how == "env":
monkeypatch.setenv("CREWAI_TRACING_ENABLED", "true")
else: # the flag alone must carry it, or this case proves nothing
monkeypatch.delenv("CREWAI_TRACING_ENABLED", raising=False)
class Conversation(Flow):
@start()
def turn(self):
return "said out loud"
flow = Conversation(tracing=(how == "flag") or None)
assert flow.kickoff() == "said out loud"
assert crewai_event_bus.flush()
prompt.assert_not_called() # asked once, not twice
assert grants and recorders # and the trace went where it was told to go
@pytest.mark.parametrize(
"closed", ["no terminal", "under test", "messages suppressed"]
)
def test_where_the_prompt_could_not_be_shown_nothing_is_uploaded(
traces, monkeypatch, closed
):
"""The switch answers a question; where the question could not have been
put to anybody, there is nothing to answer. A copied `.env` reaching CI
carries the variable, not the person — and a suite under test, and a host
that suppressed tracing messages, are the same case. All three have always
failed closed, and still do."""
buffers, grants, recorders, prompt = traces
monkeypatch.setenv("CREWAI_TRACING_ENABLED", "true")
utils = "crewai.events.listeners.tracing.utils"
monkeypatch.setattr(f"{utils}._is_interactive_terminal", lambda: closed != "no terminal")
monkeypatch.setattr(f"{utils}._is_test_environment", lambda: closed == "under test")
monkeypatch.setattr(
f"{utils}.should_suppress_tracing_messages",
lambda: closed == "messages suppressed",
)
prompt.return_value = False # what the prompt answers where nobody can answer
class Conversation(Flow):
@start()
def turn(self):
return "said in a container"
assert Conversation(tracing=True).kickoff() == "said in a container"
assert crewai_event_bus.flush()
assert not grants and not recorders # nothing left the machine
@pytest.mark.parametrize(
"closed", ["no terminal", "under test", "messages suppressed"]
)
def test_a_run_crewai_eval_started_is_uploaded_with_nobody_to_ask(
traces, monkeypatch, tmp_path, closed
):
"""`crewai eval` ran the crew to grade it, and the grade is read off the
trace: running the command was the yes. A coding agent with no terminal
gets its run uploaded."""
from crewai_core import eval_handshake
buffers, grants, recorders, prompt = traces
monkeypatch.setattr(eval_handshake, "awaiting_dir", lambda: tmp_path / "eval-awaiting")
monkeypatch.delenv("CREWAI_TRACING_ENABLED", raising=False)
utils = "crewai.events.listeners.tracing.utils"
monkeypatch.setattr(f"{utils}._is_interactive_terminal", lambda: closed != "no terminal")
monkeypatch.setattr(f"{utils}._is_test_environment", lambda: closed == "under test")
monkeypatch.setattr(
f"{utils}.should_suppress_tracing_messages",
lambda: closed == "messages suppressed",
)
class Conversation(Flow):
@start()
def turn(self):
return "graded"
with eval_handshake.awaiting_evaluation() as told:
assert told
assert Conversation().kickoff() == "graded"
assert crewai_event_bus.flush()
prompt.assert_not_called()
assert grants and recorders # the trace reached AMP
@pytest.mark.parametrize("off", ["false", "0"])
def test_a_project_that_turned_tracing_off_is_not_traced_even_for_crewai_eval(
traces, monkeypatch, tmp_path, off
):
"""An explicit no is the project's; `crewai eval` asks to turn tracing on
before it runs anything, and nothing here overrules it."""
from crewai_core import eval_handshake
buffers, grants, recorders, prompt = traces
monkeypatch.setattr(eval_handshake, "awaiting_dir", lambda: tmp_path / "eval-awaiting")
monkeypatch.setenv("CREWAI_TRACING_ENABLED", off)
monkeypatch.setattr(
"crewai.events.listeners.tracing.utils._is_interactive_terminal", lambda: False
)
class Conversation(Flow):
@start()
def turn(self):
return "kept here"
with eval_handshake.awaiting_evaluation():
assert Conversation().kickoff() == "kept here"
assert crewai_event_bus.flush()
assert not buffers and not grants and not recorders # never even traced
@pytest.mark.parametrize(
("amp", "tier"),
[("https://app.crewai.com", "authenticated"), ("https://evil.example", "ephemeral")],
)
def test_the_saved_login_goes_only_to_an_amp_crewai_eval_trusted(
traces, monkeypatch, tmp_path, amp, tier
):
"""`crewai eval` decides which AMP may receive the saved login before the
project's `.env` is loaded; an AMP that `.env` chose gets an anonymous
grant for the run the command started, never the login."""
from crewai.telemetry.tracing.grants import TraceGrantClient
from crewai_core import eval_handshake
# `traces` installs the scripted grants and exporters
monkeypatch.setattr(eval_handshake, "awaiting_dir", lambda: tmp_path / "eval-awaiting")
monkeypatch.delenv("CREWAI_TRACING_ENABLED", raising=False)
monkeypatch.setenv("CREWAI_PLUS_URL", amp)
monkeypatch.setattr(
"crewai.telemetry.tracing.grants.resolve_tracing_credential",
lambda: ("login", "the-saved-login"),
)
monkeypatch.setattr(
"crewai.events.listeners.tracing.utils._is_interactive_terminal", lambda: False
)
tiers: list[str] = []
real_create = TraceGrantClient.create
monkeypatch.setattr(
TraceGrantClient,
"create",
lambda client, execution_uuid: tiers.append(client._tier)
or real_create(client, execution_uuid),
)
class Conversation(Flow):
@start()
def turn(self):
return "graded"
with eval_handshake.awaiting_evaluation(
trusted_amp_origins=["https://app.crewai.com"]
):
assert Conversation().kickoff() == "graded"
assert crewai_event_bus.flush()
assert tiers == [tier]
def test_a_token_a_dotenv_could_set_approves_nothing(traces, monkeypatch, tmp_path):
"""The variable alone is what a project's `.env` could carry; without the
file `crewai eval` writes, it counts for nothing."""
from crewai_core import eval_handshake
buffers, grants, recorders, prompt = traces
monkeypatch.setattr(eval_handshake, "awaiting_dir", lambda: tmp_path / "eval-awaiting")
monkeypatch.setenv(eval_handshake.AWAITING_ENV, "0" * 32)
monkeypatch.setenv("CREWAI_TRACING_ENABLED", "true")
utils = "crewai.events.listeners.tracing.utils"
monkeypatch.setattr(f"{utils}._is_interactive_terminal", lambda: False)
prompt.return_value = False
class Conversation(Flow):
@start()
def turn(self):
return "in a container"
assert Conversation(tracing=True).kickoff() == "in a container"
assert crewai_event_bus.flush()
assert not grants and not recorders # nothing left the machine
@pytest.mark.parametrize("anonymous", [True, False])
def test_the_telemetry_opt_out_does_not_turn_tracing_off(traces, monkeypatch, anonymous):
"""`OTEL_SDK_DISABLED` is the telemetry opt-out, a separate setting: a run
with tracing on is still traced and uploaded, every span recorded."""
buffers, grants, recorders, prompt = traces
monkeypatch.setenv("OTEL_SDK_DISABLED", "true")
utils = "crewai.events.listeners.tracing.utils"
monkeypatch.setattr(f"{utils}._is_interactive_terminal", lambda: True)
monkeypatch.setattr(f"{utils}._is_test_environment", lambda: False)
if not anonymous:
monkeypatch.setattr(
"crewai.telemetry.tracing.grants.resolve_tracing_credential",
lambda: ("a-login", "login"),
)
class Conversation(Flow):
@start()
def turn(self):
return "traced"
assert Conversation(tracing=True).kickoff() == "traced"
assert crewai_event_bus.flush()
assert grants and recorders
assert any(recorder.get_finished_spans() for recorder in recorders)
def test_crewais_trace_provider_records_with_the_sdk_switched_off(monkeypatch):
"""Pins the SDK detail the provider relies on (its private `_disabled`)."""
from crewai.telemetry.tracing.session import _trace_provider
from opentelemetry.sdk.resources import Resource
monkeypatch.setenv("OTEL_SDK_DISABLED", "true")
span = _trace_provider(Resource({"service.name": "crewai"})).get_tracer("t").start_span("s")
assert span.is_recording()
span.end()
def test_enabling_deferred_trace_opens_a_new_flow_root(traces):
buffers, grants, recorders, prompt = traces
sessions = []
class Conversation(Flow):
@start()
def turn(self):
sessions.append(get_trace_session())
return "draft"
flow = Conversation(tracing=False, defer_trace_finalization=True)
consent = Mock(return_value=True)
with trace_consent(consent):
flow.kickoff()
assert sessions == [None] and not buffers
flow.tracing = True
flow.kickoff()
assert sessions[1] is not None
flow.finalize_session_traces()
consent.assert_called_once()
prompt.assert_not_called()
assert len(grants) == len(recorders) == 1
spans = recorders[0].get_finished_spans()
roots = [span for span in spans if span.parent is None]
assert [span.name for span in roots] == ["execute flow"]
assert roots[0].status.status_code == trace.StatusCode.OK
methods = [span for span in spans if span.name == "call method"]
assert len(methods) == 1 and methods[0].parent == roots[0].context
assert get_trace_session() is None and get_execution_uuid() is None
class JsonPersistence(FlowPersistence):
saved: dict[str, str] = Field(default_factory=dict)
def init_db(self):
pass
def save_state(self, flow_uuid, method_name, state_data):
pass
def load_state(self, flow_uuid):
loaded = self.load_pending_feedback(flow_uuid)
return loaded[0] if loaded else None
def save_pending_feedback(self, flow_uuid, context, state_data):
state = state_data if isinstance(state_data, dict) else state_data.model_dump()
self.saved[flow_uuid] = json.dumps(
{"state": state, "context": context.to_dict()}
)
def load_pending_feedback(self, flow_uuid):
if flow_uuid not in self.saved:
return None
data = json.loads(self.saved[flow_uuid])
return data["state"], PendingFeedbackContext.from_dict(data["context"])
def clear_pending_feedback(self, flow_uuid):
self.saved.pop(flow_uuid, None)
@pytest.mark.parametrize("legacy_context", [False, True])
def test_parent_pause_persists_its_own_trace_and_resume_feedback(
traces, legacy_context
):
_, grants, recorders, _ = traces
class Child(Flow):
@start()
def work(self):
return "child output"
class Parent(Flow):
@start()
async def review(self):
await Child(tracing=True).kickoff_async()
assert crewai_event_bus.flush()
context = PendingFeedbackContext(
flow_id=self.flow_id,
flow_class="Parent",
method_name="review",
method_output="draft",
message="Approve draft",
execution_uuid=get_execution_uuid(),
)
self._pending_feedback_context = context
raise HumanFeedbackPending(context)
@listen(review)
def finish(self, feedback):
return feedback.feedback
persistence = JsonPersistence()
parent = Parent(tracing=True, persistence=persistence)
consent = Mock(return_value=True)
with trace_consent(consent):
paused = parent.kickoff()
assert isinstance(paused, HumanFeedbackPending)
paused_spans = recorders[0].get_finished_spans()
parent_span = next(
span
for span in paused_spans
if span.name == "execute flow"
and span.attributes["crewai.flow.id"] == parent.flow_id
)
child_span = next(
span
for span in paused_spans
if span.name == "execute flow" and span is not parent_span
)
_, restored_context = persistence.load_pending_feedback(parent.flow_id)
assert restored_context.trace_context == (
parent_span.context.trace_id,
parent_span.context.span_id,
)
assert restored_context.trace_context != (
child_span.context.trace_id,
child_span.context.span_id,
)
if legacy_context:
data = json.loads(persistence.saved[parent.flow_id])
data["context"].pop("trace_context")
persistence.saved[parent.flow_id] = json.dumps(data)
resumed = Parent.from_pending(
parent.flow_id, persistence=persistence, tracing=True
)
assert resumed.resume("Approved by reviewer") == "Approved by reviewer"
assert len(grants) == len(recorders) == 2
assert consent.call_count == 2
resumed_root = next(
span for span in recorders[1].get_finished_spans() if span.parent is None
)
assert resumed_root.name == "execute flow"
assert "Approved by reviewer" in resumed_root.attributes["gen_ai.input.messages"]
if legacy_context:
assert not resumed_root.links
else:
(link,) = resumed_root.links
assert link.context.trace_id == parent_span.context.trace_id
assert link.context.span_id == parent_span.context.span_id
assert link.attributes["crewai.link.type"] == "follows_from"