Framework for orchestrating role-playing, autonomous AI agents. By fostering collaborative intelligence, CrewAI empowers agents to work together seamlessly, tackling complex tasks.
"""The execute task, execute agent and call tool spans carry what a reader needs.
A consumer of a run's spans could see a task's raw text but not the format it
declared, nor whether a Pydantic object or a JSON dict actually came out of it;
it could see an agent's goal, backstory and model but not the prompt the agent
was handed or the answer it gave; and it could see a tool's result but not
whether the tool ran or the cache answered. Each had to be reconstructed from
other spans, or could not be known at all.
"""
from __future__ import annotations
from datetime import datetime, timezone
import json
from crewai import Agent, Crew, Task
from crewai.events.types.agent_events import (
AgentExecutionCompletedEvent,
AgentExecutionStartedEvent,
)
from crewai.events.types.task_events import (
TaskCompletedEvent,
TaskFailedEvent,
TaskStartedEvent,
)
from crewai.events.types.tool_usage_events import (
ToolUsageFinishedEvent,
ToolUsageStartedEvent,
)
from crewai.tasks.output_format import OutputFormat
from crewai.tasks.task_output import TaskOutput
from crewai.telemetry.tracing import handlers
from crewai.telemetry.tracing.context import TelemetryExecutionContext
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace.export import SimpleSpanProcessor
from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter
from pydantic import BaseModel
import pytest
# Long and non-repeating, so a truncated or elided copy cannot compare equal.
# Under the default attribute bound, so it must arrive whole.
LONG_TEXT = "".join(
f"paragraph {i}: the quick brown fox jumps over the lazy dog\n" for i in range(400)
)
assert 20_000 < len(LONG_TEXT.encode("utf-8")) < 32 * 1024
@pytest.fixture(autouse=True)
def enable_otel_sdk(monkeypatch: pytest.MonkeyPatch) -> None:
"""The suite otherwise runs with OTEL_SDK_DISABLED, which makes every
assertion here pass vacuously against non-recording spans."""
monkeypatch.delenv("OTEL_SDK_DISABLED", raising=False)
monkeypatch.delenv("CREWAI_DISABLE_TELEMETRY", raising=False)
monkeypatch.delenv("CREWAI_DISABLE_TRACKING", raising=False)
class _Providers:
"""All a handler asks of its providers: a tracer and somewhere to log."""
def __init__(self, tracer) -> None:
self._tracer = tracer
def get_tracer(self, name: str | None = None):
return self._tracer
def emit_log(self, *args, **kwargs) -> None:
pass
@pytest.fixture
def pipeline():
"""Handler inputs whose spans land in memory, exported as each one ends."""
exporter = InMemorySpanExporter()
provider = TracerProvider()
provider.add_span_processor(SimpleSpanProcessor(exporter))
tracer = provider.get_tracer("test")
ctx = TelemetryExecutionContext(
kickoff_id="kickoff", automation_name="test", tracer=tracer
)
# yield, not return: the frame keeps `provider` alive, and a collected
# provider shuts down its processor and silently loses spans.
yield _Providers(tracer), ctx, exporter
def _only_span(exporter: InMemorySpanExporter, name: str):
"""The single exported span with ``name``, failing loudly if not unique."""
matches = [s for s in exporter.get_finished_spans() if s.name == name]
assert len(matches) == 1, [s.name for s in exporter.get_finished_spans()]
return matches[0]
class Finding(BaseModel):
title: str
def _crewed_task(**declaration) -> Task:
"""A task whose agent belongs to a crew, as the task handlers require."""
agent = Agent(role="Researcher", goal="g", backstory="b")
task = Task(description="d", expected_output="e", agent=agent, **declaration)
crew = Crew(agents=[agent], tasks=[task])
# A kickoff attaches the crew to its agents; these events skip the kickoff.
agent.crew = crew
return task
@pytest.mark.parametrize(
("declaration", "produced", "expected"),
[
(
{"output_pydantic": Finding},
{"pydantic": Finding(title="t"), "output_format": OutputFormat.PYDANTIC},
("pydantic", True, False),
),
(
{"output_json": Finding},
{"json_dict": {"title": "t"}, "output_format": OutputFormat.JSON},
("json", False, True),
),
(
{},
{"output_format": OutputFormat.RAW},
("raw", False, False),
),
],
ids=["pydantic", "json", "raw"],
)
def test_execute_task_span_says_what_was_declared_and_what_came_out(
pipeline, declaration, produced, expected
) -> None:
providers, ctx, exporter = pipeline
task = _crewed_task(**declaration)
output = TaskOutput(
description="d", raw='{"title": "t"}', agent="Researcher", **produced
)
started = TaskStartedEvent(context=None, task=task)
handlers.handle_task_started(providers, ctx, task, started)
handlers.handle_task_completed(
providers,
ctx,
task,
TaskCompletedEvent(output=output, task=task, started_event_id=started.event_id),
)
span = _only_span(exporter, "execute task")
output_format, pydantic_produced, json_produced = expected
assert span.attributes["crewai.task.output_format"] == output_format
assert span.attributes["crewai.task.output_pydantic_produced"] is pydantic_produced
assert span.attributes["crewai.task.output_json_produced"] is json_produced
# The raw text is still there, as before.
assert span.attributes["crewai.task.output"] == '{"title": "t"}'
def test_a_declared_pydantic_task_that_produced_none_is_told_apart(pipeline) -> None:
"""The gap the two flags close: declared is not the same as produced."""
providers, ctx, exporter = pipeline
task = _crewed_task(output_pydantic=Finding)
# The producer could not convert, so the output carries only raw text.
output = TaskOutput(
description="d",
raw="not json at all",
agent="Researcher",
output_format=OutputFormat.PYDANTIC,
)
started = TaskStartedEvent(context=None, task=task)
handlers.handle_task_started(providers, ctx, task, started)
handlers.handle_task_completed(
providers,
ctx,
task,
TaskCompletedEvent(output=output, task=task, started_event_id=started.event_id),
)
span = _only_span(exporter, "execute task")
assert span.attributes["crewai.task.output_format"] == "pydantic"
assert span.attributes["crewai.task.output_pydantic_produced"] is False
assert span.attributes["crewai.task.output_json_produced"] is False
def test_a_failed_task_still_says_what_it_declared(pipeline) -> None:
"""There is no output to inspect, so the format comes from the declaration."""
providers, ctx, exporter = pipeline
task = _crewed_task(output_pydantic=Finding)
started = TaskStartedEvent(context=None, task=task)
handlers.handle_task_started(providers, ctx, task, started)
handlers.handle_task_failed(
providers,
ctx,
task,
TaskFailedEvent(error="boom", task=task, started_event_id=started.event_id),
)
span = _only_span(exporter, "execute task")
assert span.attributes["crewai.task.output_format"] == "pydantic"
assert "crewai.task.output_pydantic_produced" not in span.attributes
assert "crewai.task.output_json_produced" not in span.attributes
def _agent_span(pipeline, *, prompt: str, output: str):
providers, ctx, exporter = pipeline
agent = Agent(role="Researcher", goal="g", backstory="b")
started = AgentExecutionStartedEvent(
agent=agent, task=None, tools=None, task_prompt=prompt
)
handlers.handle_agent_execution_started(providers, ctx, agent, started)
# The completed handler waits (up to 5 s) for the agent's LLM-call count to
# settle; one recorded call is what a real run would have signalled.
handlers._record_agent_llm_call(ctx, str(agent.id))
handlers.handle_agent_execution_completed(
providers,
ctx,
agent,
AgentExecutionCompletedEvent(
agent=agent, task=None, output=output, started_event_id=started.event_id
),
)
return _only_span(exporter, "execute agent")
def test_execute_agent_span_carries_the_exact_prompt_and_answer(pipeline) -> None:
span = _agent_span(pipeline, prompt=LONG_TEXT, output="The fox is quick.")
# The same spec shape the task span already uses for its own text.
prompt = json.loads(span.attributes["gen_ai.input.messages"])
assert prompt[0]["parts"][0]["content"] == LONG_TEXT
assert "gen_ai.input.messages.truncated" not in span.attributes
answer = json.loads(span.attributes["gen_ai.output.messages"])
assert answer[0]["parts"][0]["content"] == "The fox is quick."
assert "gen_ai.output.messages.truncated" not in span.attributes
def test_the_agent_text_travels_under_the_standard_keys_only(pipeline) -> None:
"""The prompt and the answer leave under the two keys the ``call llm`` span
already uses for its messages, and under no other. Whatever rule an
exporter or a redaction processor applies to LLM message content applies
to these unchanged; a copy under a ``crewai.agent.*`` key would escape a
rule that selects by key name."""
span = _agent_span(pipeline, prompt="PROMPT-7597", output="ANSWER-7597")
carrying = {
key
for key, value in span.attributes.items()
if "PROMPT-7597" in str(value) or "ANSWER-7597" in str(value)
}
assert carrying == {"gen_ai.input.messages", "gen_ai.output.messages"}
def test_an_over_cap_prompt_is_declared_truncated_never_silently_cut(
pipeline, monkeypatch: pytest.MonkeyPatch
) -> None:
cap = 2048
monkeypatch.setenv("CREWAI_OTEL_MAX_ATTR_BYTES", str(cap))
span = _agent_span(pipeline, prompt=LONG_TEXT, output="short")
payload = span.attributes["gen_ai.input.messages"]
assert span.attributes["gen_ai.input.messages.truncated"] is True
assert span.attributes["gen_ai.input.messages.original_size_bytes"] > cap
assert len(payload.encode("utf-8")) <= cap
# The answer fit, so it arrives whole and unmarked.
answer = json.loads(span.attributes["gen_ai.output.messages"])
assert answer[0]["parts"][0]["content"] == "short"
assert "gen_ai.output.messages.truncated" not in span.attributes
@pytest.mark.parametrize("from_cache", [True, False], ids=["cached", "ran"])
def test_call_tool_span_says_whether_the_cache_answered(pipeline, from_cache) -> None:
providers, ctx, exporter = pipeline
now = datetime.now(timezone.utc)
started = ToolUsageStartedEvent(
tool_name="search", tool_args={"q": "fox"}, agent_key="k", agent_role="r"
)
handlers.handle_tool_usage_started(providers, ctx, None, started)
handlers.handle_tool_usage_finished(
providers,
ctx,
None,
ToolUsageFinishedEvent(
tool_name="search",
tool_args={"q": "fox"},
agent_key="k",
agent_role="r",
started_at=now,
finished_at=now,
output="found",
from_cache=from_cache,
started_event_id=started.event_id,
),
)
span = _only_span(exporter, "call tool")
assert span.attributes["crewai.tool.from_cache"] is from_cache