Framework for orchestrating role-playing, autonomous AI agents. By fostering collaborative intelligence, CrewAI empowers agents to work together seamlessly, tackling complex tasks.
from typing import Any
from unittest.mock import patch
import pytest
from pydantic import BaseModel
from crewai.events.types.llm_events import LLMCallCompletedEvent, LLMCallType
from crewai.llm import LLM
from crewai.llms.base_llm import BaseLLM
class TestLLMCallCompletedEventUsageField:
def test_accepts_usage_dict(self):
event = LLMCallCompletedEvent(
response="hello",
call_type=LLMCallType.LLM_CALL,
call_id="test-id",
usage={"prompt_tokens": 10, "completion_tokens": 20, "total_tokens": 30},
)
assert event.usage == {
"prompt_tokens": 10,
"completion_tokens": 20,
"total_tokens": 30,
}
def test_usage_defaults_to_none(self):
event = LLMCallCompletedEvent(
response="hello",
call_type=LLMCallType.LLM_CALL,
call_id="test-id",
)
assert event.usage is None
def test_accepts_none_usage(self):
event = LLMCallCompletedEvent(
response="hello",
call_type=LLMCallType.LLM_CALL,
call_id="test-id",
usage=None,
)
assert event.usage is None
def test_accepts_nested_usage_dict(self):
usage = {
"prompt_tokens": 100,
"completion_tokens": 200,
"total_tokens": 300,
"prompt_tokens_details": {"cached_tokens": 50},
}
event = LLMCallCompletedEvent(
response="hello",
call_type=LLMCallType.LLM_CALL,
call_id="test-id",
usage=usage,
)
assert event.usage["prompt_tokens_details"]["cached_tokens"] == 50
class TestUsageToDict:
def test_none_returns_none(self):
assert LLM._usage_to_dict(None) is None
def test_dict_without_nested_shapes_is_returned_unchanged(self):
usage = {"prompt_tokens": 10, "total_tokens": 30}
result = LLM._usage_to_dict(usage)
assert result == usage
# The input dict is copied, not mutated, so derived keys are not added.
assert "cached_prompt_tokens" not in result
@pytest.mark.parametrize(
("usage", "expected"),
[
pytest.param(
{"prompt_tokens": 100, "prompt_tokens_details": {"cached_tokens": 40}},
{"cached_prompt_tokens": 40},
id="openai-nested-cached-tokens",
),
pytest.param(
{"prompt_tokens": 100, "cached_tokens": 30},
{"cached_prompt_tokens": 30},
id="flat-cached-tokens",
),
pytest.param(
{"input_tokens": 100, "cache_read_input_tokens": 25},
{"cached_prompt_tokens": 25},
id="anthropic-cache-read-input-tokens",
),
pytest.param(
{
"completion_tokens": 200,
"completion_tokens_details": {"reasoning_tokens": 60},
},
{"reasoning_tokens": 60},
id="openai-nested-reasoning-tokens",
),
pytest.param(
{"input_tokens": 100, "cache_creation_input_tokens": 70},
{"cache_creation_tokens": 70},
id="anthropic-cache-creation-input-tokens",
),
pytest.param(
{
"prompt_tokens": 100,
"completion_tokens": 200,
"prompt_tokens_details": {"cached_tokens": 40},
"completion_tokens_details": {"reasoning_tokens": 60},
"cache_creation_input_tokens": 10,
},
{
"cached_prompt_tokens": 40,
"reasoning_tokens": 60,
"cache_creation_tokens": 10,
},
id="all-buckets-from-nested-shapes",
),
],
)
def test_normalizes_nested_litellm_buckets(self, usage, expected):
result = LLM._usage_to_dict(usage)
for key, value in expected.items():
assert result[key] == value
def test_does_not_alter_core_token_counts(self):
usage = {
"prompt_tokens": 100,
"completion_tokens": 200,
"total_tokens": 300,
"prompt_tokens_details": {"cached_tokens": 40},
}
result = LLM._usage_to_dict(usage)
assert result["prompt_tokens"] == 100
assert result["completion_tokens"] == 200
assert result["total_tokens"] == 300
def test_absent_buckets_are_not_added(self):
usage = {"prompt_tokens": 100, "completion_tokens": 200, "total_tokens": 300}
result = LLM._usage_to_dict(usage)
assert "cached_prompt_tokens" not in result
assert "reasoning_tokens" not in result
assert "cache_creation_tokens" not in result
def test_pydantic_model_uses_model_dump(self):
class Usage(BaseModel):
prompt_tokens: int = 10
completion_tokens: int = 20
total_tokens: int = 30
result = LLM._usage_to_dict(Usage())
assert result == {
"prompt_tokens": 10,
"completion_tokens": 20,
"total_tokens": 30,
}
def test_object_with_dict_attr(self):
class UsageObj:
def __init__(self):
self.prompt_tokens = 5
self.completion_tokens = 15
self.total_tokens = 20
result = LLM._usage_to_dict(UsageObj())
assert result == {
"prompt_tokens": 5,
"completion_tokens": 15,
"total_tokens": 20,
}
def test_object_with_dict_excludes_private_attrs(self):
class UsageObj:
def __init__(self):
self.total_tokens = 42
self._internal = "hidden"
result = LLM._usage_to_dict(UsageObj())
assert result == {"total_tokens": 42}
assert "_internal" not in result
def test_unsupported_type_returns_none(self):
assert LLM._usage_to_dict(42) is None
assert LLM._usage_to_dict("string") is None
class _StubLLM(BaseLLM):
"""Minimal concrete BaseLLM for testing event emission."""
model: str = "test-model"
def call(self, *args: Any, **kwargs: Any) -> str:
return ""
async def acall(self, *args: Any, **kwargs: Any) -> str:
return ""
def supports_function_calling(self) -> bool:
return False
def supports_stop_words(self) -> bool:
return True
class TestEmitCallCompletedEventPassesUsage:
@pytest.fixture
def mock_emit(self):
from crewai.events.event_bus import crewai_event_bus
with patch.object(crewai_event_bus, "emit") as mock:
yield mock
@pytest.fixture
def llm(self):
return _StubLLM(model="test-model")
def test_usage_is_passed_to_event(self, mock_emit, llm):
usage_data = {"prompt_tokens": 10, "completion_tokens": 20, "total_tokens": 30}
llm._emit_call_completed_event(
response="hello",
call_type=LLMCallType.LLM_CALL,
messages="test prompt",
usage=usage_data,
)
mock_emit.assert_called_once()
event = mock_emit.call_args[1]["event"]
assert isinstance(event, LLMCallCompletedEvent)
assert event.usage == usage_data
def test_none_usage_is_passed_to_event(self, mock_emit, llm):
llm._emit_call_completed_event(
response="hello",
call_type=LLMCallType.LLM_CALL,
messages="test prompt",
usage=None,
)
mock_emit.assert_called_once()
event = mock_emit.call_args[1]["event"]
assert isinstance(event, LLMCallCompletedEvent)
assert event.usage is None
def test_usage_omitted_defaults_to_none(self, mock_emit, llm):
llm._emit_call_completed_event(
response="hello",
call_type=LLMCallType.LLM_CALL,
messages="test prompt",
)
mock_emit.assert_called_once()
event = mock_emit.call_args[1]["event"]
assert isinstance(event, LLMCallCompletedEvent)
assert event.usage is None
class TestUsageMetricsNewFields:
def test_add_usage_metrics_aggregates_reasoning_and_cache_creation(self):
from crewai.types.usage_metrics import UsageMetrics
metrics1 = UsageMetrics(
total_tokens=100,
prompt_tokens=60,
completion_tokens=40,
cached_prompt_tokens=10,
reasoning_tokens=15,
cache_creation_tokens=5,
successful_requests=1,
)
metrics2 = UsageMetrics(
total_tokens=200,
prompt_tokens=120,
completion_tokens=80,
cached_prompt_tokens=20,
reasoning_tokens=25,
cache_creation_tokens=10,
successful_requests=1,
)
metrics1.add_usage_metrics(metrics2)
assert metrics1.total_tokens == 300
assert metrics1.prompt_tokens == 180
assert metrics1.completion_tokens == 120
assert metrics1.cached_prompt_tokens == 30
assert metrics1.reasoning_tokens == 40
assert metrics1.cache_creation_tokens == 15
assert metrics1.successful_requests == 2
def test_new_fields_default_to_zero(self):
from crewai.types.usage_metrics import UsageMetrics
metrics = UsageMetrics()
assert metrics.reasoning_tokens == 0
assert metrics.cache_creation_tokens == 0
def test_model_dump_includes_new_fields(self):
from crewai.types.usage_metrics import UsageMetrics
metrics = UsageMetrics(reasoning_tokens=10, cache_creation_tokens=5)
dumped = metrics.model_dump()
assert dumped["reasoning_tokens"] == 10
assert dumped["cache_creation_tokens"] == 5
class TestFromProviderDictAnthropicCacheTokens:
def test_cache_read_tokens_included_in_prompt_and_total(self):
from crewai.types.usage_metrics import UsageMetrics
metrics = UsageMetrics.from_provider_dict(
{
"input_tokens": 3,
"output_tokens": 44,
"cache_read_input_tokens": 2061,
}
)
assert metrics is not None
assert metrics.prompt_tokens == 2064
assert metrics.completion_tokens == 44
assert metrics.total_tokens == 2108
assert metrics.cached_prompt_tokens == 2061
def test_cache_creation_tokens_included_in_prompt_and_total(self):
from crewai.types.usage_metrics import UsageMetrics
metrics = UsageMetrics.from_provider_dict(
{
"input_tokens": 100,
"output_tokens": 50,
"cache_creation_input_tokens": 20,
}
)
assert metrics is not None
assert metrics.prompt_tokens == 120
assert metrics.total_tokens == 170
assert metrics.cache_creation_tokens == 20
def test_cache_read_and_creation_tokens_both_included(self):
from crewai.types.usage_metrics import UsageMetrics
metrics = UsageMetrics.from_provider_dict(
{
"input_tokens": 100,
"output_tokens": 50,
"cache_read_input_tokens": 30,
"cache_creation_input_tokens": 20,
}
)
assert metrics is not None
assert metrics.prompt_tokens == 150
assert metrics.total_tokens == 200
assert metrics.cached_prompt_tokens == 30
assert metrics.cache_creation_tokens == 20
def test_missing_cache_fields_preserve_non_cached_totals(self):
from crewai.types.usage_metrics import UsageMetrics
metrics = UsageMetrics.from_provider_dict(
{"input_tokens": 100, "output_tokens": 50}
)
assert metrics is not None
assert metrics.prompt_tokens == 100
assert metrics.total_tokens == 150
assert metrics.cached_prompt_tokens == 0
assert metrics.cache_creation_tokens == 0
def test_reconciled_native_dict_is_not_double_counted(self):
from crewai.types.usage_metrics import UsageMetrics
metrics = UsageMetrics.from_provider_dict(
{
"input_tokens": 150,
"output_tokens": 50,
"cached_prompt_tokens": 30,
"cache_creation_tokens": 20,
}
)
assert metrics is not None
assert metrics.prompt_tokens == 150
assert metrics.total_tokens == 200
assert metrics.cache_creation_tokens == 20
def test_openai_cached_prompt_tokens_are_not_added_twice(self):
from crewai.types.usage_metrics import UsageMetrics
metrics = UsageMetrics.from_provider_dict(
{
"prompt_tokens": 100,
"completion_tokens": 50,
"prompt_tokens_details": {"cached_tokens": 30},
}
)
assert metrics is not None
assert metrics.prompt_tokens == 100
assert metrics.total_tokens == 150
assert metrics.cached_prompt_tokens == 30
def test_cumulative_usage_via_add_usage_metrics(self):
from crewai.types.usage_metrics import UsageMetrics
first = UsageMetrics.from_provider_dict(
{
"input_tokens": 100,
"output_tokens": 50,
"cache_read_input_tokens": 30,
}
)
second = UsageMetrics.from_provider_dict(
{
"input_tokens": 40,
"output_tokens": 20,
}
)
assert first is not None and second is not None
first.add_usage_metrics(second)
assert first.prompt_tokens == 170
assert first.completion_tokens == 70
assert first.total_tokens == 240
assert first.cached_prompt_tokens == 30
assert first.successful_requests == 2