Skip to content

Framework for orchestrating role-playing, autonomous AI agents. By fostering collaborative intelligence, CrewAI empowers agents to work together seamlessly, tackling complex tasks.

13.8 KB
Raw Blame
from typing import Any
from unittest.mock import patch

import pytest
from pydantic import BaseModel

from crewai.events.types.llm_events import LLMCallCompletedEvent, LLMCallType
from crewai.llm import LLM
from crewai.llms.base_llm import BaseLLM


class TestLLMCallCompletedEventUsageField:
    def test_accepts_usage_dict(self):
        event = LLMCallCompletedEvent(
            response="hello",
            call_type=LLMCallType.LLM_CALL,
            call_id="test-id",
            usage={"prompt_tokens": 10, "completion_tokens": 20, "total_tokens": 30},
        )
        assert event.usage == {
            "prompt_tokens": 10,
            "completion_tokens": 20,
            "total_tokens": 30,
        }

    def test_usage_defaults_to_none(self):
        event = LLMCallCompletedEvent(
            response="hello",
            call_type=LLMCallType.LLM_CALL,
            call_id="test-id",
        )
        assert event.usage is None

    def test_accepts_none_usage(self):
        event = LLMCallCompletedEvent(
            response="hello",
            call_type=LLMCallType.LLM_CALL,
            call_id="test-id",
            usage=None,
        )
        assert event.usage is None

    def test_accepts_nested_usage_dict(self):
        usage = {
            "prompt_tokens": 100,
            "completion_tokens": 200,
            "total_tokens": 300,
            "prompt_tokens_details": {"cached_tokens": 50},
        }
        event = LLMCallCompletedEvent(
            response="hello",
            call_type=LLMCallType.LLM_CALL,
            call_id="test-id",
            usage=usage,
        )
        assert event.usage["prompt_tokens_details"]["cached_tokens"] == 50


class TestUsageToDict:
    def test_none_returns_none(self):
        assert LLM._usage_to_dict(None) is None

    def test_dict_without_nested_shapes_is_returned_unchanged(self):
        usage = {"prompt_tokens": 10, "total_tokens": 30}
        result = LLM._usage_to_dict(usage)
        assert result == usage
        # The input dict is copied, not mutated, so derived keys are not added.
        assert "cached_prompt_tokens" not in result

    @pytest.mark.parametrize(
        ("usage", "expected"),
        [
            pytest.param(
                {"prompt_tokens": 100, "prompt_tokens_details": {"cached_tokens": 40}},
                {"cached_prompt_tokens": 40},
                id="openai-nested-cached-tokens",
            ),
            pytest.param(
                {"prompt_tokens": 100, "cached_tokens": 30},
                {"cached_prompt_tokens": 30},
                id="flat-cached-tokens",
            ),
            pytest.param(
                {"input_tokens": 100, "cache_read_input_tokens": 25},
                {"cached_prompt_tokens": 25},
                id="anthropic-cache-read-input-tokens",
            ),
            pytest.param(
                {
                    "completion_tokens": 200,
                    "completion_tokens_details": {"reasoning_tokens": 60},
                },
                {"reasoning_tokens": 60},
                id="openai-nested-reasoning-tokens",
            ),
            pytest.param(
                {"input_tokens": 100, "cache_creation_input_tokens": 70},
                {"cache_creation_tokens": 70},
                id="anthropic-cache-creation-input-tokens",
            ),
            pytest.param(
                {
                    "prompt_tokens": 100,
                    "completion_tokens": 200,
                    "prompt_tokens_details": {"cached_tokens": 40},
                    "completion_tokens_details": {"reasoning_tokens": 60},
                    "cache_creation_input_tokens": 10,
                },
                {
                    "cached_prompt_tokens": 40,
                    "reasoning_tokens": 60,
                    "cache_creation_tokens": 10,
                },
                id="all-buckets-from-nested-shapes",
            ),
        ],
    )
    def test_normalizes_nested_litellm_buckets(self, usage, expected):
        result = LLM._usage_to_dict(usage)
        for key, value in expected.items():
            assert result[key] == value

    def test_does_not_alter_core_token_counts(self):
        usage = {
            "prompt_tokens": 100,
            "completion_tokens": 200,
            "total_tokens": 300,
            "prompt_tokens_details": {"cached_tokens": 40},
        }
        result = LLM._usage_to_dict(usage)
        assert result["prompt_tokens"] == 100
        assert result["completion_tokens"] == 200
        assert result["total_tokens"] == 300

    def test_absent_buckets_are_not_added(self):
        usage = {"prompt_tokens": 100, "completion_tokens": 200, "total_tokens": 300}
        result = LLM._usage_to_dict(usage)
        assert "cached_prompt_tokens" not in result
        assert "reasoning_tokens" not in result
        assert "cache_creation_tokens" not in result

    def test_pydantic_model_uses_model_dump(self):
        class Usage(BaseModel):
            prompt_tokens: int = 10
            completion_tokens: int = 20
            total_tokens: int = 30

        result = LLM._usage_to_dict(Usage())
        assert result == {
            "prompt_tokens": 10,
            "completion_tokens": 20,
            "total_tokens": 30,
        }

    def test_object_with_dict_attr(self):
        class UsageObj:
            def __init__(self):
                self.prompt_tokens = 5
                self.completion_tokens = 15
                self.total_tokens = 20

        result = LLM._usage_to_dict(UsageObj())
        assert result == {
            "prompt_tokens": 5,
            "completion_tokens": 15,
            "total_tokens": 20,
        }

    def test_object_with_dict_excludes_private_attrs(self):
        class UsageObj:
            def __init__(self):
                self.total_tokens = 42
                self._internal = "hidden"

        result = LLM._usage_to_dict(UsageObj())
        assert result == {"total_tokens": 42}
        assert "_internal" not in result

    def test_unsupported_type_returns_none(self):
        assert LLM._usage_to_dict(42) is None
        assert LLM._usage_to_dict("string") is None


class _StubLLM(BaseLLM):
    """Minimal concrete BaseLLM for testing event emission."""

    model: str = "test-model"

    def call(self, *args: Any, **kwargs: Any) -> str:
        return ""

    async def acall(self, *args: Any, **kwargs: Any) -> str:
        return ""

    def supports_function_calling(self) -> bool:
        return False

    def supports_stop_words(self) -> bool:
        return True


class TestEmitCallCompletedEventPassesUsage:
    @pytest.fixture
    def mock_emit(self):
        from crewai.events.event_bus import crewai_event_bus

        with patch.object(crewai_event_bus, "emit") as mock:
            yield mock

    @pytest.fixture
    def llm(self):
        return _StubLLM(model="test-model")

    def test_usage_is_passed_to_event(self, mock_emit, llm):
        usage_data = {"prompt_tokens": 10, "completion_tokens": 20, "total_tokens": 30}

        llm._emit_call_completed_event(
            response="hello",
            call_type=LLMCallType.LLM_CALL,
            messages="test prompt",
            usage=usage_data,
        )

        mock_emit.assert_called_once()
        event = mock_emit.call_args[1]["event"]
        assert isinstance(event, LLMCallCompletedEvent)
        assert event.usage == usage_data

    def test_none_usage_is_passed_to_event(self, mock_emit, llm):
        llm._emit_call_completed_event(
            response="hello",
            call_type=LLMCallType.LLM_CALL,
            messages="test prompt",
            usage=None,
        )

        mock_emit.assert_called_once()
        event = mock_emit.call_args[1]["event"]
        assert isinstance(event, LLMCallCompletedEvent)
        assert event.usage is None

    def test_usage_omitted_defaults_to_none(self, mock_emit, llm):
        llm._emit_call_completed_event(
            response="hello",
            call_type=LLMCallType.LLM_CALL,
            messages="test prompt",
        )

        mock_emit.assert_called_once()
        event = mock_emit.call_args[1]["event"]
        assert isinstance(event, LLMCallCompletedEvent)
        assert event.usage is None

class TestUsageMetricsNewFields:
    def test_add_usage_metrics_aggregates_reasoning_and_cache_creation(self):
        from crewai.types.usage_metrics import UsageMetrics

        metrics1 = UsageMetrics(
            total_tokens=100,
            prompt_tokens=60,
            completion_tokens=40,
            cached_prompt_tokens=10,
            reasoning_tokens=15,
            cache_creation_tokens=5,
            successful_requests=1,
        )
        metrics2 = UsageMetrics(
            total_tokens=200,
            prompt_tokens=120,
            completion_tokens=80,
            cached_prompt_tokens=20,
            reasoning_tokens=25,
            cache_creation_tokens=10,
            successful_requests=1,
        )

        metrics1.add_usage_metrics(metrics2)

        assert metrics1.total_tokens == 300
        assert metrics1.prompt_tokens == 180
        assert metrics1.completion_tokens == 120
        assert metrics1.cached_prompt_tokens == 30
        assert metrics1.reasoning_tokens == 40
        assert metrics1.cache_creation_tokens == 15
        assert metrics1.successful_requests == 2

    def test_new_fields_default_to_zero(self):
        from crewai.types.usage_metrics import UsageMetrics

        metrics = UsageMetrics()
        assert metrics.reasoning_tokens == 0
        assert metrics.cache_creation_tokens == 0

    def test_model_dump_includes_new_fields(self):
        from crewai.types.usage_metrics import UsageMetrics

        metrics = UsageMetrics(reasoning_tokens=10, cache_creation_tokens=5)
        dumped = metrics.model_dump()
        assert dumped["reasoning_tokens"] == 10
        assert dumped["cache_creation_tokens"] == 5


class TestFromProviderDictAnthropicCacheTokens:
    def test_cache_read_tokens_included_in_prompt_and_total(self):
        from crewai.types.usage_metrics import UsageMetrics

        metrics = UsageMetrics.from_provider_dict(
            {
                "input_tokens": 3,
                "output_tokens": 44,
                "cache_read_input_tokens": 2061,
            }
        )

        assert metrics is not None
        assert metrics.prompt_tokens == 2064
        assert metrics.completion_tokens == 44
        assert metrics.total_tokens == 2108
        assert metrics.cached_prompt_tokens == 2061

    def test_cache_creation_tokens_included_in_prompt_and_total(self):
        from crewai.types.usage_metrics import UsageMetrics

        metrics = UsageMetrics.from_provider_dict(
            {
                "input_tokens": 100,
                "output_tokens": 50,
                "cache_creation_input_tokens": 20,
            }
        )

        assert metrics is not None
        assert metrics.prompt_tokens == 120
        assert metrics.total_tokens == 170
        assert metrics.cache_creation_tokens == 20

    def test_cache_read_and_creation_tokens_both_included(self):
        from crewai.types.usage_metrics import UsageMetrics

        metrics = UsageMetrics.from_provider_dict(
            {
                "input_tokens": 100,
                "output_tokens": 50,
                "cache_read_input_tokens": 30,
                "cache_creation_input_tokens": 20,
            }
        )

        assert metrics is not None
        assert metrics.prompt_tokens == 150
        assert metrics.total_tokens == 200
        assert metrics.cached_prompt_tokens == 30
        assert metrics.cache_creation_tokens == 20

    def test_missing_cache_fields_preserve_non_cached_totals(self):
        from crewai.types.usage_metrics import UsageMetrics

        metrics = UsageMetrics.from_provider_dict(
            {"input_tokens": 100, "output_tokens": 50}
        )

        assert metrics is not None
        assert metrics.prompt_tokens == 100
        assert metrics.total_tokens == 150
        assert metrics.cached_prompt_tokens == 0
        assert metrics.cache_creation_tokens == 0

    def test_reconciled_native_dict_is_not_double_counted(self):
        from crewai.types.usage_metrics import UsageMetrics

        metrics = UsageMetrics.from_provider_dict(
            {
                "input_tokens": 150,
                "output_tokens": 50,
                "cached_prompt_tokens": 30,
                "cache_creation_tokens": 20,
            }
        )

        assert metrics is not None
        assert metrics.prompt_tokens == 150
        assert metrics.total_tokens == 200
        assert metrics.cache_creation_tokens == 20

    def test_openai_cached_prompt_tokens_are_not_added_twice(self):
        from crewai.types.usage_metrics import UsageMetrics

        metrics = UsageMetrics.from_provider_dict(
            {
                "prompt_tokens": 100,
                "completion_tokens": 50,
                "prompt_tokens_details": {"cached_tokens": 30},
            }
        )

        assert metrics is not None
        assert metrics.prompt_tokens == 100
        assert metrics.total_tokens == 150
        assert metrics.cached_prompt_tokens == 30

    def test_cumulative_usage_via_add_usage_metrics(self):
        from crewai.types.usage_metrics import UsageMetrics

        first = UsageMetrics.from_provider_dict(
            {
                "input_tokens": 100,
                "output_tokens": 50,
                "cache_read_input_tokens": 30,
            }
        )
        second = UsageMetrics.from_provider_dict(
            {
                "input_tokens": 40,
                "output_tokens": 20,
            }
        )

        assert first is not None and second is not None
        first.add_usage_metrics(second)

        assert first.prompt_tokens == 170
        assert first.completion_tokens == 70
        assert first.total_tokens == 240
        assert first.cached_prompt_tokens == 30
        assert first.successful_requests == 2