AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.
test(platform/admin): add cache token and retry cost accumulation tests
Zamil Majdyaef9298be69a
diff --git a/autogpt_platform/backend/backend/blocks/test/test_llm.py b/autogpt_platform/backend/backend/blocks/test/test_llm.pyindex 2f12d6ab6e..aa654ac98b 100644--- a/autogpt_platform/backend/backend/blocks/test/test_llm.py+++ b/autogpt_platform/backend/backend/blocks/test/test_llm.py@@ -46,6 +46,56 @@ class TestLLMStatsTracking:assert response.completion_tokens == 20assert response.response == "Test response"+ @pytest.mark.asyncio+ async def test_llm_call_anthropic_returns_cache_tokens(self):+ """Test that llm_call returns cache read/creation tokens from Anthropic."""+ from pydantic import SecretStr++ import backend.blocks.llm as llm+ from backend.data.model import APIKeyCredentials++ anthropic_creds = APIKeyCredentials(+ id="test-anthropic-id",+ provider="anthropic",+ api_key=SecretStr("mock-anthropic-key"),+ title="Mock Anthropic key",+ expires_at=None,+ )++ mock_content_block = MagicMock()+ mock_content_block.type = "text"+ mock_content_block.text = "Test anthropic response"++ mock_usage = MagicMock()+ mock_usage.input_tokens = 15+ mock_usage.output_tokens = 25+ mock_usage.cache_read_input_tokens = 100+ mock_usage.cache_creation_input_tokens = 50++ mock_response = MagicMock()+ mock_response.content = [mock_content_block]+ mock_response.usage = mock_usage+ mock_response.stop_reason = "end_turn"++ with patch("anthropic.AsyncAnthropic") as mock_anthropic:+ mock_client = AsyncMock()+ mock_anthropic.return_value = mock_client+ mock_client.messages.create = AsyncMock(return_value=mock_response)++ response = await llm.llm_call(+ credentials=anthropic_creds,+ llm_model=llm.LlmModel.CLAUDE_3_HAIKU,+ prompt=[{"role": "user", "content": "Hello"}],+ max_tokens=100,+ )++ assert isinstance(response, llm.LLMResponse)+ assert response.prompt_tokens == 15+ assert response.completion_tokens == 25+ assert response.cache_read_tokens == 100+ assert response.cache_creation_tokens == 50+ assert response.response == "Test anthropic response"+@pytest.mark.asyncioasync def test_ai_structured_response_block_tracks_stats(self):"""Test that AIStructuredResponseGeneratorBlock correctly tracks stats."""@@ -258,6 +308,80 @@ class TestLLMStatsTracking:assert block.execution_stats.input_token_count == 30assert block.execution_stats.output_token_count == 15+ @pytest.mark.asyncio+ async def test_cache_tokens_accumulated_in_stats(self):+ """Cache read/creation tokens are tracked per-attempt and accumulated."""+ import backend.blocks.llm as llm++ block = llm.AIStructuredResponseGeneratorBlock()++ async def mock_llm_call(*args, **kwargs):+ return llm.LLMResponse(+ raw_response="",+ prompt=[],+ response='<json_output id="tok123456">{"key1": "v1", "key2": "v2"}</json_output>',+ tool_calls=None,+ prompt_tokens=10,+ completion_tokens=5,+ cache_read_tokens=20,+ cache_creation_tokens=8,+ reasoning=None,+ provider_cost=0.005,+ )++ block.llm_call = mock_llm_call # type: ignore++ input_data = llm.AIStructuredResponseGeneratorBlock.Input(+ prompt="Test prompt",+ expected_format={"key1": "desc1", "key2": "desc2"},+ model=llm.DEFAULT_LLM_MODEL,+ credentials=llm.TEST_CREDENTIALS_INPUT, # type: ignore+ retry=1,+ )++ with patch("secrets.token_hex", return_value="tok123456"):+ async for _ in block.run(input_data, credentials=llm.TEST_CREDENTIALS):+ pass++ assert block.execution_stats.cache_read_token_count == 20+ assert block.execution_stats.cache_creation_token_count == 8++ @pytest.mark.asyncio+ async def test_failure_path_persists_accumulated_cost(self):+ """When all retries are exhausted, accumulated provider_cost is preserved."""+ import backend.blocks.llm as llm++ block = llm.AIStructuredResponseGeneratorBlock()++ async def mock_llm_call(*args, **kwargs):+ return llm.LLMResponse(+ raw_response="",+ prompt=[],+ response="not valid json at all",+ tool_calls=None,+ prompt_tokens=10,+ completion_tokens=5,+ reasoning=None,+ provider_cost=0.01,+ )++ block.llm_call = mock_llm_call # type: ignore++ input_data = llm.AIStructuredResponseGeneratorBlock.Input(+ prompt="Test prompt",+ expected_format={"key1": "desc1"},+ model=llm.DEFAULT_LLM_MODEL,+ credentials=llm.TEST_CREDENTIALS_INPUT, # type: ignore+ retry=2,+ )++ with pytest.raises(RuntimeError):+ async for _ in block.run(input_data, credentials=llm.TEST_CREDENTIALS):+ pass++ # Both retry attempts each cost $0.01, total $0.02+ assert block.execution_stats.provider_cost == pytest.approx(0.02)+@pytest.mark.asyncioasync def test_ai_text_summarizer_multiple_chunks(self):"""Test that AITextSummarizerBlock correctly accumulates stats across multiple chunks."""diff --git a/autogpt_platform/backend/backend/data/platform_cost_integration_test.py b/autogpt_platform/backend/backend/data/platform_cost_integration_test.pyindex 10fe35d748..ef457a1105 100644--- a/autogpt_platform/backend/backend/data/platform_cost_integration_test.py+++ b/autogpt_platform/backend/backend/data/platform_cost_integration_test.py@@ -77,3 +77,25 @@ async def test_log_platform_cost_metadata_none(cost_log_user):rows = await PrismaLog.prisma().find_many(where={"userId": user_id})assert len(rows) == 1assert rows[0].metadata == {}++[email protected](loop_scope="session")+async def test_log_platform_cost_cache_tokens(cost_log_user):+ """Verify that cache_read_tokens and cache_creation_tokens are persisted."""+ user_id = cost_log_user+ entry = PlatformCostEntry(+ user_id=user_id,+ block_name="TestBlock",+ provider="anthropic",+ input_tokens=200,+ output_tokens=100,+ cache_read_tokens=50,+ cache_creation_tokens=25,+ model="claude-3-5-sonnet-20241022",+ )+ await log_platform_cost(entry)++ rows = await PrismaLog.prisma().find_many(where={"userId": user_id})+ assert len(rows) == 1+ assert rows[0].cacheReadTokens == 50+ assert rows[0].cacheCreationTokens == 25diff --git a/autogpt_platform/backend/backend/data/platform_cost_test.py b/autogpt_platform/backend/backend/data/platform_cost_test.pyindex 7c5b75e78a..65b1a20099 100644--- a/autogpt_platform/backend/backend/data/platform_cost_test.py+++ b/autogpt_platform/backend/backend/data/platform_cost_test.py@@ -17,9 +17,24 @@ from .platform_cost import (get_platform_cost_logs_for_export,log_platform_cost,log_platform_cost_safe,+ usd_to_microdollars,)+class TestUsdToMicrodollars:+ def test_none_returns_none(self):+ assert usd_to_microdollars(None) is None++ def test_zero_returns_zero(self):+ assert usd_to_microdollars(0.0) == 0++ def test_positive_value(self):+ assert usd_to_microdollars(0.001) == 1000++ def test_large_value(self):+ assert usd_to_microdollars(1.0) == 1_000_000++class TestMaskEmail:def test_typical_email(self):assert _mask_email("[email protected]") == "us***@example.com"@@ -335,6 +350,14 @@ class TestGetPlatformCostLogs:logs, total = await get_platform_cost_logs()assert total == 0+ @pytest.mark.asyncio+ async def test_explicit_start_skips_default(self):+ start = datetime(2026, 1, 1, tzinfo=timezone.utc)+ mock_query = AsyncMock(side_effect=[[{"cnt": 0}], []])+ with patch("backend.data.platform_cost.query_raw_with_schema", new=mock_query):+ logs, total = await get_platform_cost_logs(start=start)+ assert total == 0+def _make_log_row(i: int = 0) -> dict:return {@@ -409,3 +432,12 @@ class TestGetPlatformCostLogsForExport:logs, _ = await get_platform_cost_logs_for_export()assert logs[0].cache_read_tokens == 50assert logs[0].cache_creation_tokens == 25++ @pytest.mark.asyncio+ async def test_explicit_start_skips_default(self):+ start = datetime(2026, 1, 1, tzinfo=timezone.utc)+ mock_query = AsyncMock(return_value=[])+ with patch("backend.data.platform_cost.query_raw_with_schema", new=mock_query):+ logs, truncated = await get_platform_cost_logs_for_export(start=start)+ assert logs == []+ assert truncated is False