Build and run agents you can see, understand and trust.
Python agentchatbotlarge-language-modelsllmllm-agentmcpmulti-agentmulti-modalreact-agentrealtime-agentvoice-agentvoice-assistant
fix(agent): keep prompt cache tokens in usage (#2318)
yingz2363e64aefd25a
diff --git a/src/agentscope/agent/_agent.py b/src/agentscope/agent/_agent.pyindex b98333bb5..de04813d5 100644--- a/src/agentscope/agent/_agent.py+++ b/src/agentscope/agent/_agent.py@@ -1595,14 +1595,15 @@ class Agent:)# Send the model call ended event with usage if available+ usage = completed_response.usageyield ModelCallEndEvent(reply_id=self.state.reply_id,- input_tokens=completed_response.usage.input_tokens- if completed_response.usage- else 0,- output_tokens=completed_response.usage.output_tokens- if completed_response.usage- else 0,+ input_tokens=usage.input_tokens if usage else 0,+ output_tokens=usage.output_tokens if usage else 0,+ cache_input_tokens=(usage.cache_input_tokens or 0) if usage else 0,+ cache_creation_input_tokens=(+ (usage.cache_creation_input_tokens or 0) if usage else 0+ ),finished_reason=completed_response.finished_reason,)@@ -1633,6 +1634,12 @@ class Agent:Usage(input_tokens=last_ctx.usage.input_tokens,output_tokens=last_ctx.usage.output_tokens,+ cache_input_tokens=(+ last_ctx.usage.cache_input_tokens or 0+ ),+ cache_creation_input_tokens=(+ last_ctx.usage.cache_creation_input_tokens or 0+ ),)if last_ctx is not None and last_ctx.usage is not Noneelse None@@ -3189,6 +3196,10 @@ class Agent:Usage(input_tokens=usage.input_tokens,output_tokens=usage.output_tokens,+ cache_input_tokens=usage.cache_input_tokens or 0,+ cache_creation_input_tokens=(+ usage.cache_creation_input_tokens or 0+ ),)if usage is not Noneelse None@@ -3220,6 +3231,10 @@ class Agent:else:tail.usage.input_tokens += msg_usage.input_tokenstail.usage.output_tokens += msg_usage.output_tokens+ tail.usage.cache_input_tokens += msg_usage.cache_input_tokens+ tail.usage.cache_creation_input_tokens += (+ msg_usage.cache_creation_input_tokens+ )def _get_last_msg(self) -> Msg | None:"""Get the last message in the context that belongs to this agent."""diff --git a/src/agentscope/event/_event.py b/src/agentscope/event/_event.pyindex 774a60bcf..1203f5b40 100644--- a/src/agentscope/event/_event.py+++ b/src/agentscope/event/_event.py@@ -147,6 +147,10 @@ class ModelCallEndEvent(EventBase):"""Number of input tokens consumed."""output_tokens: int"""Number of output tokens generated."""+ cache_input_tokens: int = 0+ """Number of input tokens read from the prompt cache."""+ cache_creation_input_tokens: int = 0+ """Number of input tokens used to create the prompt cache."""finished_reason: FinishedReason = Field(default=FinishedReason.COMPLETED,)diff --git a/src/agentscope/message/_base.py b/src/agentscope/message/_base.pyindex 0a7239d08..0d51f35aa 100644--- a/src/agentscope/message/_base.py+++ b/src/agentscope/message/_base.py@@ -62,6 +62,10 @@ class Usage(BaseModel):"""The number of input tokens."""output_tokens: int"""The number of output tokens."""+ cache_input_tokens: int = 0+ """The number of input tokens read from the prompt cache."""+ cache_creation_input_tokens: int = 0+ """The number of input tokens used to create the prompt cache."""class Msg(BaseModel):@@ -283,10 +287,18 @@ class Msg(BaseModel):self.usage = Usage(input_tokens=event.input_tokens,output_tokens=event.output_tokens,+ cache_input_tokens=event.cache_input_tokens,+ cache_creation_input_tokens=(+ event.cache_creation_input_tokens+ ),)else:self.usage.input_tokens += event.input_tokensself.usage.output_tokens += event.output_tokens+ self.usage.cache_input_tokens += event.cache_input_tokens+ self.usage.cache_creation_input_tokens += (+ event.cache_creation_input_tokens+ )case EventType.TEXT_BLOCK_START:self.content.append(TextBlock(id=event.block_id, text=""))diff --git a/tests/agent_basic_test.py b/tests/agent_basic_test.pyindex b667eec9d..383653c7e 100644--- a/tests/agent_basic_test.py+++ b/tests/agent_basic_test.py@@ -6,7 +6,7 @@ from unittest.async_case import IsolatedAsyncioTestCasefrom utils import AnyString, MockModelfrom agentscope.agent import Agent, ContextConfig, InjectionConfig, ReActConfig-from agentscope.model import ChatResponse+from agentscope.model import ChatResponse, ChatUsagefrom agentscope.tool import (ToolBase,Toolkit,@@ -281,6 +281,8 @@ class AgentBasicTest(IsolatedAsyncioTestCase):"type": "MODEL_CALL_END","input_tokens": 0,"output_tokens": 0,+ "cache_input_tokens": 0,+ "cache_creation_input_tokens": 0,"finished_reason": "completed",},{@@ -493,6 +495,8 @@ class AgentBasicTest(IsolatedAsyncioTestCase):"type": "MODEL_CALL_END","input_tokens": 0,"output_tokens": 0,+ "cache_input_tokens": 0,+ "cache_creation_input_tokens": 0,"finished_reason": "completed",},{@@ -659,6 +663,193 @@ class AgentBasicTest(IsolatedAsyncioTestCase):context_dicts = [msg.model_dump() for msg in self.agent.state.context]self.assertListEqual(context_dicts, expected_context_after_reply)+ async def test_usage_cache_tokens_are_not_dropped(self) -> None:+ """Regression test for issue #2305.++ Prompt cache tokens (``cache_input_tokens`` and+ ``cache_creation_input_tokens``) parsed by the model adapter into+ ``ChatUsage`` must survive the agent layer:++ 1. ``ModelCallEndEvent`` must carry them so middleware reading the+ event can record cache hit rates.+ 2. ``_save_to_context`` must merge them into ``Msg.usage``.+ """+ self.model.set_responses(+ [+ ChatResponse(+ content=[TextBlock(text="Hello world!")],+ is_last=True,+ usage=ChatUsage(+ input_tokens=100,+ output_tokens=20,+ time=0.5,+ cache_input_tokens=60,+ cache_creation_input_tokens=40,+ ),+ ),+ ],+ )++ end_events = []+ async for event in self.agent.reply_stream(+ UserMsg(name="user", content="Hi"),+ ):+ if event.type == "MODEL_CALL_END":+ end_events.append(event)++ # 1) The end event must carry the cache tokens.+ self.assertEqual(len(end_events), 1)+ self.assertEqual(+ end_events[0].model_dump(),+ {+ **self._get_event_base(self.agent.state.reply_id),+ "type": "MODEL_CALL_END",+ "input_tokens": 100,+ "output_tokens": 20,+ "cache_input_tokens": 60,+ "cache_creation_input_tokens": 40,+ "finished_reason": "completed",+ },+ )++ # 2) The saved assistant message must expose the cache tokens in its+ # usage.+ assistant_msgs = [+ msg for msg in self.agent.state.context if msg.role == "assistant"+ ]+ self.assertEqual(len(assistant_msgs), 1)+ usage = assistant_msgs[0].usage+ self.assertIsNotNone(usage)+ assert usage is not None+ self.assertEqual(+ usage.model_dump(),+ {+ "input_tokens": 100,+ "output_tokens": 20,+ "cache_input_tokens": 60,+ "cache_creation_input_tokens": 40,+ },+ )++ async def test_usage_cache_tokens_none_normalized_to_zero(self) -> None:+ """Regression test: None-valued cache fields must not raise Pydantic.++ ChatUsage is a plain dataclass that does not enforce int annotations at+ runtime. Adapters copy optional SDK fields with getattr(x, 0),+ which can still return None when the attribute exists but is null.+ None-valued cache fields must be normalized to zero before+ entering any Pydantic model.+ """+ usage_with_none_cache = ChatUsage(+ input_tokens=10,+ output_tokens=5,+ time=0.1,+ )+ # Manually assign None to the two cache fields (dataclass will not+ # complain at runtime because there is no runtime type check).+ usage_with_none_cache.cache_input_tokens = None+ usage_with_none_cache.cache_creation_input_tokens = None++ self.model.set_responses(+ [+ ChatResponse(+ content=[TextBlock(text="Hi")],+ is_last=True,+ usage=usage_with_none_cache,+ ),+ ],+ )++ end_events = []+ # Must not raise Pydantic ValidationError; reply completes cleanly.+ async for event in self.agent.reply_stream(+ UserMsg(name="user", content="Hello"),+ ):+ if event.type == "MODEL_CALL_END":+ end_events.append(event)++ self.assertEqual(len(end_events), 1)+ self.assertEqual(+ end_events[0].model_dump(),+ {+ **self._get_event_base(self.agent.state.reply_id),+ "type": "MODEL_CALL_END",+ "input_tokens": 10,+ "output_tokens": 5,+ # None must be normalized to 0.+ "cache_input_tokens": 0,+ "cache_creation_input_tokens": 0,+ "finished_reason": "completed",+ },+ )++ assistant_msgs = [+ msg for msg in self.agent.state.context if msg.role == "assistant"+ ]+ self.assertEqual(len(assistant_msgs), 1)+ usage = assistant_msgs[0].usage+ self.assertIsNotNone(usage)+ assert usage is not None+ # None from dataclass must also be normalized to 0 in Msg.usage.+ self.assertEqual(+ usage.model_dump(),+ {+ "input_tokens": 10,+ "output_tokens": 5,+ "cache_input_tokens": 0,+ "cache_creation_input_tokens": 0,+ },+ )++ async def test_usage_cache_tokens_in_reply_return(self) -> None:+ """Regression test: Agent.reply() result must keep the cache tokens.++ ``final_usage`` is rebuilt from the saved context message when+ returning the final ``AssistantMsg``. It must copy both cache+ fields, otherwise the returned message silently reports zeros even+ though the saved context carried the correct values.+ """+ self.model.set_responses(+ [+ ChatResponse(+ content=[TextBlock(text="Hello world!")],+ is_last=True,+ usage=ChatUsage(+ input_tokens=100,+ output_tokens=20,+ time=0.5,+ cache_input_tokens=60,+ cache_creation_input_tokens=40,+ ),+ ),+ ],+ )++ msg = await self.agent.reply(UserMsg(name="user", content="Hi"))++ self.assertDictEqual(+ msg.model_dump(),+ {+ **self._get_msg_base(),+ "content": [+ {+ "type": "text",+ "created_at": AnyString(),+ "finished_at": None,+ "id": AnyString(),+ "text": "Hello world!",+ },+ ],+ "finished_reason": ReplyFinishedReason.COMPLETED,+ "usage": {+ "input_tokens": 100,+ "output_tokens": 20,+ "cache_input_tokens": 60,+ "cache_creation_input_tokens": 40,+ },+ },+ )+async def test_max_iters_counts_reasoning_acting_round_once(self) -> None:"""A tool round consumes one iteration before final reasoning."""self.agent.toolkit = Toolkit(tools=[MockSequentialTool()])@@ -923,6 +1114,8 @@ class AgentBasicTest(IsolatedAsyncioTestCase):"type": "MODEL_CALL_END","input_tokens": 0,"output_tokens": 0,+ "cache_input_tokens": 0,+ "cache_creation_input_tokens": 0,"finished_reason": "completed",},{@@ -967,6 +1160,8 @@ class AgentBasicTest(IsolatedAsyncioTestCase):"type": "MODEL_CALL_END","input_tokens": 0,"output_tokens": 0,+ "cache_input_tokens": 0,+ "cache_creation_input_tokens": 0,"finished_reason": "completed",},{@@ -1169,6 +1364,8 @@ class AgentBasicTest(IsolatedAsyncioTestCase):"type": "MODEL_CALL_END","input_tokens": 0,"output_tokens": 0,+ "cache_input_tokens": 0,+ "cache_creation_input_tokens": 0,"finished_reason": "completed",},]@@ -1221,6 +1418,8 @@ class AgentBasicTest(IsolatedAsyncioTestCase):"type": "MODEL_CALL_END","input_tokens": 0,"output_tokens": 0,+ "cache_input_tokens": 0,+ "cache_creation_input_tokens": 0,"finished_reason": "completed",},{@@ -1441,6 +1640,8 @@ class AgentBasicTest(IsolatedAsyncioTestCase):"type": "MODEL_CALL_END","input_tokens": 0,"output_tokens": 0,+ "cache_input_tokens": 0,+ "cache_creation_input_tokens": 0,"finished_reason": "completed",},]@@ -1514,6 +1715,8 @@ class AgentBasicTest(IsolatedAsyncioTestCase):"type": "MODEL_CALL_END","input_tokens": 0,"output_tokens": 0,+ "cache_input_tokens": 0,+ "cache_creation_input_tokens": 0,"finished_reason": "completed",},{diff --git a/tests/event_to_message_test.py b/tests/event_to_message_test.pyindex 98ff1f482..5985306c2 100644--- a/tests/event_to_message_test.py+++ b/tests/event_to_message_test.py@@ -807,22 +807,36 @@ class EventToMessageTest(IsolatedAsyncioTestCase):reply_id=_REPLY_ID,input_tokens=10,output_tokens=20,+ cache_input_tokens=60,+ cache_creation_input_tokens=40,finished_reason=FinishedReason.COMPLETED,)gt_model_call_end_1 = _base(_final_content,- usage={"input_tokens": 10, "output_tokens": 20},+ usage={+ "input_tokens": 10,+ "output_tokens": 20,+ "cache_input_tokens": 60,+ "cache_creation_input_tokens": 40,+ },)ev_model_call_end_2 = ModelCallEndEvent(reply_id=_REPLY_ID,input_tokens=5,output_tokens=8,+ cache_input_tokens=12,+ cache_creation_input_tokens=7,finished_reason=FinishedReason.COMPLETED,)gt_model_call_end_2 = _base(_final_content,- usage={"input_tokens": 15, "output_tokens": 28},+ usage={+ "input_tokens": 15,+ "output_tokens": 28,+ "cache_input_tokens": 72,+ "cache_creation_input_tokens": 47,+ },)# ================================================================@@ -838,7 +852,12 @@ class EventToMessageTest(IsolatedAsyncioTestCase):_final_content,finished_at=_FIXED_END_TS,finished_reason="completed",- usage={"input_tokens": 15, "output_tokens": 28},+ usage={+ "input_tokens": 15,+ "output_tokens": 28,+ "cache_input_tokens": 72,+ "cache_creation_input_tokens": 47,+ },)# ================================================================diff --git a/tests/hitl_external_execution_test.py b/tests/hitl_external_execution_test.pyindex 0ca00627f..d28c0048b 100644--- a/tests/hitl_external_execution_test.py+++ b/tests/hitl_external_execution_test.py@@ -150,6 +150,8 @@ class AgentExternalExecutionTest(IsolatedAsyncioTestCase):"type": "MODEL_CALL_END","input_tokens": 0,"output_tokens": 0,+ "cache_input_tokens": 0,+ "cache_creation_input_tokens": 0,"finished_reason": "completed",},]@@ -344,6 +346,8 @@ class AgentExternalExecutionTest(IsolatedAsyncioTestCase):"type": "MODEL_CALL_END","input_tokens": 0,"output_tokens": 0,+ "cache_input_tokens": 0,+ "cache_creation_input_tokens": 0,"finished_reason": "completed",},*self._get_require_external_execution_events(@@ -599,6 +603,8 @@ class AgentExternalExecutionTest(IsolatedAsyncioTestCase):"type": "MODEL_CALL_END","input_tokens": 0,"output_tokens": 0,+ "cache_input_tokens": 0,+ "cache_creation_input_tokens": 0,"finished_reason": "completed",},*self._get_require_external_execution_events(@@ -933,6 +939,8 @@ class AgentExternalExecutionTest(IsolatedAsyncioTestCase):"type": "MODEL_CALL_END","input_tokens": 0,"output_tokens": 0,+ "cache_input_tokens": 0,+ "cache_creation_input_tokens": 0,"finished_reason": "completed",},*self._get_require_external_execution_events(@@ -1263,6 +1271,8 @@ class AgentExternalExecutionTest(IsolatedAsyncioTestCase):"type": "MODEL_CALL_END","input_tokens": 0,"output_tokens": 0,+ "cache_input_tokens": 0,+ "cache_creation_input_tokens": 0,"finished_reason": "completed",},*self._get_require_external_execution_events(diff --git a/tests/hitl_mixed_test.py b/tests/hitl_mixed_test.pyindex cdd98e59d..f4a2a8c80 100644--- a/tests/hitl_mixed_test.py+++ b/tests/hitl_mixed_test.py@@ -154,6 +154,8 @@ class AgentMixTest(IsolatedAsyncioTestCase):"type": "MODEL_CALL_END","input_tokens": 0,"output_tokens": 0,+ "cache_input_tokens": 0,+ "cache_creation_input_tokens": 0,"finished_reason": "completed",},]@@ -480,6 +482,8 @@ class AgentMixTest(IsolatedAsyncioTestCase):"type": "MODEL_CALL_END","input_tokens": 0,"output_tokens": 0,+ "cache_input_tokens": 0,+ "cache_creation_input_tokens": 0,"finished_reason": "completed",},self._get_require_user_confirm_event(@@ -673,6 +677,8 @@ class AgentMixTest(IsolatedAsyncioTestCase):"type": "MODEL_CALL_END","input_tokens": 0,"output_tokens": 0,+ "cache_input_tokens": 0,+ "cache_creation_input_tokens": 0,"finished_reason": "completed",},self._get_require_user_confirm_event(@@ -996,6 +1002,8 @@ class AgentMixTest(IsolatedAsyncioTestCase):"type": "MODEL_CALL_END","input_tokens": 0,"output_tokens": 0,+ "cache_input_tokens": 0,+ "cache_creation_input_tokens": 0,"finished_reason": "completed",},self._get_require_user_confirm_event(diff --git a/tests/hitl_user_confirmation_test.py b/tests/hitl_user_confirmation_test.pyindex cf4a38f7e..c7d2bce93 100644--- a/tests/hitl_user_confirmation_test.py+++ b/tests/hitl_user_confirmation_test.py@@ -244,6 +244,8 @@ class AgentUserConfirmationTest(IsolatedAsyncioTestCase):"type": "MODEL_CALL_END","input_tokens": 0,"output_tokens": 0,+ "cache_input_tokens": 0,+ "cache_creation_input_tokens": 0,"finished_reason": "completed",},]@@ -359,6 +361,8 @@ class AgentUserConfirmationTest(IsolatedAsyncioTestCase):"type": "MODEL_CALL_END","input_tokens": 0,"output_tokens": 0,+ "cache_input_tokens": 0,+ "cache_creation_input_tokens": 0,"finished_reason": "completed",},{@@ -646,6 +650,8 @@ class AgentUserConfirmationTest(IsolatedAsyncioTestCase):"type": "MODEL_CALL_END","input_tokens": 0,"output_tokens": 0,+ "cache_input_tokens": 0,+ "cache_creation_input_tokens": 0,"finished_reason": "completed",},{@@ -1046,6 +1052,8 @@ class AgentUserConfirmationTest(IsolatedAsyncioTestCase):"type": "MODEL_CALL_END","input_tokens": 0,"output_tokens": 0,+ "cache_input_tokens": 0,+ "cache_creation_input_tokens": 0,"finished_reason": "completed",},{@@ -1528,6 +1536,8 @@ class AgentUserConfirmationTest(IsolatedAsyncioTestCase):"type": "MODEL_CALL_END","input_tokens": 0,"output_tokens": 0,+ "cache_input_tokens": 0,+ "cache_creation_input_tokens": 0,"finished_reason": "completed",},{