Skip to content

Build and run agents you can see, understand and trust.

fix(agent): keep prompt cache tokens in usage (#2318)

yingz2363e64aefd25a
diff --git a/src/agentscope/agent/_agent.py b/src/agentscope/agent/_agent.py
index b98333bb5..de04813d5 100644
--- a/src/agentscope/agent/_agent.py
+++ b/src/agentscope/agent/_agent.py
@@ -1595,14 +1595,15 @@ class Agent:
)
# Send the model call ended event with usage if available
+ usage = completed_response.usage
yield ModelCallEndEvent(
reply_id=self.state.reply_id,
- input_tokens=completed_response.usage.input_tokens
- if completed_response.usage
- else 0,
- output_tokens=completed_response.usage.output_tokens
- if completed_response.usage
- else 0,
+ input_tokens=usage.input_tokens if usage else 0,
+ output_tokens=usage.output_tokens if usage else 0,
+ cache_input_tokens=(usage.cache_input_tokens or 0) if usage else 0,
+ cache_creation_input_tokens=(
+ (usage.cache_creation_input_tokens or 0) if usage else 0
+ ),
finished_reason=completed_response.finished_reason,
)
@@ -1633,6 +1634,12 @@ class Agent:
Usage(
input_tokens=last_ctx.usage.input_tokens,
output_tokens=last_ctx.usage.output_tokens,
+ cache_input_tokens=(
+ last_ctx.usage.cache_input_tokens or 0
+ ),
+ cache_creation_input_tokens=(
+ last_ctx.usage.cache_creation_input_tokens or 0
+ ),
)
if last_ctx is not None and last_ctx.usage is not None
else None
@@ -3189,6 +3196,10 @@ class Agent:
Usage(
input_tokens=usage.input_tokens,
output_tokens=usage.output_tokens,
+ cache_input_tokens=usage.cache_input_tokens or 0,
+ cache_creation_input_tokens=(
+ usage.cache_creation_input_tokens or 0
+ ),
)
if usage is not None
else None
@@ -3220,6 +3231,10 @@ class Agent:
else:
tail.usage.input_tokens += msg_usage.input_tokens
tail.usage.output_tokens += msg_usage.output_tokens
+ tail.usage.cache_input_tokens += msg_usage.cache_input_tokens
+ tail.usage.cache_creation_input_tokens += (
+ msg_usage.cache_creation_input_tokens
+ )
def _get_last_msg(self) -> Msg | None:
"""Get the last message in the context that belongs to this agent."""
diff --git a/src/agentscope/event/_event.py b/src/agentscope/event/_event.py
index 774a60bcf..1203f5b40 100644
--- a/src/agentscope/event/_event.py
+++ b/src/agentscope/event/_event.py
@@ -147,6 +147,10 @@ class ModelCallEndEvent(EventBase):
"""Number of input tokens consumed."""
output_tokens: int
"""Number of output tokens generated."""
+ cache_input_tokens: int = 0
+ """Number of input tokens read from the prompt cache."""
+ cache_creation_input_tokens: int = 0
+ """Number of input tokens used to create the prompt cache."""
finished_reason: FinishedReason = Field(
default=FinishedReason.COMPLETED,
)
diff --git a/src/agentscope/message/_base.py b/src/agentscope/message/_base.py
index 0a7239d08..0d51f35aa 100644
--- a/src/agentscope/message/_base.py
+++ b/src/agentscope/message/_base.py
@@ -62,6 +62,10 @@ class Usage(BaseModel):
"""The number of input tokens."""
output_tokens: int
"""The number of output tokens."""
+ cache_input_tokens: int = 0
+ """The number of input tokens read from the prompt cache."""
+ cache_creation_input_tokens: int = 0
+ """The number of input tokens used to create the prompt cache."""
class Msg(BaseModel):
@@ -283,10 +287,18 @@ class Msg(BaseModel):
self.usage = Usage(
input_tokens=event.input_tokens,
output_tokens=event.output_tokens,
+ cache_input_tokens=event.cache_input_tokens,
+ cache_creation_input_tokens=(
+ event.cache_creation_input_tokens
+ ),
)
else:
self.usage.input_tokens += event.input_tokens
self.usage.output_tokens += event.output_tokens
+ self.usage.cache_input_tokens += event.cache_input_tokens
+ self.usage.cache_creation_input_tokens += (
+ event.cache_creation_input_tokens
+ )
case EventType.TEXT_BLOCK_START:
self.content.append(TextBlock(id=event.block_id, text=""))
diff --git a/tests/agent_basic_test.py b/tests/agent_basic_test.py
index b667eec9d..383653c7e 100644
--- a/tests/agent_basic_test.py
+++ b/tests/agent_basic_test.py
@@ -6,7 +6,7 @@ from unittest.async_case import IsolatedAsyncioTestCase
from utils import AnyString, MockModel
from agentscope.agent import Agent, ContextConfig, InjectionConfig, ReActConfig
-from agentscope.model import ChatResponse
+from agentscope.model import ChatResponse, ChatUsage
from agentscope.tool import (
ToolBase,
Toolkit,
@@ -281,6 +281,8 @@ class AgentBasicTest(IsolatedAsyncioTestCase):
"type": "MODEL_CALL_END",
"input_tokens": 0,
"output_tokens": 0,
+ "cache_input_tokens": 0,
+ "cache_creation_input_tokens": 0,
"finished_reason": "completed",
},
{
@@ -493,6 +495,8 @@ class AgentBasicTest(IsolatedAsyncioTestCase):
"type": "MODEL_CALL_END",
"input_tokens": 0,
"output_tokens": 0,
+ "cache_input_tokens": 0,
+ "cache_creation_input_tokens": 0,
"finished_reason": "completed",
},
{
@@ -659,6 +663,193 @@ class AgentBasicTest(IsolatedAsyncioTestCase):
context_dicts = [msg.model_dump() for msg in self.agent.state.context]
self.assertListEqual(context_dicts, expected_context_after_reply)
+ async def test_usage_cache_tokens_are_not_dropped(self) -> None:
+ """Regression test for issue #2305.
+
+ Prompt cache tokens (``cache_input_tokens`` and
+ ``cache_creation_input_tokens``) parsed by the model adapter into
+ ``ChatUsage`` must survive the agent layer:
+
+ 1. ``ModelCallEndEvent`` must carry them so middleware reading the
+ event can record cache hit rates.
+ 2. ``_save_to_context`` must merge them into ``Msg.usage``.
+ """
+ self.model.set_responses(
+ [
+ ChatResponse(
+ content=[TextBlock(text="Hello world!")],
+ is_last=True,
+ usage=ChatUsage(
+ input_tokens=100,
+ output_tokens=20,
+ time=0.5,
+ cache_input_tokens=60,
+ cache_creation_input_tokens=40,
+ ),
+ ),
+ ],
+ )
+
+ end_events = []
+ async for event in self.agent.reply_stream(
+ UserMsg(name="user", content="Hi"),
+ ):
+ if event.type == "MODEL_CALL_END":
+ end_events.append(event)
+
+ # 1) The end event must carry the cache tokens.
+ self.assertEqual(len(end_events), 1)
+ self.assertEqual(
+ end_events[0].model_dump(),
+ {
+ **self._get_event_base(self.agent.state.reply_id),
+ "type": "MODEL_CALL_END",
+ "input_tokens": 100,
+ "output_tokens": 20,
+ "cache_input_tokens": 60,
+ "cache_creation_input_tokens": 40,
+ "finished_reason": "completed",
+ },
+ )
+
+ # 2) The saved assistant message must expose the cache tokens in its
+ # usage.
+ assistant_msgs = [
+ msg for msg in self.agent.state.context if msg.role == "assistant"
+ ]
+ self.assertEqual(len(assistant_msgs), 1)
+ usage = assistant_msgs[0].usage
+ self.assertIsNotNone(usage)
+ assert usage is not None
+ self.assertEqual(
+ usage.model_dump(),
+ {
+ "input_tokens": 100,
+ "output_tokens": 20,
+ "cache_input_tokens": 60,
+ "cache_creation_input_tokens": 40,
+ },
+ )
+
+ async def test_usage_cache_tokens_none_normalized_to_zero(self) -> None:
+ """Regression test: None-valued cache fields must not raise Pydantic.
+
+ ChatUsage is a plain dataclass that does not enforce int annotations at
+ runtime. Adapters copy optional SDK fields with getattr(x, 0),
+ which can still return None when the attribute exists but is null.
+ None-valued cache fields must be normalized to zero before
+ entering any Pydantic model.
+ """
+ usage_with_none_cache = ChatUsage(
+ input_tokens=10,
+ output_tokens=5,
+ time=0.1,
+ )
+ # Manually assign None to the two cache fields (dataclass will not
+ # complain at runtime because there is no runtime type check).
+ usage_with_none_cache.cache_input_tokens = None
+ usage_with_none_cache.cache_creation_input_tokens = None
+
+ self.model.set_responses(
+ [
+ ChatResponse(
+ content=[TextBlock(text="Hi")],
+ is_last=True,
+ usage=usage_with_none_cache,
+ ),
+ ],
+ )
+
+ end_events = []
+ # Must not raise Pydantic ValidationError; reply completes cleanly.
+ async for event in self.agent.reply_stream(
+ UserMsg(name="user", content="Hello"),
+ ):
+ if event.type == "MODEL_CALL_END":
+ end_events.append(event)
+
+ self.assertEqual(len(end_events), 1)
+ self.assertEqual(
+ end_events[0].model_dump(),
+ {
+ **self._get_event_base(self.agent.state.reply_id),
+ "type": "MODEL_CALL_END",
+ "input_tokens": 10,
+ "output_tokens": 5,
+ # None must be normalized to 0.
+ "cache_input_tokens": 0,
+ "cache_creation_input_tokens": 0,
+ "finished_reason": "completed",
+ },
+ )
+
+ assistant_msgs = [
+ msg for msg in self.agent.state.context if msg.role == "assistant"
+ ]
+ self.assertEqual(len(assistant_msgs), 1)
+ usage = assistant_msgs[0].usage
+ self.assertIsNotNone(usage)
+ assert usage is not None
+ # None from dataclass must also be normalized to 0 in Msg.usage.
+ self.assertEqual(
+ usage.model_dump(),
+ {
+ "input_tokens": 10,
+ "output_tokens": 5,
+ "cache_input_tokens": 0,
+ "cache_creation_input_tokens": 0,
+ },
+ )
+
+ async def test_usage_cache_tokens_in_reply_return(self) -> None:
+ """Regression test: Agent.reply() result must keep the cache tokens.
+
+ ``final_usage`` is rebuilt from the saved context message when
+ returning the final ``AssistantMsg``. It must copy both cache
+ fields, otherwise the returned message silently reports zeros even
+ though the saved context carried the correct values.
+ """
+ self.model.set_responses(
+ [
+ ChatResponse(
+ content=[TextBlock(text="Hello world!")],
+ is_last=True,
+ usage=ChatUsage(
+ input_tokens=100,
+ output_tokens=20,
+ time=0.5,
+ cache_input_tokens=60,
+ cache_creation_input_tokens=40,
+ ),
+ ),
+ ],
+ )
+
+ msg = await self.agent.reply(UserMsg(name="user", content="Hi"))
+
+ self.assertDictEqual(
+ msg.model_dump(),
+ {
+ **self._get_msg_base(),
+ "content": [
+ {
+ "type": "text",
+ "created_at": AnyString(),
+ "finished_at": None,
+ "id": AnyString(),
+ "text": "Hello world!",
+ },
+ ],
+ "finished_reason": ReplyFinishedReason.COMPLETED,
+ "usage": {
+ "input_tokens": 100,
+ "output_tokens": 20,
+ "cache_input_tokens": 60,
+ "cache_creation_input_tokens": 40,
+ },
+ },
+ )
+
async def test_max_iters_counts_reasoning_acting_round_once(self) -> None:
"""A tool round consumes one iteration before final reasoning."""
self.agent.toolkit = Toolkit(tools=[MockSequentialTool()])
@@ -923,6 +1114,8 @@ class AgentBasicTest(IsolatedAsyncioTestCase):
"type": "MODEL_CALL_END",
"input_tokens": 0,
"output_tokens": 0,
+ "cache_input_tokens": 0,
+ "cache_creation_input_tokens": 0,
"finished_reason": "completed",
},
{
@@ -967,6 +1160,8 @@ class AgentBasicTest(IsolatedAsyncioTestCase):
"type": "MODEL_CALL_END",
"input_tokens": 0,
"output_tokens": 0,
+ "cache_input_tokens": 0,
+ "cache_creation_input_tokens": 0,
"finished_reason": "completed",
},
{
@@ -1169,6 +1364,8 @@ class AgentBasicTest(IsolatedAsyncioTestCase):
"type": "MODEL_CALL_END",
"input_tokens": 0,
"output_tokens": 0,
+ "cache_input_tokens": 0,
+ "cache_creation_input_tokens": 0,
"finished_reason": "completed",
},
]
@@ -1221,6 +1418,8 @@ class AgentBasicTest(IsolatedAsyncioTestCase):
"type": "MODEL_CALL_END",
"input_tokens": 0,
"output_tokens": 0,
+ "cache_input_tokens": 0,
+ "cache_creation_input_tokens": 0,
"finished_reason": "completed",
},
{
@@ -1441,6 +1640,8 @@ class AgentBasicTest(IsolatedAsyncioTestCase):
"type": "MODEL_CALL_END",
"input_tokens": 0,
"output_tokens": 0,
+ "cache_input_tokens": 0,
+ "cache_creation_input_tokens": 0,
"finished_reason": "completed",
},
]
@@ -1514,6 +1715,8 @@ class AgentBasicTest(IsolatedAsyncioTestCase):
"type": "MODEL_CALL_END",
"input_tokens": 0,
"output_tokens": 0,
+ "cache_input_tokens": 0,
+ "cache_creation_input_tokens": 0,
"finished_reason": "completed",
},
{
diff --git a/tests/event_to_message_test.py b/tests/event_to_message_test.py
index 98ff1f482..5985306c2 100644
--- a/tests/event_to_message_test.py
+++ b/tests/event_to_message_test.py
@@ -807,22 +807,36 @@ class EventToMessageTest(IsolatedAsyncioTestCase):
reply_id=_REPLY_ID,
input_tokens=10,
output_tokens=20,
+ cache_input_tokens=60,
+ cache_creation_input_tokens=40,
finished_reason=FinishedReason.COMPLETED,
)
gt_model_call_end_1 = _base(
_final_content,
- usage={"input_tokens": 10, "output_tokens": 20},
+ usage={
+ "input_tokens": 10,
+ "output_tokens": 20,
+ "cache_input_tokens": 60,
+ "cache_creation_input_tokens": 40,
+ },
)
ev_model_call_end_2 = ModelCallEndEvent(
reply_id=_REPLY_ID,
input_tokens=5,
output_tokens=8,
+ cache_input_tokens=12,
+ cache_creation_input_tokens=7,
finished_reason=FinishedReason.COMPLETED,
)
gt_model_call_end_2 = _base(
_final_content,
- usage={"input_tokens": 15, "output_tokens": 28},
+ usage={
+ "input_tokens": 15,
+ "output_tokens": 28,
+ "cache_input_tokens": 72,
+ "cache_creation_input_tokens": 47,
+ },
)
# ================================================================
@@ -838,7 +852,12 @@ class EventToMessageTest(IsolatedAsyncioTestCase):
_final_content,
finished_at=_FIXED_END_TS,
finished_reason="completed",
- usage={"input_tokens": 15, "output_tokens": 28},
+ usage={
+ "input_tokens": 15,
+ "output_tokens": 28,
+ "cache_input_tokens": 72,
+ "cache_creation_input_tokens": 47,
+ },
)
# ================================================================
diff --git a/tests/hitl_external_execution_test.py b/tests/hitl_external_execution_test.py
index 0ca00627f..d28c0048b 100644
--- a/tests/hitl_external_execution_test.py
+++ b/tests/hitl_external_execution_test.py
@@ -150,6 +150,8 @@ class AgentExternalExecutionTest(IsolatedAsyncioTestCase):
"type": "MODEL_CALL_END",
"input_tokens": 0,
"output_tokens": 0,
+ "cache_input_tokens": 0,
+ "cache_creation_input_tokens": 0,
"finished_reason": "completed",
},
]
@@ -344,6 +346,8 @@ class AgentExternalExecutionTest(IsolatedAsyncioTestCase):
"type": "MODEL_CALL_END",
"input_tokens": 0,
"output_tokens": 0,
+ "cache_input_tokens": 0,
+ "cache_creation_input_tokens": 0,
"finished_reason": "completed",
},
*self._get_require_external_execution_events(
@@ -599,6 +603,8 @@ class AgentExternalExecutionTest(IsolatedAsyncioTestCase):
"type": "MODEL_CALL_END",
"input_tokens": 0,
"output_tokens": 0,
+ "cache_input_tokens": 0,
+ "cache_creation_input_tokens": 0,
"finished_reason": "completed",
},
*self._get_require_external_execution_events(
@@ -933,6 +939,8 @@ class AgentExternalExecutionTest(IsolatedAsyncioTestCase):
"type": "MODEL_CALL_END",
"input_tokens": 0,
"output_tokens": 0,
+ "cache_input_tokens": 0,
+ "cache_creation_input_tokens": 0,
"finished_reason": "completed",
},
*self._get_require_external_execution_events(
@@ -1263,6 +1271,8 @@ class AgentExternalExecutionTest(IsolatedAsyncioTestCase):
"type": "MODEL_CALL_END",
"input_tokens": 0,
"output_tokens": 0,
+ "cache_input_tokens": 0,
+ "cache_creation_input_tokens": 0,
"finished_reason": "completed",
},
*self._get_require_external_execution_events(
diff --git a/tests/hitl_mixed_test.py b/tests/hitl_mixed_test.py
index cdd98e59d..f4a2a8c80 100644
--- a/tests/hitl_mixed_test.py
+++ b/tests/hitl_mixed_test.py
@@ -154,6 +154,8 @@ class AgentMixTest(IsolatedAsyncioTestCase):
"type": "MODEL_CALL_END",
"input_tokens": 0,
"output_tokens": 0,
+ "cache_input_tokens": 0,
+ "cache_creation_input_tokens": 0,
"finished_reason": "completed",
},
]
@@ -480,6 +482,8 @@ class AgentMixTest(IsolatedAsyncioTestCase):
"type": "MODEL_CALL_END",
"input_tokens": 0,
"output_tokens": 0,
+ "cache_input_tokens": 0,
+ "cache_creation_input_tokens": 0,
"finished_reason": "completed",
},
self._get_require_user_confirm_event(
@@ -673,6 +677,8 @@ class AgentMixTest(IsolatedAsyncioTestCase):
"type": "MODEL_CALL_END",
"input_tokens": 0,
"output_tokens": 0,
+ "cache_input_tokens": 0,
+ "cache_creation_input_tokens": 0,
"finished_reason": "completed",
},
self._get_require_user_confirm_event(
@@ -996,6 +1002,8 @@ class AgentMixTest(IsolatedAsyncioTestCase):
"type": "MODEL_CALL_END",
"input_tokens": 0,
"output_tokens": 0,
+ "cache_input_tokens": 0,
+ "cache_creation_input_tokens": 0,
"finished_reason": "completed",
},
self._get_require_user_confirm_event(
diff --git a/tests/hitl_user_confirmation_test.py b/tests/hitl_user_confirmation_test.py
index cf4a38f7e..c7d2bce93 100644
--- a/tests/hitl_user_confirmation_test.py
+++ b/tests/hitl_user_confirmation_test.py
@@ -244,6 +244,8 @@ class AgentUserConfirmationTest(IsolatedAsyncioTestCase):
"type": "MODEL_CALL_END",
"input_tokens": 0,
"output_tokens": 0,
+ "cache_input_tokens": 0,
+ "cache_creation_input_tokens": 0,
"finished_reason": "completed",
},
]
@@ -359,6 +361,8 @@ class AgentUserConfirmationTest(IsolatedAsyncioTestCase):
"type": "MODEL_CALL_END",
"input_tokens": 0,
"output_tokens": 0,
+ "cache_input_tokens": 0,
+ "cache_creation_input_tokens": 0,
"finished_reason": "completed",
},
{
@@ -646,6 +650,8 @@ class AgentUserConfirmationTest(IsolatedAsyncioTestCase):
"type": "MODEL_CALL_END",
"input_tokens": 0,
"output_tokens": 0,
+ "cache_input_tokens": 0,
+ "cache_creation_input_tokens": 0,
"finished_reason": "completed",
},
{
@@ -1046,6 +1052,8 @@ class AgentUserConfirmationTest(IsolatedAsyncioTestCase):
"type": "MODEL_CALL_END",
"input_tokens": 0,
"output_tokens": 0,
+ "cache_input_tokens": 0,
+ "cache_creation_input_tokens": 0,
"finished_reason": "completed",
},
{
@@ -1528,6 +1536,8 @@ class AgentUserConfirmationTest(IsolatedAsyncioTestCase):
"type": "MODEL_CALL_END",
"input_tokens": 0,
"output_tokens": 0,
+ "cache_input_tokens": 0,
+ "cache_creation_input_tokens": 0,
"finished_reason": "completed",
},
{