Skip to content

Build and run agents you can see, understand and trust.

fix(model): count cached prompt tokens in anthropic input_tokens (#3477)

zhaoytab08e9075ed0a
diff --git a/src/agentscope/model/_anthropic/_model.py b/src/agentscope/model/_anthropic/_model.py
index f1b584a87..e112e0d25 100644
--- a/src/agentscope/model/_anthropic/_model.py
+++ b/src/agentscope/model/_anthropic/_model.py
@@ -24,6 +24,20 @@ else:
AsyncStream = Any
+def _total_input_tokens(usage: Any) -> int:
+ """The input tokens of an Anthropic usage, including the cached ones.
+
+ Anthropic reports the cache reads and writes apart from
+ ``input_tokens``, while the other providers count them in it. Summing
+ them keeps ``ChatUsage.input_tokens`` the whole prompt everywhere.
+ """
+ return (
+ (getattr(usage, "input_tokens", 0) or 0)
+ + (getattr(usage, "cache_creation_input_tokens", 0) or 0)
+ + (getattr(usage, "cache_read_input_tokens", 0) or 0)
+ )
+
+
class AnthropicChatModel(ChatModelBase):
"""The Anthropic chat model."""
@@ -391,7 +405,7 @@ class AnthropicChatModel(ChatModelBase):
if response.usage:
u = response.usage
usage = ChatUsage(
- input_tokens=u.input_tokens or 0,
+ input_tokens=_total_input_tokens(u),
output_tokens=u.output_tokens or 0,
time=(datetime.now() - start_datetime).total_seconds(),
cache_creation_input_tokens=getattr(
@@ -467,7 +481,7 @@ class AnthropicChatModel(ChatModelBase):
if message.usage:
u = message.usage
usage = ChatUsage(
- input_tokens=u.input_tokens or 0,
+ input_tokens=_total_input_tokens(u),
output_tokens=getattr(u, "output_tokens", 0) or 0,
time=(
datetime.now() - start_datetime
diff --git a/src/agentscope/model/_model_usage.py b/src/agentscope/model/_model_usage.py
index 952cc6de4..f191124ef 100644
--- a/src/agentscope/model/_model_usage.py
+++ b/src/agentscope/model/_model_usage.py
@@ -11,7 +11,8 @@ class ChatUsage(DictMixin):
"""The usage of a chat model API invocation."""
input_tokens: int
- """The number of input tokens."""
+ """The number of input tokens, including the cache reads and writes
+ counted in ``cache_input_tokens`` and ``cache_creation_input_tokens``."""
output_tokens: int
"""The number of output tokens."""
diff --git a/tests/model_anthropic_test.py b/tests/model_anthropic_test.py
index cb9f74644..93c261406 100644
--- a/tests/model_anthropic_test.py
+++ b/tests/model_anthropic_test.py
@@ -827,6 +827,49 @@ class TestAnthropicStream(IsolatedAsyncioTestCase):
ChatUsage(input_tokens=10, output_tokens=5, time=20.0),
)
+ async def test_cached_tokens_counted_in_input_tokens(self) -> None:
+ """The cache reads and writes count in ``input_tokens``, as for
+ the other providers, in both the stream and non-stream paths."""
+ completion = anthropic_types.Message.model_validate(
+ {
+ "id": "msg-c",
+ "type": "message",
+ "role": "assistant",
+ "model": self.model.model,
+ "content": [{"type": "text", "text": "Hi"}],
+ "stop_reason": "end_turn",
+ "stop_sequence": None,
+ "usage": {
+ "input_tokens": 10,
+ "output_tokens": 5,
+ "cache_creation_input_tokens": 200,
+ "cache_read_input_tokens": 3000,
+ },
+ },
+ )
+ non_stream_model = _make_model(stream=False)
+ non_stream_model.client = MagicMock()
+ non_stream_model.client.messages.create = AsyncMock(
+ return_value=completion,
+ )
+ self.mock_client.messages.create = AsyncMock(
+ return_value=_MockAsyncEventStream(_completion_events(completion)),
+ )
+
+ non_stream_usage = (await non_stream_model([])).usage
+ stream_usage = [r async for r in await self.model([])][-1].usage
+
+ for usage in (non_stream_usage, stream_usage):
+ self.assertEqual(
+ (
+ usage.input_tokens,
+ usage.cache_input_tokens,
+ usage.cache_creation_input_tokens,
+ usage.output_tokens,
+ ),
+ (3210, 3000, 200, 5),
+ )
+
async def test_stream_redacted_thinking(self) -> None:
"""Stream redacted_thinking block is emitted at
content_block_start."""