Skip to content

Build and run agents you can see, understand and trust.

feat(model): add cache_creation_input_tokens and cache_input_tokens in ChatUsage (#1602)

qbc8a32a3661819
diff --git a/src/agentscope/model/_anthropic/_model.py b/src/agentscope/model/_anthropic/_model.py
index b92848858..80481cf9d 100644
--- a/src/agentscope/model/_anthropic/_model.py
+++ b/src/agentscope/model/_anthropic/_model.py
@@ -251,10 +251,21 @@ class AnthropicChatModel(ChatModelBase):
usage = None
if response.usage:
+ u = response.usage
usage = ChatUsage(
- input_tokens=response.usage.input_tokens,
- output_tokens=response.usage.output_tokens,
+ input_tokens=u.input_tokens,
+ output_tokens=u.output_tokens,
time=(datetime.now() - start_datetime).total_seconds(),
+ cache_creation_input_tokens=getattr(
+ u,
+ "cache_creation_input_tokens",
+ 0,
+ ),
+ cache_input_tokens=getattr(
+ u,
+ "cache_read_input_tokens",
+ 0,
+ ),
)
resp_kwargs: dict[str, Any] = {
@@ -306,14 +317,21 @@ class AnthropicChatModel(ChatModelBase):
if response_id is None:
response_id = getattr(message, "id", None)
if message.usage:
+ u = message.usage
usage = ChatUsage(
- input_tokens=message.usage.input_tokens,
- output_tokens=getattr(
- message.usage,
- "output_tokens",
+ input_tokens=u.input_tokens,
+ output_tokens=getattr(u, "output_tokens", 0),
+ time=(datetime.now() - start_datetime).total_seconds(),
+ cache_creation_input_tokens=getattr(
+ u,
+ "cache_creation_input_tokens",
+ 0,
+ ),
+ cache_input_tokens=getattr(
+ u,
+ "cache_read_input_tokens",
0,
),
- time=(datetime.now() - start_datetime).total_seconds(),
)
elif event.type == "content_block_start":
diff --git a/src/agentscope/model/_dashscope/_model.py b/src/agentscope/model/_dashscope/_model.py
index fef8b3159..918c2bd77 100644
--- a/src/agentscope/model/_dashscope/_model.py
+++ b/src/agentscope/model/_dashscope/_model.py
@@ -366,11 +366,20 @@ class DashScopeChatModel(ChatModelBase):
# The chunk usage
if chunk.usage:
+ u = chunk.usage
+ ptd = getattr(u, "prompt_tokens_details", None)
+ if isinstance(ptd, dict):
+ cache_creation = ptd.get("cache_creation_input_tokens", 0)
+ cache_read = ptd.get("cached_tokens", 0)
+ else:
+ cache_creation = 0
+ cache_read = getattr(u, "cached_tokens", 0)
usage = ChatUsage(
- input_tokens=chunk.usage.input_tokens,
- output_tokens=chunk.usage.output_tokens,
+ input_tokens=u.input_tokens,
+ output_tokens=u.output_tokens,
time=(datetime.now() - start_datetime).total_seconds(),
- metadata=chunk.usage,
+ cache_creation_input_tokens=cache_creation,
+ cache_input_tokens=cache_read,
)
# Yield the chat response
@@ -476,11 +485,20 @@ class DashScopeChatModel(ChatModelBase):
# Usage information
usage = None
if response.usage:
+ u = response.usage
+ ptd = getattr(u, "prompt_tokens_details", None)
+ if isinstance(ptd, dict):
+ cache_creation = ptd.get("cache_creation_input_tokens", 0)
+ cache_read = ptd.get("cached_tokens", 0)
+ else:
+ cache_creation = 0
+ cache_read = getattr(u, "cached_tokens", 0)
usage = ChatUsage(
- input_tokens=response.usage.input_tokens,
- output_tokens=response.usage.output_tokens,
+ input_tokens=u.input_tokens,
+ output_tokens=u.output_tokens,
time=(datetime.now() - start_datetime).total_seconds(),
- metadata=response.usage,
+ cache_creation_input_tokens=cache_creation,
+ cache_input_tokens=cache_read,
)
return ChatResponse(
diff --git a/src/agentscope/model/_deepseek/_model.py b/src/agentscope/model/_deepseek/_model.py
index 284ce0995..29bce6dae 100644
--- a/src/agentscope/model/_deepseek/_model.py
+++ b/src/agentscope/model/_deepseek/_model.py
@@ -209,11 +209,16 @@ class DeepSeekChatModel(ChatModelBase):
async with response as stream:
async for chunk in stream:
if chunk.usage:
+ u = chunk.usage
usage = ChatUsage(
- input_tokens=chunk.usage.prompt_tokens,
- output_tokens=chunk.usage.completion_tokens,
+ input_tokens=u.prompt_tokens,
+ output_tokens=u.completion_tokens,
time=(datetime.now() - start_datetime).total_seconds(),
- metadata=chunk.usage,
+ cache_input_tokens=getattr(
+ u,
+ "prompt_cache_hit_tokens",
+ 0,
+ ),
)
# Capture response_id from the first chunk that carries it
@@ -334,11 +339,16 @@ class DeepSeekChatModel(ChatModelBase):
usage = None
if response.usage:
+ u = response.usage
usage = ChatUsage(
- input_tokens=response.usage.prompt_tokens,
- output_tokens=response.usage.completion_tokens,
+ input_tokens=u.prompt_tokens,
+ output_tokens=u.completion_tokens,
time=(datetime.now() - start_datetime).total_seconds(),
- metadata=response.usage,
+ cache_input_tokens=getattr(
+ u,
+ "prompt_cache_hit_tokens",
+ 0,
+ ),
)
resp_kwargs: dict[str, Any] = {
diff --git a/src/agentscope/model/_gemini/_model.py b/src/agentscope/model/_gemini/_model.py
index cb9fa9713..941f2d50d 100644
--- a/src/agentscope/model/_gemini/_model.py
+++ b/src/agentscope/model/_gemini/_model.py
@@ -449,7 +449,11 @@ class GeminiChatModel(ChatModelBase):
input_tokens=prompt_tokens,
output_tokens=total_tokens - prompt_tokens,
time=(datetime.now() - start_datetime).total_seconds(),
- metadata=usage_metadata,
+ cache_input_tokens=getattr(
+ usage_metadata,
+ "cached_content_token_count",
+ 0,
+ ),
)
return None
diff --git a/src/agentscope/model/_kimi/_model.py b/src/agentscope/model/_kimi/_model.py
index 9b97191c4..c01f3d9fe 100644
--- a/src/agentscope/model/_kimi/_model.py
+++ b/src/agentscope/model/_kimi/_model.py
@@ -200,11 +200,16 @@ class KimiChatModel(ChatModelBase):
async with response as stream:
async for chunk in stream:
if chunk.usage:
+ u = chunk.usage
usage = ChatUsage(
- input_tokens=chunk.usage.prompt_tokens,
- output_tokens=chunk.usage.completion_tokens,
+ input_tokens=u.prompt_tokens,
+ output_tokens=u.completion_tokens,
time=(datetime.now() - start_datetime).total_seconds(),
- metadata=chunk.usage,
+ cache_input_tokens=getattr(
+ u,
+ "cached_tokens",
+ 0,
+ ),
)
# Capture response_id from the first chunk that carries it
@@ -325,11 +330,12 @@ class KimiChatModel(ChatModelBase):
usage = None
if response.usage:
+ u = response.usage
usage = ChatUsage(
- input_tokens=response.usage.prompt_tokens,
- output_tokens=response.usage.completion_tokens,
+ input_tokens=u.prompt_tokens,
+ output_tokens=u.completion_tokens,
time=(datetime.now() - start_datetime).total_seconds(),
- metadata=response.usage,
+ cache_input_tokens=getattr(u, "cached_tokens", 0),
)
resp_kwargs: dict[str, Any] = {
diff --git a/src/agentscope/model/_model_usage.py b/src/agentscope/model/_model_usage.py
index 6478105bd..70e17be61 100644
--- a/src/agentscope/model/_model_usage.py
+++ b/src/agentscope/model/_model_usage.py
@@ -1,7 +1,7 @@
# -*- coding: utf-8 -*-
"""The model usage class in agentscope."""
from dataclasses import dataclass, field
-from typing import Literal, Any
+from typing import Any, Literal
from .._utils._mixin import DictMixin
@@ -22,5 +22,11 @@ class ChatUsage(DictMixin):
type: Literal["chat"] = field(default_factory=lambda: "chat")
"""The type of the usage, must be `chat`."""
+ cache_creation_input_tokens: int = field(default_factory=lambda: 0)
+ """The number of input tokens used to create the prompt cache."""
+
+ cache_input_tokens: int = field(default_factory=lambda: 0)
+ """The number of input tokens read from the prompt cache."""
+
metadata: dict[str, Any] | None = field(default_factory=lambda: None)
- """The metadata of the usage."""
+ """Optional metadata associated with the usage."""
diff --git a/src/agentscope/model/_openai_chat/_model.py b/src/agentscope/model/_openai_chat/_model.py
index d44cfafb3..bd342232e 100644
--- a/src/agentscope/model/_openai_chat/_model.py
+++ b/src/agentscope/model/_openai_chat/_model.py
@@ -218,11 +218,19 @@ class OpenAIChatModel(ChatModelBase):
async with response as stream:
async for chunk in stream:
if chunk.usage:
+ u = chunk.usage
+ details = getattr(u, "prompt_tokens_details", None)
usage = ChatUsage(
- input_tokens=chunk.usage.prompt_tokens,
- output_tokens=chunk.usage.completion_tokens,
+ input_tokens=u.prompt_tokens,
+ output_tokens=u.completion_tokens,
time=(datetime.now() - start_datetime).total_seconds(),
- metadata=chunk.usage,
+ cache_input_tokens=getattr(
+ details,
+ "cached_tokens",
+ 0,
+ )
+ if details
+ else 0,
)
# Capture response_id from the first chunk that carries it
@@ -344,11 +352,19 @@ class OpenAIChatModel(ChatModelBase):
usage = None
if response.usage:
+ u = response.usage
+ details = getattr(u, "prompt_tokens_details", None)
usage = ChatUsage(
- input_tokens=response.usage.prompt_tokens,
- output_tokens=response.usage.completion_tokens,
+ input_tokens=u.prompt_tokens,
+ output_tokens=u.completion_tokens,
time=(datetime.now() - start_datetime).total_seconds(),
- metadata=response.usage,
+ cache_input_tokens=getattr(
+ details,
+ "cached_tokens",
+ 0,
+ )
+ if details
+ else 0,
)
resp_kwargs: dict[str, Any] = {
diff --git a/src/agentscope/model/_openai_response/_model.py b/src/agentscope/model/_openai_response/_model.py
index c6c160b59..d87ca496d 100644
--- a/src/agentscope/model/_openai_response/_model.py
+++ b/src/agentscope/model/_openai_response/_model.py
@@ -259,11 +259,19 @@ class OpenAIResponseModel(ChatModelBase):
if response_id is None:
response_id = getattr(resp, "id", None)
if resp.usage:
+ u = resp.usage
+ details = getattr(u, "input_tokens_details", None)
usage = ChatUsage(
- input_tokens=resp.usage.input_tokens,
- output_tokens=resp.usage.output_tokens,
+ input_tokens=u.input_tokens,
+ output_tokens=u.output_tokens,
time=(datetime.now() - start_datetime).total_seconds(),
- metadata=resp.usage,
+ cache_input_tokens=getattr(
+ details,
+ "cached_tokens",
+ 0,
+ )
+ if details
+ else 0,
)
# Attach reasoning item IDs from the completed response so the
# formatter can echo them back in multi-turn history.
@@ -380,11 +388,19 @@ class OpenAIResponseModel(ChatModelBase):
usage = None
if response.usage:
+ u = response.usage
+ details = getattr(u, "input_tokens_details", None)
usage = ChatUsage(
- input_tokens=response.usage.input_tokens,
- output_tokens=response.usage.output_tokens,
+ input_tokens=u.input_tokens,
+ output_tokens=u.output_tokens,
time=(datetime.now() - start_datetime).total_seconds(),
- metadata=response.usage,
+ cache_input_tokens=getattr(
+ details,
+ "cached_tokens",
+ 0,
+ )
+ if details
+ else 0,
)
resp_kwargs: dict[str, Any] = {
diff --git a/src/agentscope/model/_xai/_model.py b/src/agentscope/model/_xai/_model.py
index 546316f3c..e8c395dd9 100644
--- a/src/agentscope/model/_xai/_model.py
+++ b/src/agentscope/model/_xai/_model.py
@@ -338,7 +338,11 @@ class XAIChatModel(ChatModelBase):
input_tokens=u.prompt_tokens,
output_tokens=u.completion_tokens,
time=(datetime.now() - start_datetime).total_seconds(),
- metadata=u,
+ cache_input_tokens=getattr(
+ u,
+ "cached_prompt_text_tokens",
+ 0,
+ ),
)
final_kwargs: dict[str, Any] = {
@@ -392,7 +396,11 @@ class XAIChatModel(ChatModelBase):
input_tokens=u.prompt_tokens,
output_tokens=u.completion_tokens,
time=(datetime.now() - start_datetime).total_seconds(),
- metadata=u,
+ cache_input_tokens=getattr(
+ u,
+ "cached_prompt_text_tokens",
+ 0,
+ ),
)
resp_kwargs: dict[str, Any] = {