Skip to content

Drop in a screenshot and convert it to clean code (HTML/Tailwind/React/Vue)

Python

Unified token usage tracking across all providers

Claude760676665c59
diff --git a/backend/agent/providers/anthropic.py b/backend/agent/providers/anthropic.py
index 1865feec..73021673 100644
--- a/backend/agent/providers/anthropic.py
+++ b/backend/agent/providers/anthropic.py
@@ -18,6 +18,7 @@ from agent.providers.base import (
ProviderSession,
ProviderTurn,
StreamEvent,
+ TokenUsage,
)
from agent.tools import CanonicalToolDefinition, ToolCall, parse_json_arguments
from llm import Llm
@@ -224,6 +225,28 @@ def _extract_tool_calls(final_message: Any) -> List[ToolCall]:
return tool_calls
+def _extract_anthropic_usage(final_message: Any) -> TokenUsage:
+ """Extract unified token usage from an Anthropic final message.
+
+ Anthropic includes thinking tokens in ``output_tokens`` so no extra
+ addition is needed. ``total`` is computed since the API doesn't provide it.
+ """
+ usage = getattr(final_message, "usage", None)
+ if usage is None:
+ return TokenUsage()
+ input_tokens = getattr(usage, "input_tokens", 0) or 0
+ output_tokens = getattr(usage, "output_tokens", 0) or 0
+ cache_read = getattr(usage, "cache_read_input_tokens", 0) or 0
+ cache_write = getattr(usage, "cache_creation_input_tokens", 0) or 0
+ return TokenUsage(
+ input=input_tokens,
+ output=output_tokens,
+ cache_read=cache_read,
+ cache_write=cache_write,
+ total=input_tokens + output_tokens + cache_read + cache_write,
+ )
+
+
class AnthropicProviderSession(ProviderSession):
def __init__(
self,
@@ -235,6 +258,7 @@ class AnthropicProviderSession(ProviderSession):
self._client = client
self._model = model
self._tools = tools
+ self._total_usage = TokenUsage()
system_prompt, claude_messages = _convert_openai_messages_to_claude(prompt_messages)
self._system_prompt = system_prompt
self._messages = claude_messages
@@ -272,6 +296,8 @@ class AnthropicProviderSession(ProviderSession):
await _parse_stream_event(event, state, on_event)
final_message = await stream.get_final_message()
+ self._total_usage.accumulate(_extract_anthropic_usage(final_message))
+
tool_calls = _extract_tool_calls(final_message)
return ProviderTurn(
assistant_text=state.assistant_text,
@@ -314,4 +340,11 @@ class AnthropicProviderSession(ProviderSession):
self._messages.append({"role": "user", "content": tool_result_blocks})
async def close(self) -> None:
+ u = self._total_usage
+ print(
+ f"[TOKEN USAGE] provider=anthropic model={self._model.value} | "
+ f"input={u.input} output={u.output} "
+ f"cache_read={u.cache_read} cache_write={u.cache_write} "
+ f"total={u.total}"
+ )
await self._client.close()
diff --git a/backend/agent/providers/base.py b/backend/agent/providers/base.py
index 604cc50c..33a820ff 100644
--- a/backend/agent/providers/base.py
+++ b/backend/agent/providers/base.py
@@ -20,6 +20,32 @@ class StreamEvent:
tool_arguments: Any = None
+@dataclass
+class TokenUsage:
+ """Unified token usage across all providers.
+
+ Fields:
+ input: Non-cached input (prompt) tokens.
+ output: Output tokens (includes thinking/reasoning where applicable).
+ cache_read: Cached input tokens read.
+ cache_write: Cached input tokens written (Anthropic only).
+ total: Total tokens as reported by the provider, or computed.
+ """
+
+ input: int = 0
+ output: int = 0
+ cache_read: int = 0
+ cache_write: int = 0
+ total: int = 0
+
+ def accumulate(self, other: "TokenUsage") -> None:
+ self.input += other.input
+ self.output += other.output
+ self.cache_read += other.cache_read
+ self.cache_write += other.cache_write
+ self.total += other.total
+
+
@dataclass
class ProviderTurn:
assistant_text: str
diff --git a/backend/agent/providers/gemini.py b/backend/agent/providers/gemini.py
index 497d4fd1..22d805f6 100644
--- a/backend/agent/providers/gemini.py
+++ b/backend/agent/providers/gemini.py
@@ -15,6 +15,7 @@ from agent.providers.base import (
ProviderSession,
ProviderTurn,
StreamEvent,
+ TokenUsage,
)
from agent.tools import CanonicalToolDefinition, ToolCall
from llm import Llm
@@ -173,15 +174,6 @@ def _convert_message_to_gemini_content(
return types.Content(role=gemini_role, parts=parts) # type: ignore
-@dataclass
-class GeminiTokenUsage:
- prompt_token_count: int = 0
- cached_content_token_count: int = 0
- candidates_token_count: int = 0
- thoughts_token_count: int = 0
- total_token_count: int = 0
-
-
@dataclass
class GeminiParseState:
assistant_text: str = ""
@@ -190,16 +182,23 @@ class GeminiParseState:
model_role: str = "model"
-def _extract_usage(chunk: types.GenerateContentResponse) -> GeminiTokenUsage | None:
+def _extract_usage(chunk: types.GenerateContentResponse) -> TokenUsage | None:
+ """Extract unified token usage from a Gemini streaming chunk.
+
+ Gemini reports thinking tokens separately; they are folded into ``output``
+ to match the unified schema used by the other providers.
+ """
meta = chunk.usage_metadata
if meta is None:
return None
- return GeminiTokenUsage(
- prompt_token_count=meta.prompt_token_count or 0,
- cached_content_token_count=meta.cached_content_token_count or 0,
- candidates_token_count=meta.candidates_token_count or 0,
- thoughts_token_count=meta.thoughts_token_count or 0,
- total_token_count=meta.total_token_count or 0,
+ candidates = meta.candidates_token_count or 0
+ thoughts = meta.thoughts_token_count or 0
+ return TokenUsage(
+ input=meta.prompt_token_count or 0,
+ output=candidates + thoughts,
+ cache_read=meta.cached_content_token_count or 0,
+ cache_write=0,
+ total=meta.total_token_count or 0,
)
@@ -265,22 +264,13 @@ class GeminiProviderSession(ProviderSession):
self._client = client
self._model = model
self._tools = tools
- self._total_usage = GeminiTokenUsage()
+ self._total_usage = TokenUsage()
self._system_prompt = str(prompt_messages[0].get("content", ""))
self._contents: List[types.Content] = [
_convert_message_to_gemini_content(msg) for msg in prompt_messages[1:]
]
- def _accumulate_usage(self, turn_usage: GeminiTokenUsage) -> None:
- self._total_usage.prompt_token_count += turn_usage.prompt_token_count
- self._total_usage.cached_content_token_count += (
- turn_usage.cached_content_token_count
- )
- self._total_usage.candidates_token_count += turn_usage.candidates_token_count
- self._total_usage.thoughts_token_count += turn_usage.thoughts_token_count
- self._total_usage.total_token_count += turn_usage.total_token_count
-
async def stream_turn(self, on_event: EventSink) -> ProviderTurn:
thinking_level = _get_thinking_level_for_model(self._model)
config = types.GenerateContentConfig(
@@ -301,7 +291,7 @@ class GeminiProviderSession(ProviderSession):
)
state = GeminiParseState()
- turn_usage: GeminiTokenUsage | None = None
+ turn_usage: TokenUsage | None = None
async for chunk in stream:
await _parse_chunk(chunk, state, on_event)
chunk_usage = _extract_usage(chunk)
@@ -309,7 +299,7 @@ class GeminiProviderSession(ProviderSession):
turn_usage = chunk_usage
if turn_usage is not None:
- self._accumulate_usage(turn_usage)
+ self._total_usage.accumulate(turn_usage)
assistant_turn = (
types.Content(role=state.model_role, parts=state.model_parts)
@@ -351,10 +341,8 @@ class GeminiProviderSession(ProviderSession):
u = self._total_usage
model_name = _get_gemini_api_model_name(self._model)
print(
- f"[GEMINI TOKEN USAGE] model={model_name} | "
- f"input={u.prompt_token_count} "
- f"cached={u.cached_content_token_count} "
- f"output={u.candidates_token_count} "
- f"thinking={u.thoughts_token_count} "
- f"total={u.total_token_count}"
+ f"[TOKEN USAGE] provider=gemini model={model_name} | "
+ f"input={u.input} output={u.output} "
+ f"cache_read={u.cache_read} cache_write={u.cache_write} "
+ f"total={u.total}"
)
diff --git a/backend/agent/providers/openai.py b/backend/agent/providers/openai.py
index 28c6d2cf..5735db45 100644
--- a/backend/agent/providers/openai.py
+++ b/backend/agent/providers/openai.py
@@ -14,6 +14,7 @@ from agent.providers.base import (
ProviderSession,
ProviderTurn,
StreamEvent,
+ TokenUsage,
)
from agent.state import ensure_str
from agent.tools import CanonicalToolDefinition, ToolCall, parse_json_arguments
@@ -128,6 +129,32 @@ class OpenAIResponsesParseState:
output_items_by_index: Dict[int, Dict[str, Any]] = field(default_factory=dict)
saw_reasoning_summary_text_delta: bool = False
last_emitted_reasoning_summary_part: str = ""
+ turn_usage: TokenUsage | None = None
+
+
+def _extract_openai_usage(response: Any) -> TokenUsage:
+ """Extract unified token usage from an OpenAI Responses ``response.completed`` event.
+
+ OpenAI includes cached tokens inside ``input_tokens``, so they are subtracted
+ to get the non-cached input count.
+ """
+ usage = _get_event_attr(response, "usage")
+ if usage is None:
+ return TokenUsage()
+ input_tokens = _get_event_attr(usage, "input_tokens", 0) or 0
+ output_tokens = _get_event_attr(usage, "output_tokens", 0) or 0
+ total_tokens = _get_event_attr(usage, "total_tokens", 0) or 0
+
+ details = _get_event_attr(usage, "input_tokens_details") or {}
+ cached_tokens = _get_event_attr(details, "cached_tokens", 0) or 0
+
+ return TokenUsage(
+ input=input_tokens - cached_tokens,
+ output=output_tokens,
+ cache_read=cached_tokens,
+ cache_write=0,
+ total=total_tokens,
+ )
async def parse_event(
@@ -142,6 +169,10 @@ async def parse_event(
"response.done",
"response.output_item.done",
):
+ if event_type == "response.completed":
+ response = _get_event_attr(event, "response")
+ if response:
+ state.turn_usage = _extract_openai_usage(response)
if event_type == "response.output_item.done":
output_index = _get_event_attr(event, "output_index")
item = _get_event_attr(event, "item")
@@ -383,6 +414,7 @@ class OpenAIProviderSession(ProviderSession):
self._client = client
self._model = model
self._tools = tools
+ self._total_usage = TokenUsage()
self._input_items: List[Dict[str, Any]] = [
_convert_message_to_responses_input(message) for message in prompt_messages
]
@@ -404,6 +436,10 @@ class OpenAIProviderSession(ProviderSession):
stream = await self._client.responses.create(**params) # type: ignore
async for event in stream: # type: ignore
await parse_event(event, state, on_event)
+
+ if state.turn_usage is not None:
+ self._total_usage.accumulate(state.turn_usage)
+
return _build_provider_turn(state)
def append_tool_results(
@@ -427,4 +463,12 @@ class OpenAIProviderSession(ProviderSession):
self._input_items.extend(tool_output_items)
async def close(self) -> None:
+ u = self._total_usage
+ model_name = get_openai_api_name(self._model)
+ print(
+ f"[TOKEN USAGE] provider=openai model={model_name} | "
+ f"input={u.input} output={u.output} "
+ f"cache_read={u.cache_read} cache_write={u.cache_write} "
+ f"total={u.total}"
+ )
await self._client.close()