Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

refactor(proxy): one copy of the handler helpers duplicated across providers (#4128)

Tejas Chopraefd6e157e5db
diff --git a/headroom/cache/prefix_tracker.py b/headroom/cache/prefix_tracker.py
index bf5dc2e61..58ded7b05 100644
--- a/headroom/cache/prefix_tracker.py
+++ b/headroom/cache/prefix_tracker.py
@@ -589,6 +589,36 @@ def extract_cache_stable_delta(
)
+def restore_frozen_prefix(
+ original_messages: list[dict[str, Any]],
+ candidate_messages: list[dict[str, Any]],
+ *,
+ frozen_message_count: int,
+) -> tuple[list[dict[str, Any]], int]:
+ """Force the frozen prefix of ``candidate_messages`` to match the original bytes.
+
+ Returns ``(restored_messages, restored_count)``. Messages a transform dropped
+ from the prefix are put back; any frozen message it changed is replaced by
+ the original. ``restored_count`` is how many prefix messages were restored.
+ """
+ if frozen_message_count <= 0 or not original_messages:
+ return candidate_messages, 0
+
+ frozen = min(frozen_message_count, len(original_messages))
+ restored = list(candidate_messages)
+
+ # Defensive: if a transform dropped prefix messages, restore them.
+ if len(restored) < frozen:
+ return list(original_messages[:frozen]) + restored, frozen
+
+ changed = 0
+ for idx in range(frozen):
+ if restored[idx] != original_messages[idx]:
+ restored[idx] = original_messages[idx]
+ changed += 1
+ return restored, changed
+
+
def overlay_cached_prefix(
optimized_messages: list[dict[str, Any]],
current_original_messages: list[dict[str, Any]],
diff --git a/headroom/ccr/tool_calls.py b/headroom/ccr/tool_calls.py
index 8318b0564..8261e31e2 100644
--- a/headroom/ccr/tool_calls.py
+++ b/headroom/ccr/tool_calls.py
@@ -109,6 +109,25 @@ def is_ccr_tool_call(tool_call: dict[str, Any]) -> bool:
)
+def has_ccr_retrieve_tool(tools: Any) -> bool:
+ """Return true when a tool list defines the CCR retrieval tool.
+
+ Accepts flat definitions (Anthropic, OpenAI Responses: ``{"name": ...}``) and
+ chat-completions definitions nested under ``"function"``.
+ """
+ if not isinstance(tools, list):
+ return False
+ for tool in tools:
+ if not isinstance(tool, dict):
+ continue
+ if tool.get("name") == CCR_TOOL_NAME:
+ return True
+ function = tool.get("function")
+ if isinstance(function, dict) and function.get("name") == CCR_TOOL_NAME:
+ return True
+ return False
+
+
def has_ccr_tool_calls(response: dict[str, Any], provider: str) -> bool:
"""Return true when ``response`` contains at least one CCR tool call."""
return any(is_ccr_tool_call(tool_call) for tool_call in extract_tool_calls(response, provider))
diff --git a/headroom/proxy/handlers/anthropic.py b/headroom/proxy/handlers/anthropic.py
index ed5550045..10d6b8ba7 100644
--- a/headroom/proxy/handlers/anthropic.py
+++ b/headroom/proxy/handlers/anthropic.py
@@ -33,6 +33,7 @@ import httpx
from headroom.agent_savings import proxy_pipeline_kwargs
from headroom.ccr.context_tracker import looks_like_claude_code_compact_summary
from headroom.ccr.marker_resolution import resolve_markers_in_response
+from headroom.ccr.tool_calls import has_ccr_retrieve_tool
from headroom.copilot_auth import apply_copilot_api_auth, is_copilot_upstream_url
from headroom.pipeline import PipelineStage, summarize_routing_markers
from headroom.proxy import public_errors
@@ -68,6 +69,7 @@ from headroom.proxy.model_router import estimate_input_tokens, request_max_token
from headroom.proxy.nonstream_sse_policy import should_recover_sse_reply
from headroom.proxy.outcome import RequestOutcome
from headroom.proxy.output_shaper import shaper_enabled_for, steering_allowed_for
+from headroom.proxy.provider_usage import anthropic_cache_ttl_buckets, usage_int
from headroom.proxy.rate_limit_identity import rate_limit_identity
from headroom.proxy.semantic_cache_key_policy import compute_request_cache_partition
from headroom.proxy.tenant_key import resolve_tenant_key, set_request_tenant_key
@@ -171,28 +173,21 @@ class _AnthropicTurnHookUsage:
self.cache_write_1h_tokens = 0
self.extra_calls = 0
- @staticmethod
- def _int(value: Any) -> int:
- try:
- return max(0, int(value or 0))
- except (TypeError, ValueError):
- return 0
-
def record(self, payload: Any) -> None:
usage = payload.get("usage") if isinstance(payload, dict) else None
if not isinstance(usage, dict):
self._seen.append((payload, 0, 0, 0, 0, 0, 0))
return
- cache_read = self._int(usage.get("cache_read_input_tokens"))
- cache_write = self._int(usage.get("cache_creation_input_tokens"))
+ cache_read = usage_int(usage.get("cache_read_input_tokens"))
+ cache_write = usage_int(usage.get("cache_creation_input_tokens"))
creation = usage.get("cache_creation")
cache_write_5m = (
- self._int(creation.get("ephemeral_5m_input_tokens"))
+ usage_int(creation.get("ephemeral_5m_input_tokens"))
if isinstance(creation, dict)
else 0
)
cache_write_1h = (
- self._int(creation.get("ephemeral_1h_input_tokens"))
+ usage_int(creation.get("ephemeral_1h_input_tokens"))
if isinstance(creation, dict)
else 0
)
@@ -200,7 +195,7 @@ class _AnthropicTurnHookUsage:
(
payload,
_anthropic_provider_input_tokens(usage),
- self._int(usage.get("output_tokens")),
+ usage_int(usage.get("output_tokens")),
cache_read,
cache_write,
cache_write_5m,
@@ -240,7 +235,7 @@ def _anthropic_provider_input_tokens(usage: Any) -> int:
if not isinstance(usage, dict):
return 0
return sum(
- _AnthropicTurnHookUsage._int(usage.get(key))
+ usage_int(usage.get(key))
for key in (
"input_tokens",
"cache_read_input_tokens",
@@ -505,21 +500,6 @@ class AnthropicHandlerMixin:
and name not in client_tool_names
)
- @staticmethod
- def _has_headroom_retrieve_tool(tools: Any) -> bool:
- """Return True when the final Anthropic tool list includes CCR retrieve."""
- if not isinstance(tools, list):
- return False
- for tool in tools:
- if not isinstance(tool, dict):
- continue
- if tool.get("name") == "headroom_retrieve":
- return True
- function = tool.get("function")
- if isinstance(function, dict) and function.get("name") == "headroom_retrieve":
- return True
- return False
-
def _can_salvage_buffered_upstream(self, resp_json: Any) -> bool:
"""May this upstream response be relayed when post-processing failed?
@@ -590,19 +570,10 @@ class AnthropicHandlerMixin:
def _extract_anthropic_cache_ttl_metrics(usage: dict[str, Any] | None) -> tuple[int, int]:
"""Extract observed Anthropic cache-write TTL bucket usage.
- HeadroomProxy also inherits StreamingMixin, which exposes the same
- helper for SSE usage parsing. Keep this local copy so the Anthropic
- handler remains safe when tested or embedded without StreamingMixin.
+ StreamingMixin defines the same method; both delegate to one helper so
+ this handler stays usable without StreamingMixin (tests, embedding).
"""
- if not isinstance(usage, dict):
- return (0, 0)
- cache_creation = usage.get("cache_creation")
- if not isinstance(cache_creation, dict):
- return (0, 0)
- return (
- int(cache_creation.get("ephemeral_5m_input_tokens", 0) or 0),
- int(cache_creation.get("ephemeral_1h_input_tokens", 0) or 0),
- )
+ return anthropic_cache_ttl_buckets(usage)
def _anthropic_buffered_request_timeout(self) -> httpx.Timeout:
"""Timeout for buffered Anthropic reads."""
@@ -790,22 +761,13 @@ class AnthropicHandlerMixin:
frozen_message_count: int,
) -> tuple[list[dict[str, Any]], int]:
"""Force frozen prefix bytes to match the original request exactly."""
- if frozen_message_count <= 0 or not original_messages:
- return candidate_messages, 0
-
- frozen = min(frozen_message_count, len(original_messages))
- restored = list(candidate_messages)
+ from headroom.cache.prefix_tracker import restore_frozen_prefix
- # Defensive: if a transform dropped prefix messages, restore them.
- if len(restored) < frozen:
- return list(original_messages[:frozen]) + restored, frozen
-
- changed = 0
- for idx in range(frozen):
- if restored[idx] != original_messages[idx]:
- restored[idx] = original_messages[idx]
- changed += 1
- return restored, changed
+ return restore_frozen_prefix(
+ original_messages,
+ candidate_messages,
+ frozen_message_count=frozen_message_count,
+ )
@staticmethod
def _extract_cache_stable_delta(
@@ -4006,9 +3968,7 @@ class AnthropicHandlerMixin:
retrieve_tool_is_offered = (
stream
and ccr_response_handler_enabled
- and self._has_headroom_retrieve_tool(
- tools if tools is not None else body.get("tools")
- )
+ and has_ccr_retrieve_tool(tools if tools is not None else body.get("tools"))
)
buffered_retrieval_can_help = (
retrieve_tool_is_offered and self._outgoing_body_has_redeemable_marker(body)
diff --git a/headroom/proxy/handlers/openai.py b/headroom/proxy/handlers/openai.py
index 27bbef992..9604102ee 100644
--- a/headroom/proxy/handlers/openai.py
+++ b/headroom/proxy/handlers/openai.py
@@ -37,7 +37,11 @@ from headroom.proxy.helpers import (
from headroom.proxy.identity import resolve_memory_identity
from headroom.proxy.loopback_guard import is_loopback_host
from headroom.proxy.modes import is_cache_mode
-from headroom.proxy.provider_usage import billed_input_for_provider, billed_input_from_usage
+from headroom.proxy.provider_usage import (
+ billed_input_for_provider,
+ billed_input_from_usage,
+ usage_int,
+)
from headroom.proxy.rate_limit_identity import rate_limit_identity
from headroom.proxy.semantic_cache_key_policy import compute_request_cache_partition
from headroom.proxy.stage_timer import StageTimer, emit_stage_timings_log
@@ -60,6 +64,7 @@ import httpx
from headroom.agent_savings import proxy_pipeline_kwargs
from headroom.ccr.marker_resolution import resolve_markers_in_response
+from headroom.ccr.tool_calls import has_ccr_retrieve_tool
from headroom.config import is_tool_excluded, unwrap_tool_call_name
from headroom.copilot_auth import (
apply_copilot_api_auth,
@@ -361,20 +366,14 @@ class TurnHookUsage:
500 a request by returning an odd shape."""
usage = payload.get("usage") if isinstance(payload, dict) else None
- def _int(value: Any) -> int:
- try:
- return max(int(value), 0)
- except (TypeError, ValueError):
- return 0
-
if isinstance(usage, dict):
details = usage.get(details_key)
- cached = _int(details.get("cached_tokens")) if isinstance(details, dict) else 0
+ cached = usage_int(details.get("cached_tokens")) if isinstance(details, dict) else 0
entry = (
id(payload),
payload,
- _int(usage.get(input_key)),
- _int(usage.get(output_key)),
+ usage_int(usage.get(input_key)),
+ usage_int(usage.get(output_key)),
cached,
)
else:
@@ -578,13 +577,6 @@ def _is_allowed_websocket_origin(headers: dict[str, str]) -> bool:
return normalized_origin in normalized_allowed
-def _usage_int(value: Any) -> int:
- try:
- return max(int(value), 0)
- except (TypeError, ValueError):
- return 0
-
-
def _passthrough_usage_from_json(payload: Any, provider: str | None = None) -> dict[str, int]:
"""Normalize usage from pass-through provider response shapes."""
if not isinstance(payload, dict):
@@ -593,9 +585,9 @@ def _passthrough_usage_from_json(payload: Any, provider: str | None = None) -> d
usage_meta = payload.get("usageMetadata")
if isinstance(usage_meta, dict):
return {
- "input_tokens": _usage_int(usage_meta.get("promptTokenCount")),
+ "input_tokens": usage_int(usage_meta.get("promptTokenCount")),
"output_tokens": gemini_output_tokens(usage_meta),
- "cache_read_input_tokens": _usage_int(usage_meta.get("cachedContentTokenCount")),
+ "cache_read_input_tokens": usage_int(usage_meta.get("cachedContentTokenCount")),
}
usage = payload.get("usage")
@@ -613,10 +605,10 @@ def _passthrough_usage_from_json(payload: Any, provider: str | None = None) -> d
# the uncached tail only (cache buckets are disjoint); OpenAI's
# headline figure is already inclusive. Callers derive uncached as
# total - read - write, which is only right on the total.
- "input_tokens": billed_input_from_usage(usage, provider) or _usage_int(input_tokens),
- "output_tokens": _usage_int(output_tokens),
- "cache_read_input_tokens": _usage_int(usage.get("cache_read_input_tokens", cache_read)),
- "cache_creation_input_tokens": _usage_int(usage.get("cache_creation_input_tokens")),
+ "input_tokens": billed_input_from_usage(usage, provider) or usage_int(input_tokens),
+ "output_tokens": usage_int(output_tokens),
+ "cache_read_input_tokens": usage_int(usage.get("cache_read_input_tokens", cache_read)),
+ "cache_creation_input_tokens": usage_int(usage.get("cache_creation_input_tokens")),
}
return {}
@@ -1217,29 +1209,6 @@ def _responses_input_to_learner_messages(
return messages
-def _has_headroom_retrieve_tool_responses(tools: Any) -> bool:
- """Return True when the Responses API tool list includes CCR retrieve.
-
- Responses API tool defs are flat (``{"type": "function", "name": ...}``)
- rather than nested under a "function" key like chat-completions
- tool_calls, so this can't reuse the chat-completions tool-list check.
- Mirrors ``AnthropicHandler._has_headroom_retrieve_tool``.
- """
- from headroom.ccr import CCR_TOOL_NAME
-
- if not isinstance(tools, list):
- return False
- for tool in tools:
- if not isinstance(tool, dict):
- continue
- if tool.get("name") == CCR_TOOL_NAME:
- return True
- function = tool.get("function")
- if isinstance(function, dict) and function.get("name") == CCR_TOOL_NAME:
- return True
- return False
-
-
def _should_buffer_openai_responses_stream_ccr(
*,
stream: bool,
@@ -1261,7 +1230,7 @@ def _should_buffer_openai_responses_stream_ccr(
and ccr_response_handler_enabled
and not is_chatgpt_auth
and not is_opencode_zen_base(upstream_base_url)
- and _has_headroom_retrieve_tool_responses(tools)
+ and has_ccr_retrieve_tool(tools)
)
@@ -1714,16 +1683,10 @@ def _extract_responses_usage(event: dict[str, Any]) -> tuple[int, int, int, int,
if not isinstance(usage, dict):
return 0, 0, 0, 0, 0
- def _int(value: Any) -> int:
- try:
- return max(int(value), 0)
- except (TypeError, ValueError):
- return 0
-
- input_tokens = _int(usage.get("input_tokens"))
- output_tokens = _int(usage.get("output_tokens"))
+ input_tokens = usage_int(usage.get("input_tokens"))
+ output_tokens = usage_int(usage.get("output_tokens"))
details = usage.get("input_tokens_details")
- cached_tokens = _int(details.get("cached_tokens")) if isinstance(details, dict) else 0
+ cached_tokens = usage_int(details.get("cached_tokens")) if isinstance(details, dict) else 0
cache_write_tokens = _infer_openai_cache_write_tokens(input_tokens, cached_tokens)
uncached_tokens = max(input_tokens - cached_tokens, 0)
return input_tokens, output_tokens, cached_tokens, cache_write_tokens, uncached_tokens
@@ -2000,22 +1963,14 @@ class OpenAIHandlerMixin:
*,
frozen_message_count: int,
) -> tuple[list[dict[str, Any]], int]:
- """Force frozen prefix bytes to match original request exactly."""
- if frozen_message_count <= 0 or not original_messages:
- return candidate_messages, 0
-
- frozen = min(frozen_message_count, len(original_messages))
- restored = list(candidate_messages)
-
- if len(restored) < frozen:
- return list(original_messages[:frozen]) + restored, frozen
+ """Force frozen prefix bytes to match the original request exactly."""
+ from headroom.cache.prefix_tracker import restore_frozen_prefix
- changed = 0
- for idx in range(frozen):
- if restored[idx] != original_messages[idx]:
- restored[idx] = original_messages[idx]
- changed += 1
- return restored, changed
+ return restore_frozen_prefix(
+ original_messages,
+ candidate_messages,
+ frozen_message_count=frozen_message_count,
+ )
def _compress_openai_responses_live_text_units_with_router(
self,
@@ -4740,12 +4695,12 @@ class OpenAIHandlerMixin:
# absent; a present-but-null count (some OpenAI-compatible
# backends emit these on a stopped/empty turn) would return
# None and crash the downstream `max(...)` arithmetic and the
- # int-typed outcome/metrics. `_usage_int` coerces both cases,
+ # int-typed outcome/metrics. `usage_int` coerces both cases,
# matching the streaming path and the guarded cache keys below
# (same class as the gemini fix in #2347).
- output_tokens = _usage_int(usage.get("completion_tokens"))
+ output_tokens = usage_int(usage.get("completion_tokens"))
_thinking = extract_from_usage(usage)
- total_input_tokens = _usage_int(usage.get("prompt_tokens")) or optimized_tokens
+ total_input_tokens = usage_int(usage.get("prompt_tokens")) or optimized_tokens
# Cache stats: prefer the Anthropic/Bedrock top-level
# keys when present (authoritative). Fall back to
@@ -5247,13 +5202,13 @@ class OpenAIHandlerMixin:
# outside this try, so a null `prompt_tokens`/`cached_tokens`
# would otherwise raise an uncaught TypeError and 500 the
# request (same class as the gemini fix in #2347).
- total_input_tokens = _usage_int(usage.get("prompt_tokens")) or optimized_tokens
- output_tokens = _usage_int(usage.get("completion_tokens"))
+ total_input_tokens = usage_int(usage.get("prompt_tokens")) or optimized_tokens
+ output_tokens = usage_int(usage.get("completion_tokens"))
_thinking = extract_from_usage(usage)
# OpenAI returns cached_tokens in prompt_tokens_details
# These are charged at 50% of the input price
prompt_details = usage.get("prompt_tokens_details") or {}
- cache_read_tokens = _usage_int(prompt_details.get("cached_tokens"))
+ cache_read_tokens = usage_int(prompt_details.get("cached_tokens"))
except (KeyError, TypeError, AttributeError) as e:
logger.debug(
f"[{request_id}] Failed to extract cached tokens from OpenAI response: {e}"
@@ -6521,21 +6476,15 @@ class OpenAIHandlerMixin:
resp_json = response.json()
usage = resp_json.get("usage", {})
- def _usage_int(value: Any, default: int = 0) -> int:
- try:
- return max(int(value), 0)
- except (TypeError, ValueError):
- return default
-
- total_input_tokens = _usage_int(
+ total_input_tokens = usage_int(
usage.get("input_tokens"),
original_tokens,
)
- output_tokens = _usage_int(usage.get("output_tokens"))
+ output_tokens = usage_int(usage.get("output_tokens"))
_thinking = extract_from_usage(usage)
details = usage.get("input_tokens_details")
if isinstance(details, dict):
- cache_read_tokens = _usage_int(details.get("cached_tokens"))
+ cache_read_tokens = usage_int(details.get("cached_tokens"))
except (
json.JSONDecodeError,
ValueError,
diff --git a/headroom/proxy/handlers/streaming.py b/headroom/proxy/handlers/streaming.py
index 63cdc8b61..8efcbb39a 100644
--- a/headroom/proxy/handlers/streaming.py
+++ b/headroom/proxy/handlers/streaming.py
@@ -22,7 +22,11 @@ from headroom.proxy.helpers import (
overload_retry_is_futile,
retry_after_ms,
)
-from headroom.proxy.provider_usage import billed_input_for_provider
+from headroom.proxy.provider_usage import (
+ anthropic_cache_ttl_buckets,
+ billed_input_for_provider,
+ usage_int,
+)
from headroom.proxy.token_counting import gemini_output_tokens
if TYPE_CHECKING:
@@ -38,7 +42,6 @@ from headroom.proxy.memory_tool_stream import (
MemoryToolStreamOverflowError,
)
from headroom.proxy.stream_output_tokens import estimate_output_tokens
-from headroom.proxy.thinking_tokens import ThinkingTokens, extract_thinking_tokens
from headroom.utils import format_exception_message
logger = logging.getLogger("headroom.proxy")
@@ -172,23 +175,6 @@ def _upstream_error_frame(error_bytes: bytes, request_id: str) -> bytes:
return f"event: error\ndata: {json.dumps(payload)}\n\n".encode()
-def _thinking_for_stream(payload: object) -> ThinkingTokens:
- """Thinking-token split for a reassembled streaming response.
-
- Shares the estimator with the non-streaming path so a streamed and an
- unstreamed response of the same shape produce the same number — otherwise
- the stratum would mix two different rulers.
-
- Never raises: accounting must not cost a caller their response.
- """
- try:
- from headroom.proxy.handlers.anthropic import _thinking_estimator
-
- return extract_thinking_tokens(payload, estimator=_thinking_estimator())
- except Exception: # noqa: BLE001 - accounting must never break a response
- return ThinkingTokens()
-
-
def _sse_dict(value: Any) -> dict[str, Any]:
"""Return an upstream SSE object, or an empty object for a wrong-shaped value."""
return value if isinstance(value, dict) else {}
@@ -301,15 +287,7 @@ class StreamingMixin:
@staticmethod
def _extract_anthropic_cache_ttl_metrics(usage: dict[str, Any] | None) -> tuple[int, int]:
"""Extract observed Anthropic cache-write TTL bucket usage."""
- if not isinstance(usage, dict):
- return (0, 0)
- cache_creation = usage.get("cache_creation")
- if not isinstance(cache_creation, dict):
- return (0, 0)
- return (
- int(cache_creation.get("ephemeral_5m_input_tokens", 0) or 0),
- int(cache_creation.get("ephemeral_1h_input_tokens", 0) or 0),
- )
+ return anthropic_cache_ttl_buckets(usage)
def _parse_sse_usage_from_buffer(
self, stream_state: dict[str, Any], provider: str
@@ -402,13 +380,6 @@ class StreamingMixin:
if isinstance(response, dict):
chunk_usage = response.get("usage")
if isinstance(chunk_usage, dict):
-
- def _usage_int(value: Any) -> int:
- try:
- return max(int(value), 0)
- except (TypeError, ValueError):
- return 0
-
# Chat Completions streams report prompt/completion tokens.
# Responses streams report input/output tokens under
# response.usage on response.completed.
@@ -418,15 +389,15 @@ class StreamingMixin:
output_tokens = chunk_usage.get("completion_tokens")
if output_tokens is None:
output_tokens = chunk_usage.get("output_tokens", 0)
- usage_found["input_tokens"] = _usage_int(input_tokens)
- usage_found["output_tokens"] = _usage_int(output_tokens)
+ usage_found["input_tokens"] = usage_int(input_tokens)
+ usage_found["output_tokens"] = usage_int(output_tokens)
details = (
chunk_usage.get("prompt_tokens_details")
or chunk_usage.get("input_tokens_details")
or {}
)
if isinstance(details, dict):
- usage_found["cache_read_input_tokens"] = _usage_int(
+ usage_found["cache_read_input_tokens"] = usage_int(
details.get("cached_tokens")
)
@@ -1002,8 +973,11 @@ class StreamingMixin:
#
# ``parsed_response`` is the reassembled SSE body: it carries the content
# blocks (thinking among them) and stop_reason, which the raw usage chunk
- # does not.
- _stream_thinking = _thinking_for_stream(parsed_response)
+ # does not. Same helper as the non-streaming path, so a streamed and an
+ # unstreamed response of the same shape count the same.
+ from headroom.proxy.handlers.anthropic import _thinking_tokens_for
+
+ _stream_thinking = _thinking_tokens_for(parsed_response)
_stream_stop_reason = (
parsed_response.get("stop_reason") if isinstance(parsed_response, dict) else None
)
diff --git a/headroom/proxy/provider_usage.py b/headroom/proxy/provider_usage.py
index 45e774cdb..6e3aa2599 100644
--- a/headroom/proxy/provider_usage.py
+++ b/headroom/proxy/provider_usage.py
@@ -41,13 +41,36 @@ __all__ = [
_ANTHROPIC_SHAPE = frozenset({"anthropic", "bedrock", "vertex", "vertex_ai", "claude"})
-def _int(value: Any) -> int:
+def usage_int(value: Any, default: int = 0) -> int:
+ """One token count from a provider usage block, as a non-negative int.
+
+ Missing (``None``), non-numeric and boolean values give ``default``;
+ negative counts clamp to 0. Never raises: usage fields are
+ upstream-controlled and must not fail a response.
+ """
if isinstance(value, bool):
- return 0
+ return default
try:
return max(int(value), 0)
except (TypeError, ValueError):
- return 0
+ return default
+
+
+def anthropic_cache_ttl_buckets(usage: Any) -> tuple[int, int]:
+ """Cache-write tokens by TTL bucket ``(5m, 1h)`` from an Anthropic usage block.
+
+ Reads ``usage.cache_creation.ephemeral_{5m,1h}_input_tokens``; returns
+ ``(0, 0)`` when the block or its ``cache_creation`` object is absent.
+ """
+ if not isinstance(usage, dict):
+ return (0, 0)
+ cache_creation = usage.get("cache_creation")
+ if not isinstance(cache_creation, dict):
+ return (0, 0)
+ return (
+ int(cache_creation.get("ephemeral_5m_input_tokens", 0) or 0),
+ int(cache_creation.get("ephemeral_1h_input_tokens", 0) or 0),
+ )
def is_anthropic_dialect(provider: str | None) -> bool:
@@ -64,7 +87,7 @@ def is_anthropic_dialect(provider: str | None) -> bool:
def anthropic_billed_input(input_tokens: Any, cache_read: Any = 0, cache_write: Any = 0) -> int:
"""Billed input for an Anthropic-shape usage block: the three disjoint buckets."""
- return _int(input_tokens) + _int(cache_read) + _int(cache_write)
+ return usage_int(input_tokens) + usage_int(cache_read) + usage_int(cache_write)
def billed_input_for_provider(
@@ -88,7 +111,7 @@ def billed_input_for_provider(
# message_start can carry input_tokens=0 alongside real cache buckets
# on a fully cached turn; that is still a provider-reported count.
return total
- return _int(input_tokens)
+ return usage_int(input_tokens)
def billed_input_from_usage(payload: Mapping[str, Any] | None, provider: str | None) -> int:
@@ -104,14 +127,14 @@ def billed_input_from_usage(payload: Mapping[str, Any] | None, provider: str | N
meta = payload.get("usageMetadata")
if isinstance(meta, Mapping):
- return _int(meta.get("promptTokenCount"))
+ return usage_int(meta.get("promptTokenCount"))
usage = payload.get("usage") if isinstance(payload.get("usage"), Mapping) else payload
if not isinstance(usage, Mapping):
return 0
if "promptTokenCount" in usage:
- return _int(usage.get("promptTokenCount"))
+ return usage_int(usage.get("promptTokenCount"))
if is_anthropic_dialect(provider):
if "input_tokens" not in usage:
@@ -124,7 +147,7 @@ def billed_input_from_usage(payload: Mapping[str, Any] | None, provider: str | N
# OpenAI and OpenAI-compatible dialects: the headline figure is inclusive.
if "prompt_tokens" in usage:
- return _int(usage.get("prompt_tokens"))
+ return usage_int(usage.get("prompt_tokens"))
if "input_tokens" in usage:
- return _int(usage.get("input_tokens"))
+ return usage_int(usage.get("input_tokens"))
return 0
diff --git a/tests/test_provider_billed_input.py b/tests/test_provider_billed_input.py
index 142380ad0..04c29f7e0 100644
--- a/tests/test_provider_billed_input.py
+++ b/tests/test_provider_billed_input.py
@@ -29,6 +29,7 @@ from headroom.proxy.prometheus_metrics import PrometheusMetrics
from headroom.proxy.provider_usage import (
billed_input_for_provider,
billed_input_from_usage,
+ usage_int,
)
from headroom.proxy.server import HeadroomProxy
from headroom.telemetry.reporter import UsageReporter
@@ -295,3 +296,19 @@ def test_estimated_input_is_reported_as_estimated() -> None:
assert payload["tokens_after"] == 400
assert payload["tokens_after_provider_reported"] == 0
assert payload["tokens_after_estimated"] == 400
+
+
+ ("value", "default", "expected"),
+ [
+ (12, 0, 12),
+ ("7", 0, 7),
+ (-3, 0, 0),
+ (None, 0, 0),
+ (None, 41, 41),
+ ("n/a", 5, 5),
+ (True, 0, 0),
+ ],
+)
+def test_usage_int_coerces_upstream_counts(value, default, expected) -> None:
+ assert usage_int(value, default) == expected
diff --git a/tests/test_thinking_tokens.py b/tests/test_thinking_tokens.py
index 0403d8f41..b491b1d96 100644
--- a/tests/test_thinking_tokens.py
+++ b/tests/test_thinking_tokens.py
@@ -219,6 +219,9 @@ class TestHandlerWiring:
def test_helper_never_raises_on_junk(self):
from headroom.proxy.handlers.anthropic import _thinking_tokens_for
+ from headroom.proxy.thinking_tokens import ThinkingTokens
for junk in (None, "string", 42, [], {"content": "not a list"}):
- assert _thinking_tokens_for(junk).tokens is None or True
+ result = _thinking_tokens_for(junk)
+ assert isinstance(result, ThinkingTokens)
+ assert not result.tokens # nothing to count: unknown (None) or zero