Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(proxy/cost): count Gemini thinking tokens in output usage (#2639)
Abhay Singh22b707fd31d7
diff --git a/headroom/proxy/handlers/gemini.py b/headroom/proxy/handlers/gemini.pyindex 0d176d129..199a79fd1 100644--- a/headroom/proxy/handlers/gemini.py+++ b/headroom/proxy/handlers/gemini.py@@ -22,6 +22,7 @@ from headroom.proxy.auth_mode import classify_clientfrom headroom.proxy.compression_decision import CompressionDecisionfrom headroom.proxy.helpers import COMPRESSION_TIMEOUT_SECONDS, extract_tagsfrom headroom.proxy.outcome import RequestOutcome+from headroom.proxy.token_counting import gemini_output_tokenslogger = logging.getLogger("headroom.proxy")@@ -462,7 +463,9 @@ class GeminiHandlerMixin:# output_tokens) would then raise TypeError on the non-error# path. Mirrors the streaming _usage_int guard.total_input_tokens = _usage_int(usage.get("promptTokenCount"))- output_tokens = _usage_int(usage.get("candidatesTokenCount"))+ output_tokens = gemini_output_tokens(+ usage+ ) # includes thinking tokens (2.5-family)cache_read_tokens = _usage_int(usage.get("cachedContentTokenCount"))except (json.JSONDecodeError, ValueError, KeyError, TypeError, AttributeError):pass@@ -714,7 +717,9 @@ class GeminiHandlerMixin:if usage.get("promptTokenCount") is Noneelse usage["promptTokenCount"])- output_tokens = _usage_int(usage.get("candidatesTokenCount"))+ output_tokens = gemini_output_tokens(+ usage+ ) # includes thinking tokens (2.5-family)# Gemini returns cachedContentTokenCount for context-cached tokens# These are charged at 10-25% of the input price depending on modelcache_read_tokens = _usage_int(usage.get("cachedContentTokenCount"))diff --git a/headroom/proxy/handlers/openai.py b/headroom/proxy/handlers/openai.pyindex f380ab4c9..62b46c03e 100644--- a/headroom/proxy/handlers/openai.py+++ b/headroom/proxy/handlers/openai.py@@ -75,6 +75,7 @@ from headroom.proxy.passthrough import (custom_base_passthrough_telemetry as _custom_base_passthrough_telemetry,)from headroom.proxy.project_context import classify_project, set_current_project+from headroom.proxy.token_counting import gemini_output_tokenslogger = logging.getLogger("headroom.proxy")@@ -331,7 +332,7 @@ def _passthrough_usage_from_json(payload: Any) -> dict[str, int]:if isinstance(usage_meta, dict):return {"input_tokens": _usage_int(usage_meta.get("promptTokenCount")),- "output_tokens": _usage_int(usage_meta.get("candidatesTokenCount")),+ "output_tokens": gemini_output_tokens(usage_meta),"cache_read_input_tokens": _usage_int(usage_meta.get("cachedContentTokenCount")),}diff --git a/headroom/proxy/handlers/streaming.py b/headroom/proxy/handlers/streaming.pyindex 3b864f9dd..7ce93a9fb 100644--- a/headroom/proxy/handlers/streaming.py+++ b/headroom/proxy/handlers/streaming.py@@ -18,6 +18,7 @@ from headroom.proxy.helpers import (jitter_delay_ms,retry_after_ms,)+from headroom.proxy.token_counting import gemini_output_tokensif TYPE_CHECKING:from fastapi.responses import Response, StreamingResponse@@ -224,7 +225,7 @@ class StreamingMixin:usage_meta = data.get("usageMetadata")if usage_meta:usage["input_tokens"] = usage_meta.get("promptTokenCount", 0)- usage["output_tokens"] = usage_meta.get("candidatesTokenCount", 0)+ usage["output_tokens"] = gemini_output_tokens(usage_meta)# Gemini also has cachedContentTokenCount for context cachingusage["cache_read_input_tokens"] = usage_meta.get("cachedContentTokenCount", 0@@ -342,7 +343,7 @@ class StreamingMixin:usage_meta = data.get("usageMetadata")if usage_meta:usage_found["input_tokens"] = usage_meta.get("promptTokenCount", 0)- usage_found["output_tokens"] = usage_meta.get("candidatesTokenCount", 0)+ usage_found["output_tokens"] = gemini_output_tokens(usage_meta)usage_found["cache_read_input_tokens"] = usage_meta.get("cachedContentTokenCount", 0)diff --git a/headroom/proxy/token_counting.py b/headroom/proxy/token_counting.pyindex fb9ca107b..0e5c10017 100644--- a/headroom/proxy/token_counting.py+++ b/headroom/proxy/token_counting.py@@ -77,3 +77,33 @@ async def count_texts_offloaded(owner: Any, model: Any, texts: Any) -> tuple[Anyreturn await _count_offloaded(owner, model, lambda counter: sum(counter.count_text(text) for text in text_list))+++def gemini_output_tokens(usage_meta: dict[str, Any]) -> int:+ """Output-token count for a Gemini ``usageMetadata``, including thinking tokens.++ Gemini reports ``candidatesTokenCount`` sometimes inclusive of the+ ``thoughtsTokenCount`` (2.5-family reasoning) and sometimes exclusive of it.+ When ``promptTokenCount + candidatesTokenCount != totalTokenCount`` the+ thinking tokens are a separate bucket and must be added, or the output cost+ (billed at the output rate) is undercounted. Mirrors litellm's+ ``is_candidate_token_count_inclusive`` rule. Robust to missing/null fields.+ """++ def _int(value: Any) -> int:+ try:+ return max(int(value), 0)+ except (TypeError, ValueError):+ return 0++ candidates = _int(usage_meta.get("candidatesTokenCount"))+ thoughts = _int(usage_meta.get("thoughtsTokenCount"))+ if thoughts <= 0:+ return candidates+ prompt = _int(usage_meta.get("promptTokenCount"))+ total = _int(usage_meta.get("totalTokenCount"))+ # Inclusive iff prompt + candidates already equals total; otherwise the+ # thinking tokens are a separate bucket that belongs in the output count.+ if prompt + candidates == total:+ return candidates+ return candidates + thoughtsdiff --git a/tests/test_proxy_handler_helpers.py b/tests/test_proxy_handler_helpers.pyindex bb86076aa..7845396b0 100644--- a/tests/test_proxy_handler_helpers.py+++ b/tests/test_proxy_handler_helpers.py@@ -381,6 +381,51 @@ def test_passthrough_usage_normalizes_vertex_usage_metadata() -> None:}+def test_gemini_output_tokens_includes_thinking_when_exclusive() -> None:+ """Gemini 2.5 thinking: when prompt + candidates != total, thoughtsTokenCount+ is a separate output bucket and must be added, or output cost undercounts."""+ from headroom.proxy.token_counting import gemini_output_tokens++ exclusive = {+ "promptTokenCount": 1000,+ "candidatesTokenCount": 200,+ "thoughtsTokenCount": 500,+ "totalTokenCount": 1700,+ }+ assert gemini_output_tokens(exclusive) == 700 # 200 visible + 500 thinking++ # Inclusive: candidatesTokenCount already covers thoughts (prompt+cand==total).+ inclusive = {+ "promptTokenCount": 1000,+ "candidatesTokenCount": 700,+ "thoughtsTokenCount": 500,+ "totalTokenCount": 1700,+ }+ assert gemini_output_tokens(inclusive) == 700++ # No thinking tokens: just the candidates count (common non-2.5 case).+ assert gemini_output_tokens({"candidatesTokenCount": 42, "totalTokenCount": 100}) == 42+ # Robust to empty / missing fields.+ assert gemini_output_tokens({}) == 0+++def test_passthrough_usage_counts_gemini_thinking_tokens() -> None:+ """_passthrough_usage_from_json must include thinking tokens in output_tokens."""+ usage = _passthrough_usage_from_json(+ {+ "usageMetadata": {+ "promptTokenCount": 1000,+ "candidatesTokenCount": 200,+ "thoughtsTokenCount": 500,+ "totalTokenCount": 1700,+ "cachedContentTokenCount": 100,+ }+ }+ )+ assert usage["output_tokens"] == 700+ assert usage["input_tokens"] == 1000++def test_vertex_passthrough_records_usage_metadata_for_dashboard() -> None:handler = object.__new__(HeadroomProxy)handler.http_client = _VertexUsageClient()