Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(proxy/openai): None-guard usage token counts on the chat path (#2431)
Abhay Singh313c290df96c
diff --git a/headroom/proxy/handlers/openai.py b/headroom/proxy/handlers/openai.pyindex 455132142..39926a430 100644--- a/headroom/proxy/handlers/openai.py+++ b/headroom/proxy/handlers/openai.py@@ -3665,8 +3665,15 @@ class OpenAIHandlerMixin:# cache stats from the LAST upstream call.total_latency = (time.time() - start_time) * 1000usage = backend_response.body.get("usage", {})- output_tokens = usage.get("completion_tokens", 0)- total_input_tokens = usage.get("prompt_tokens", optimized_tokens)+ # `.get(key, default)` only falls back when the key is+ # absent; a present-but-null count (some OpenAI-compatible+ # backends emit these on a stopped/empty turn) would return+ # None and crash the downstream `max(...)` arithmetic and the+ # int-typed outcome/metrics. `_usage_int` coerces both cases,+ # matching the streaming path and the guarded cache keys below+ # (same class as the gemini fix in #2347).+ output_tokens = _usage_int(usage.get("completion_tokens"))+ total_input_tokens = _usage_int(usage.get("prompt_tokens")) or optimized_tokens# Cache stats: prefer the Anthropic/Bedrock top-level# keys when present (authoritative). Fall back to@@ -3976,12 +3983,17 @@ class OpenAIHandlerMixin:try:resp_json = response.json()usage = resp_json.get("usage", {})- total_input_tokens = usage.get("prompt_tokens", optimized_tokens)- output_tokens = usage.get("completion_tokens", 0)+ # Coerce present-but-null counts: the arithmetic below+ # (`_infer_openai_cache_write_tokens`, `max(...)`) runs+ # outside this try, so a null `prompt_tokens`/`cached_tokens`+ # would otherwise raise an uncaught TypeError and 500 the+ # request (same class as the gemini fix in #2347).+ total_input_tokens = _usage_int(usage.get("prompt_tokens")) or optimized_tokens+ output_tokens = _usage_int(usage.get("completion_tokens"))# OpenAI returns cached_tokens in prompt_tokens_details# These are charged at 50% of the input priceprompt_details = usage.get("prompt_tokens_details") or {}- cache_read_tokens = prompt_details.get("cached_tokens", 0)+ cache_read_tokens = _usage_int(prompt_details.get("cached_tokens"))except (KeyError, TypeError, AttributeError) as e:logger.debug(f"[{request_id}] Failed to extract cached tokens from OpenAI response: {e}"diff --git a/tests/test_proxy/test_openai_chat_savings_profile.py b/tests/test_proxy/test_openai_chat_savings_profile.pyindex 3cf8ddb2a..827d2d569 100644--- a/tests/test_proxy/test_openai_chat_savings_profile.py+++ b/tests/test_proxy/test_openai_chat_savings_profile.py@@ -50,6 +50,67 @@ def _make_mock_backend() -> MagicMock:return backend+def _make_mock_backend_with_usage(usage: dict) -> MagicMock:+ backend = MagicMock()+ backend.name = "anyllm-openai"+ backend.send_openai_message = AsyncMock(+ return_value=BackendResponse(+ body={+ "id": "chatcmpl-1",+ "object": "chat.completion",+ "model": "gpt-4o",+ "choices": [+ {+ "index": 0,+ "message": {"role": "assistant", "content": "ok"},+ "finish_reason": "stop",+ }+ ],+ "usage": usage,+ },+ status_code=200,+ headers={"content-type": "application/json"},+ )+ )+ return backend+++def test_chat_completions_survives_null_usage_token_counts():+ """A backend that reports present-but-null token counts must not 500.++ `.get(key, default)` returns None for a null value, and the chat path+ feeds those counts into `max(...)`/int-typed metrics. Without coercion a+ single such response crashes the request and its outcome recording+ (same class as the gemini fix in #2347).+ """+ config = ProxyConfig(+ optimize=True,+ cache_enabled=False,+ rate_limit_enabled=False,+ backend="anyllm",+ anyllm_provider="openai",+ )++ # prompt_tokens / completion_tokens present but explicitly null.+ mock_backend = _make_mock_backend_with_usage(+ {"prompt_tokens": None, "completion_tokens": None, "total_tokens": None}+ )+ with patch("headroom.proxy.server.AnyLLMBackend", return_value=mock_backend):+ app = create_app(config)+ with TestClient(app) as client:+ resp = client.post(+ "/v1/chat/completions",+ json={+ "model": "gpt-4o",+ "messages": [{"role": "user", "content": "hello"}],+ "stream": False,+ },+ headers={"Authorization": "Bearer test-key"},+ )++ assert resp.status_code == 200, resp.text++def test_chat_completions_threads_savings_profile_kwargs_into_apply():"""With HEADROOM_SAVINGS_PROFILE=agent-90, the chat path must pass theprofile knobs (compress_user_messages, target_ratio, ...) to apply()."""