Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(proxy/openai): None-guard usage token counts on the chat path (#2431)

Abhay Singh313c290df96c
diff --git a/headroom/proxy/handlers/openai.py b/headroom/proxy/handlers/openai.py
index 455132142..39926a430 100644
--- a/headroom/proxy/handlers/openai.py
+++ b/headroom/proxy/handlers/openai.py
@@ -3665,8 +3665,15 @@ class OpenAIHandlerMixin:
# cache stats from the LAST upstream call.
total_latency = (time.time() - start_time) * 1000
usage = backend_response.body.get("usage", {})
- output_tokens = usage.get("completion_tokens", 0)
- total_input_tokens = usage.get("prompt_tokens", optimized_tokens)
+ # `.get(key, default)` only falls back when the key is
+ # absent; a present-but-null count (some OpenAI-compatible
+ # backends emit these on a stopped/empty turn) would return
+ # None and crash the downstream `max(...)` arithmetic and the
+ # int-typed outcome/metrics. `_usage_int` coerces both cases,
+ # matching the streaming path and the guarded cache keys below
+ # (same class as the gemini fix in #2347).
+ output_tokens = _usage_int(usage.get("completion_tokens"))
+ total_input_tokens = _usage_int(usage.get("prompt_tokens")) or optimized_tokens
# Cache stats: prefer the Anthropic/Bedrock top-level
# keys when present (authoritative). Fall back to
@@ -3976,12 +3983,17 @@ class OpenAIHandlerMixin:
try:
resp_json = response.json()
usage = resp_json.get("usage", {})
- total_input_tokens = usage.get("prompt_tokens", optimized_tokens)
- output_tokens = usage.get("completion_tokens", 0)
+ # Coerce present-but-null counts: the arithmetic below
+ # (`_infer_openai_cache_write_tokens`, `max(...)`) runs
+ # outside this try, so a null `prompt_tokens`/`cached_tokens`
+ # would otherwise raise an uncaught TypeError and 500 the
+ # request (same class as the gemini fix in #2347).
+ total_input_tokens = _usage_int(usage.get("prompt_tokens")) or optimized_tokens
+ output_tokens = _usage_int(usage.get("completion_tokens"))
# OpenAI returns cached_tokens in prompt_tokens_details
# These are charged at 50% of the input price
prompt_details = usage.get("prompt_tokens_details") or {}
- cache_read_tokens = prompt_details.get("cached_tokens", 0)
+ cache_read_tokens = _usage_int(prompt_details.get("cached_tokens"))
except (KeyError, TypeError, AttributeError) as e:
logger.debug(
f"[{request_id}] Failed to extract cached tokens from OpenAI response: {e}"
diff --git a/tests/test_proxy/test_openai_chat_savings_profile.py b/tests/test_proxy/test_openai_chat_savings_profile.py
index 3cf8ddb2a..827d2d569 100644
--- a/tests/test_proxy/test_openai_chat_savings_profile.py
+++ b/tests/test_proxy/test_openai_chat_savings_profile.py
@@ -50,6 +50,67 @@ def _make_mock_backend() -> MagicMock:
return backend
+def _make_mock_backend_with_usage(usage: dict) -> MagicMock:
+ backend = MagicMock()
+ backend.name = "anyllm-openai"
+ backend.send_openai_message = AsyncMock(
+ return_value=BackendResponse(
+ body={
+ "id": "chatcmpl-1",
+ "object": "chat.completion",
+ "model": "gpt-4o",
+ "choices": [
+ {
+ "index": 0,
+ "message": {"role": "assistant", "content": "ok"},
+ "finish_reason": "stop",
+ }
+ ],
+ "usage": usage,
+ },
+ status_code=200,
+ headers={"content-type": "application/json"},
+ )
+ )
+ return backend
+
+
+def test_chat_completions_survives_null_usage_token_counts():
+ """A backend that reports present-but-null token counts must not 500.
+
+ `.get(key, default)` returns None for a null value, and the chat path
+ feeds those counts into `max(...)`/int-typed metrics. Without coercion a
+ single such response crashes the request and its outcome recording
+ (same class as the gemini fix in #2347).
+ """
+ config = ProxyConfig(
+ optimize=True,
+ cache_enabled=False,
+ rate_limit_enabled=False,
+ backend="anyllm",
+ anyllm_provider="openai",
+ )
+
+ # prompt_tokens / completion_tokens present but explicitly null.
+ mock_backend = _make_mock_backend_with_usage(
+ {"prompt_tokens": None, "completion_tokens": None, "total_tokens": None}
+ )
+ with patch("headroom.proxy.server.AnyLLMBackend", return_value=mock_backend):
+ app = create_app(config)
+ with TestClient(app) as client:
+ resp = client.post(
+ "/v1/chat/completions",
+ json={
+ "model": "gpt-4o",
+ "messages": [{"role": "user", "content": "hello"}],
+ "stream": False,
+ },
+ headers={"Authorization": "Bearer test-key"},
+ )
+
+ assert resp.status_code == 200, resp.text
+
+
def test_chat_completions_threads_savings_profile_kwargs_into_apply():
"""With HEADROOM_SAVINGS_PROFILE=agent-90, the chat path must pass the
profile knobs (compress_user_messages, target_ratio, ...) to apply()."""