Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(backends/litellm): None-guard core token counts in OpenAI usage block (#2324)

Abhay Singh12f9f58cb3dc
diff --git a/headroom/backends/litellm.py b/headroom/backends/litellm.py
index 6e3afd7e4..01c73b032 100644
--- a/headroom/backends/litellm.py
+++ b/headroom/backends/litellm.py
@@ -1389,10 +1389,16 @@ class LiteLLMBackend(Backend):
# cache_creation_tokens for the OpenAI nested dialect. Surface both
# so PrefixCacheTracker.update_from_response on the backend-routed
# path observes a stable shape instead of branching on key presence.
+ # None-guard the core counts (same defensive style as the cache
+ # fields just below). A provider can leave any of these None on the
+ # Usage object; emitting None here flows into the OpenAI-shape body,
+ # and the backend-routed OpenAI handler reads them straight into
+ # arithmetic and RequestOutcome (output_tokens=..., and
+ # max(0, prompt_tokens - ...)), which raises TypeError on None.
usage_block: dict[str, Any] = {
- "prompt_tokens": response.usage.prompt_tokens,
- "completion_tokens": response.usage.completion_tokens,
- "total_tokens": response.usage.total_tokens,
+ "prompt_tokens": int(getattr(response.usage, "prompt_tokens", 0) or 0),
+ "completion_tokens": int(getattr(response.usage, "completion_tokens", 0) or 0),
+ "total_tokens": int(getattr(response.usage, "total_tokens", 0) or 0),
}
# Defensive getattr: LiteLLM only attaches these top-level attrs
diff --git a/tests/test_backends/test_litellm_cache_stats.py b/tests/test_backends/test_litellm_cache_stats.py
index 083a7a5d1..4ec6e057f 100644
--- a/tests/test_backends/test_litellm_cache_stats.py
+++ b/tests/test_backends/test_litellm_cache_stats.py
@@ -215,3 +215,30 @@ async def test_no_cache_fields_means_no_cache_keys_in_usage_block() -> None:
assert "cache_read_input_tokens" not in body_usage
assert "cache_creation_input_tokens" not in body_usage
assert "prompt_tokens_details" not in body_usage
+
+
+async def test_none_core_counts_coerced_to_zero() -> None:
+ """A provider can leave prompt/completion/total token counts None on the
+ Usage object. The OpenAI-shape usage block must emit ints, not None, so the
+ backend-routed OpenAI handler (which reads these straight into arithmetic
+ and RequestOutcome) does not crash with a TypeError."""
+ usage = _FakeUsage(
+ prompt_tokens=None, # type: ignore[arg-type]
+ completion_tokens=None, # type: ignore[arg-type]
+ total_tokens=None, # type: ignore[arg-type]
+ )
+ response = _make_response(usage)
+
+ backend = _make_backend()
+ with patch("headroom.backends.litellm.acompletion", new_callable=AsyncMock) as mock_acomp:
+ mock_acomp.return_value = response
+ result = await backend.send_openai_message(_request_body(), {})
+
+ body_usage = result.body["usage"]
+ assert body_usage["prompt_tokens"] == 0
+ assert body_usage["completion_tokens"] == 0
+ assert body_usage["total_tokens"] == 0
+ assert all(
+ isinstance(body_usage[k], int)
+ for k in ("prompt_tokens", "completion_tokens", "total_tokens")
+ )