Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(backends/litellm): None-guard core token counts in OpenAI usage block (#2324)
Abhay Singh12f9f58cb3dc
diff --git a/headroom/backends/litellm.py b/headroom/backends/litellm.pyindex 6e3afd7e4..01c73b032 100644--- a/headroom/backends/litellm.py+++ b/headroom/backends/litellm.py@@ -1389,10 +1389,16 @@ class LiteLLMBackend(Backend):# cache_creation_tokens for the OpenAI nested dialect. Surface both# so PrefixCacheTracker.update_from_response on the backend-routed# path observes a stable shape instead of branching on key presence.+ # None-guard the core counts (same defensive style as the cache+ # fields just below). A provider can leave any of these None on the+ # Usage object; emitting None here flows into the OpenAI-shape body,+ # and the backend-routed OpenAI handler reads them straight into+ # arithmetic and RequestOutcome (output_tokens=..., and+ # max(0, prompt_tokens - ...)), which raises TypeError on None.usage_block: dict[str, Any] = {- "prompt_tokens": response.usage.prompt_tokens,- "completion_tokens": response.usage.completion_tokens,- "total_tokens": response.usage.total_tokens,+ "prompt_tokens": int(getattr(response.usage, "prompt_tokens", 0) or 0),+ "completion_tokens": int(getattr(response.usage, "completion_tokens", 0) or 0),+ "total_tokens": int(getattr(response.usage, "total_tokens", 0) or 0),}# Defensive getattr: LiteLLM only attaches these top-level attrsdiff --git a/tests/test_backends/test_litellm_cache_stats.py b/tests/test_backends/test_litellm_cache_stats.pyindex 083a7a5d1..4ec6e057f 100644--- a/tests/test_backends/test_litellm_cache_stats.py+++ b/tests/test_backends/test_litellm_cache_stats.py@@ -215,3 +215,30 @@ async def test_no_cache_fields_means_no_cache_keys_in_usage_block() -> None:assert "cache_read_input_tokens" not in body_usageassert "cache_creation_input_tokens" not in body_usageassert "prompt_tokens_details" not in body_usage+++async def test_none_core_counts_coerced_to_zero() -> None:+ """A provider can leave prompt/completion/total token counts None on the+ Usage object. The OpenAI-shape usage block must emit ints, not None, so the+ backend-routed OpenAI handler (which reads these straight into arithmetic+ and RequestOutcome) does not crash with a TypeError."""+ usage = _FakeUsage(+ prompt_tokens=None, # type: ignore[arg-type]+ completion_tokens=None, # type: ignore[arg-type]+ total_tokens=None, # type: ignore[arg-type]+ )+ response = _make_response(usage)++ backend = _make_backend()+ with patch("headroom.backends.litellm.acompletion", new_callable=AsyncMock) as mock_acomp:+ mock_acomp.return_value = response+ result = await backend.send_openai_message(_request_body(), {})++ body_usage = result.body["usage"]+ assert body_usage["prompt_tokens"] == 0+ assert body_usage["completion_tokens"] == 0+ assert body_usage["total_tokens"] == 0+ assert all(+ isinstance(body_usage[k], int)+ for k in ("prompt_tokens", "completion_tokens", "total_tokens")+ )