Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(litellm): surface Bedrock cache token usage in non-streaming responses (#1848)
Dima Solodukhad604e8690452
diff --git a/headroom/backends/litellm.py b/headroom/backends/litellm.pyindex 08fadd294..6860fed58 100644--- a/headroom/backends/litellm.py+++ b/headroom/backends/litellm.py@@ -354,6 +354,33 @@ def get_provider_config(provider: str) -> ProviderConfig:)+def _anthropic_usage_from_litellm(litellm_usage: Any) -> dict[str, Any]:+ """Map LiteLLM usage to Anthropic-shape usage, surfacing cache tokens.++ LiteLLM's ``prompt_tokens`` is the *total* prompt size including cached+ tokens, while Anthropic's ``input_tokens`` excludes tokens served from or+ written to the prompt cache. Without this mapping a working Bedrock prompt+ cache is invisible to non-streaming clients: they see the full prompt count+ and no cache fields, which looks exactly like the cache being broken+ (see #1345). The streaming/OpenAI paths already surface these fields.+ """+ cache_read = int(getattr(litellm_usage, "cache_read_input_tokens", 0) or 0)+ cache_write = int(getattr(litellm_usage, "cache_creation_input_tokens", 0) or 0)+ details = getattr(litellm_usage, "prompt_tokens_details", None)+ if details is not None:+ cache_read = cache_read or int(getattr(details, "cached_tokens", 0) or 0)+ cache_write = cache_write or int(getattr(details, "cache_creation_tokens", 0) or 0)+ prompt_tokens = int(getattr(litellm_usage, "prompt_tokens", 0) or 0)+ usage: dict[str, Any] = {+ "input_tokens": max(prompt_tokens - cache_read - cache_write, 0),+ "output_tokens": getattr(litellm_usage, "completion_tokens", 0),+ }+ if cache_read or cache_write:+ usage["cache_read_input_tokens"] = cache_read+ usage["cache_creation_input_tokens"] = cache_write+ return usage++def _convert_anthropic_tool(tool: dict[str, Any]) -> dict[str, Any]:"""Convert Anthropic tool format to OpenAI function format.@@ -653,10 +680,7 @@ class LiteLLMBackend(Backend):stop_reason = stop_reason_map.get(choice.finish_reason, "end_turn")# Build usage- usage = {- "input_tokens": getattr(litellm_response.usage, "prompt_tokens", 0),- "output_tokens": getattr(litellm_response.usage, "completion_tokens", 0),- }+ usage = _anthropic_usage_from_litellm(litellm_response.usage)return {"id": msg_id,diff --git a/tests/test_litellm_nonstream_cache_usage.py b/tests/test_litellm_nonstream_cache_usage.pynew file mode 100644index 000000000..872456bcf--- /dev/null+++ b/tests/test_litellm_nonstream_cache_usage.py@@ -0,0 +1,74 @@+"""Non-streaming LiteLLM responses must surface Bedrock cache token usage (GH #1345).++LiteLLM reports ``prompt_tokens`` as the total prompt size including cached+tokens, while the Anthropic response shape expects ``input_tokens`` to exclude+cache reads/writes and to carry ``cache_read_input_tokens`` /+``cache_creation_input_tokens`` alongside. The streaming and OpenAI paths+already map these fields; the non-streaming ``complete_message`` path dropped+them, so a working Bedrock prompt cache was indistinguishable from a broken+one for non-streaming clients.+"""++from __future__ import annotations++from types import SimpleNamespace++import pytest++litellm_backend = pytest.importorskip("headroom.backends.litellm")+_anthropic_usage_from_litellm = litellm_backend._anthropic_usage_from_litellm+++def test_plain_usage_without_cache_fields() -> None:+ usage = _anthropic_usage_from_litellm(SimpleNamespace(prompt_tokens=100, completion_tokens=7))+ assert usage == {"input_tokens": 100, "output_tokens": 7}+++def test_cache_read_surfaced_and_input_excludes_cached() -> None:+ usage = _anthropic_usage_from_litellm(+ SimpleNamespace(+ prompt_tokens=1213,+ completion_tokens=4,+ cache_read_input_tokens=1202,+ cache_creation_input_tokens=0,+ )+ )+ assert usage["input_tokens"] == 11+ assert usage["cache_read_input_tokens"] == 1202+ assert usage["cache_creation_input_tokens"] == 0+++def test_cache_write_on_first_call() -> None:+ usage = _anthropic_usage_from_litellm(+ SimpleNamespace(+ prompt_tokens=1237,+ completion_tokens=4,+ cache_read_input_tokens=0,+ cache_creation_input_tokens=1226,+ )+ )+ assert usage["input_tokens"] == 11+ assert usage["cache_creation_input_tokens"] == 1226+++def test_prompt_tokens_details_fallback() -> None:+ usage = _anthropic_usage_from_litellm(+ SimpleNamespace(+ prompt_tokens=1213,+ completion_tokens=4,+ prompt_tokens_details=SimpleNamespace(cached_tokens=1202, cache_creation_tokens=0),+ )+ )+ assert usage["input_tokens"] == 11+ assert usage["cache_read_input_tokens"] == 1202+++def test_input_tokens_never_negative() -> None:+ usage = _anthropic_usage_from_litellm(+ SimpleNamespace(+ prompt_tokens=10,+ completion_tokens=1,+ cache_read_input_tokens=15,+ )+ )+ assert usage["input_tokens"] == 0