Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(litellm): surface Bedrock cache token usage in non-streaming responses (#1848)

Dima Solodukhad604e8690452
diff --git a/headroom/backends/litellm.py b/headroom/backends/litellm.py
index 08fadd294..6860fed58 100644
--- a/headroom/backends/litellm.py
+++ b/headroom/backends/litellm.py
@@ -354,6 +354,33 @@ def get_provider_config(provider: str) -> ProviderConfig:
)
+def _anthropic_usage_from_litellm(litellm_usage: Any) -> dict[str, Any]:
+ """Map LiteLLM usage to Anthropic-shape usage, surfacing cache tokens.
+
+ LiteLLM's ``prompt_tokens`` is the *total* prompt size including cached
+ tokens, while Anthropic's ``input_tokens`` excludes tokens served from or
+ written to the prompt cache. Without this mapping a working Bedrock prompt
+ cache is invisible to non-streaming clients: they see the full prompt count
+ and no cache fields, which looks exactly like the cache being broken
+ (see #1345). The streaming/OpenAI paths already surface these fields.
+ """
+ cache_read = int(getattr(litellm_usage, "cache_read_input_tokens", 0) or 0)
+ cache_write = int(getattr(litellm_usage, "cache_creation_input_tokens", 0) or 0)
+ details = getattr(litellm_usage, "prompt_tokens_details", None)
+ if details is not None:
+ cache_read = cache_read or int(getattr(details, "cached_tokens", 0) or 0)
+ cache_write = cache_write or int(getattr(details, "cache_creation_tokens", 0) or 0)
+ prompt_tokens = int(getattr(litellm_usage, "prompt_tokens", 0) or 0)
+ usage: dict[str, Any] = {
+ "input_tokens": max(prompt_tokens - cache_read - cache_write, 0),
+ "output_tokens": getattr(litellm_usage, "completion_tokens", 0),
+ }
+ if cache_read or cache_write:
+ usage["cache_read_input_tokens"] = cache_read
+ usage["cache_creation_input_tokens"] = cache_write
+ return usage
+
+
def _convert_anthropic_tool(tool: dict[str, Any]) -> dict[str, Any]:
"""Convert Anthropic tool format to OpenAI function format.
@@ -653,10 +680,7 @@ class LiteLLMBackend(Backend):
stop_reason = stop_reason_map.get(choice.finish_reason, "end_turn")
# Build usage
- usage = {
- "input_tokens": getattr(litellm_response.usage, "prompt_tokens", 0),
- "output_tokens": getattr(litellm_response.usage, "completion_tokens", 0),
- }
+ usage = _anthropic_usage_from_litellm(litellm_response.usage)
return {
"id": msg_id,
diff --git a/tests/test_litellm_nonstream_cache_usage.py b/tests/test_litellm_nonstream_cache_usage.py
new file mode 100644
index 000000000..872456bcf
--- /dev/null
+++ b/tests/test_litellm_nonstream_cache_usage.py
@@ -0,0 +1,74 @@
+"""Non-streaming LiteLLM responses must surface Bedrock cache token usage (GH #1345).
+
+LiteLLM reports ``prompt_tokens`` as the total prompt size including cached
+tokens, while the Anthropic response shape expects ``input_tokens`` to exclude
+cache reads/writes and to carry ``cache_read_input_tokens`` /
+``cache_creation_input_tokens`` alongside. The streaming and OpenAI paths
+already map these fields; the non-streaming ``complete_message`` path dropped
+them, so a working Bedrock prompt cache was indistinguishable from a broken
+one for non-streaming clients.
+"""
+
+from __future__ import annotations
+
+from types import SimpleNamespace
+
+import pytest
+
+litellm_backend = pytest.importorskip("headroom.backends.litellm")
+_anthropic_usage_from_litellm = litellm_backend._anthropic_usage_from_litellm
+
+
+def test_plain_usage_without_cache_fields() -> None:
+ usage = _anthropic_usage_from_litellm(SimpleNamespace(prompt_tokens=100, completion_tokens=7))
+ assert usage == {"input_tokens": 100, "output_tokens": 7}
+
+
+def test_cache_read_surfaced_and_input_excludes_cached() -> None:
+ usage = _anthropic_usage_from_litellm(
+ SimpleNamespace(
+ prompt_tokens=1213,
+ completion_tokens=4,
+ cache_read_input_tokens=1202,
+ cache_creation_input_tokens=0,
+ )
+ )
+ assert usage["input_tokens"] == 11
+ assert usage["cache_read_input_tokens"] == 1202
+ assert usage["cache_creation_input_tokens"] == 0
+
+
+def test_cache_write_on_first_call() -> None:
+ usage = _anthropic_usage_from_litellm(
+ SimpleNamespace(
+ prompt_tokens=1237,
+ completion_tokens=4,
+ cache_read_input_tokens=0,
+ cache_creation_input_tokens=1226,
+ )
+ )
+ assert usage["input_tokens"] == 11
+ assert usage["cache_creation_input_tokens"] == 1226
+
+
+def test_prompt_tokens_details_fallback() -> None:
+ usage = _anthropic_usage_from_litellm(
+ SimpleNamespace(
+ prompt_tokens=1213,
+ completion_tokens=4,
+ prompt_tokens_details=SimpleNamespace(cached_tokens=1202, cache_creation_tokens=0),
+ )
+ )
+ assert usage["input_tokens"] == 11
+ assert usage["cache_read_input_tokens"] == 1202
+
+
+def test_input_tokens_never_negative() -> None:
+ usage = _anthropic_usage_from_litellm(
+ SimpleNamespace(
+ prompt_tokens=10,
+ completion_tokens=1,
+ cache_read_input_tokens=15,
+ )
+ )
+ assert usage["input_tokens"] == 0