Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(proxy): cache_savings_usd silently zeroes when litellm is unavailable (#2005)

Dima Solodukha75d786117a15
diff --git a/headroom/proxy/savings_tracker.py b/headroom/proxy/savings_tracker.py
index 92852bce2..32bda91dc 100644
--- a/headroom/proxy/savings_tracker.py
+++ b/headroom/proxy/savings_tracker.py
@@ -218,16 +218,22 @@ def _estimate_cache_savings_usd(model: str, cache_read_tokens: int) -> float:
"""Estimate cache-read savings in USD — the discount delta vs list price.
Cache reads bill at the provider's discounted rate, so the saving per token
- is ``input_cost_per_token - cache_read_input_token_cost``. Unknown models or
- an unavailable litellm price as 0.0 (fail open); tokens still accumulate.
+ is ``input_cost_per_token - cache_read_input_token_cost``. Unknown models
+ price as 0.0 (fail open); tokens still accumulate. An unavailable litellm
+ falls back to ``DEFAULT_FALLBACK_INPUT_COST_PER_TOKEN``, matching
+ ``_estimate_input_cost_usd``/``_estimate_compression_savings_usd`` — otherwise
+ cache_savings_usd silently reads as $0 forever on any install without
+ litellm (e.g. Python 3.14, where headroom's own dependency spec excludes it).
Deliberately diverges from ``proxy/cost.py``'s session-scoped provider
multipliers (``_CACHE_ECONOMICS``): this lifetime figure follows the
per-model litellm pricing the rest of this module already uses.
"""
litellm = _get_litellm_module()
- if cache_read_tokens <= 0 or litellm is None:
+ if cache_read_tokens <= 0:
return 0.0
+ if litellm is None:
+ return float(cache_read_tokens) * float(DEFAULT_FALLBACK_INPUT_COST_PER_TOKEN)
try:
resolved = _resolve_litellm_model(model)
diff --git a/tests/test_proxy_savings_history.py b/tests/test_proxy_savings_history.py
index 7e353d85a..117fd1ee6 100644
--- a/tests/test_proxy_savings_history.py
+++ b/tests/test_proxy_savings_history.py
@@ -1689,7 +1689,13 @@ def test_active_display_session_without_cache_fields_reloads_safely(tmp_path, mo
assert session["requests"] == 2
-def test_cache_savings_edge_cases_zero_and_unpriced(tmp_path):
+def test_cache_savings_edge_cases_zero_and_unpriced(tmp_path, monkeypatch):
+ # Pin a litellm whose price table doesn't know the model, so this stays a
+ # test of the unpriced-model path on every environment — on installs
+ # without litellm (e.g. Python 3.14) the blended-rate fallback would
+ # otherwise kick in and produce a nonzero estimate.
+ fake_litellm = SimpleNamespace(model_cost={})
+ monkeypatch.setattr(savings_tracker_module, "_get_litellm_module", lambda: fake_litellm)
path = tmp_path / "proxy_savings.json"
tracker = SavingsTracker(path=str(path))
@@ -1785,6 +1791,35 @@ def test_cache_savings_usd_uses_litellm_discount_delta(tmp_path, monkeypatch):
assert tracker.snapshot()["lifetime"]["cache_savings_usd"] == pytest.approx(2.7)
+def test_cache_savings_usd_falls_back_when_litellm_unavailable(tmp_path, monkeypatch):
+ # Regression: on any install without litellm (e.g. Python 3.14, where
+ # headroom-ai's own dependency spec excludes it), cache_savings_usd must
+ # use the same DEFAULT_FALLBACK_INPUT_COST_PER_TOKEN estimate that
+ # _estimate_input_cost_usd already falls back to — not silently read as
+ # $0 forever while cache_read_tokens and total_input_cost_usd keep
+ # accumulating normally.
+ monkeypatch.setattr(savings_tracker_module, "LITELLM_AVAILABLE", False)
+ monkeypatch.setattr(savings_tracker_module, "litellm", None)
+
+ fallback_rate = savings_tracker_module.DEFAULT_FALLBACK_INPUT_COST_PER_TOKEN
+ assert savings_tracker_module._estimate_cache_savings_usd(
+ "claude-sonnet-4-6", 1_000_000
+ ) == pytest.approx(1_000_000 * fallback_rate)
+
+ tracker = SavingsTracker(path=str(tmp_path / "proxy_savings.json"))
+ tracker.record_request(
+ model="claude-sonnet-4-6",
+ input_tokens=1_000,
+ tokens_saved=0,
+ cache_read_tokens=1_000_000,
+ timestamp="2026-07-02T00:00:00Z",
+ )
+ snapshot = tracker.snapshot()
+ assert snapshot["lifetime"]["cache_read_tokens"] == 1_000_000
+ assert snapshot["lifetime"]["cache_savings_usd"] > 0.0
+ assert snapshot["lifetime"]["cache_savings_usd"] == pytest.approx(1_000_000 * fallback_rate)
+
+
def test_non_finite_state_values_coerce_to_defaults(tmp_path):
path = tmp_path / "proxy_savings.json"
# json accepts bare Infinity/NaN literals; a corrupted file must not crash