Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(proxy): cache_savings_usd silently zeroes when litellm is unavailable (#2005)
Dima Solodukha75d786117a15
diff --git a/headroom/proxy/savings_tracker.py b/headroom/proxy/savings_tracker.pyindex 92852bce2..32bda91dc 100644--- a/headroom/proxy/savings_tracker.py+++ b/headroom/proxy/savings_tracker.py@@ -218,16 +218,22 @@ def _estimate_cache_savings_usd(model: str, cache_read_tokens: int) -> float:"""Estimate cache-read savings in USD — the discount delta vs list price.Cache reads bill at the provider's discounted rate, so the saving per token- is ``input_cost_per_token - cache_read_input_token_cost``. Unknown models or- an unavailable litellm price as 0.0 (fail open); tokens still accumulate.+ is ``input_cost_per_token - cache_read_input_token_cost``. Unknown models+ price as 0.0 (fail open); tokens still accumulate. An unavailable litellm+ falls back to ``DEFAULT_FALLBACK_INPUT_COST_PER_TOKEN``, matching+ ``_estimate_input_cost_usd``/``_estimate_compression_savings_usd`` — otherwise+ cache_savings_usd silently reads as $0 forever on any install without+ litellm (e.g. Python 3.14, where headroom's own dependency spec excludes it).Deliberately diverges from ``proxy/cost.py``'s session-scoped providermultipliers (``_CACHE_ECONOMICS``): this lifetime figure follows theper-model litellm pricing the rest of this module already uses."""litellm = _get_litellm_module()- if cache_read_tokens <= 0 or litellm is None:+ if cache_read_tokens <= 0:return 0.0+ if litellm is None:+ return float(cache_read_tokens) * float(DEFAULT_FALLBACK_INPUT_COST_PER_TOKEN)try:resolved = _resolve_litellm_model(model)diff --git a/tests/test_proxy_savings_history.py b/tests/test_proxy_savings_history.pyindex 7e353d85a..117fd1ee6 100644--- a/tests/test_proxy_savings_history.py+++ b/tests/test_proxy_savings_history.py@@ -1689,7 +1689,13 @@ def test_active_display_session_without_cache_fields_reloads_safely(tmp_path, moassert session["requests"] == 2-def test_cache_savings_edge_cases_zero_and_unpriced(tmp_path):+def test_cache_savings_edge_cases_zero_and_unpriced(tmp_path, monkeypatch):+ # Pin a litellm whose price table doesn't know the model, so this stays a+ # test of the unpriced-model path on every environment — on installs+ # without litellm (e.g. Python 3.14) the blended-rate fallback would+ # otherwise kick in and produce a nonzero estimate.+ fake_litellm = SimpleNamespace(model_cost={})+ monkeypatch.setattr(savings_tracker_module, "_get_litellm_module", lambda: fake_litellm)path = tmp_path / "proxy_savings.json"tracker = SavingsTracker(path=str(path))@@ -1785,6 +1791,35 @@ def test_cache_savings_usd_uses_litellm_discount_delta(tmp_path, monkeypatch):assert tracker.snapshot()["lifetime"]["cache_savings_usd"] == pytest.approx(2.7)+def test_cache_savings_usd_falls_back_when_litellm_unavailable(tmp_path, monkeypatch):+ # Regression: on any install without litellm (e.g. Python 3.14, where+ # headroom-ai's own dependency spec excludes it), cache_savings_usd must+ # use the same DEFAULT_FALLBACK_INPUT_COST_PER_TOKEN estimate that+ # _estimate_input_cost_usd already falls back to — not silently read as+ # $0 forever while cache_read_tokens and total_input_cost_usd keep+ # accumulating normally.+ monkeypatch.setattr(savings_tracker_module, "LITELLM_AVAILABLE", False)+ monkeypatch.setattr(savings_tracker_module, "litellm", None)++ fallback_rate = savings_tracker_module.DEFAULT_FALLBACK_INPUT_COST_PER_TOKEN+ assert savings_tracker_module._estimate_cache_savings_usd(+ "claude-sonnet-4-6", 1_000_000+ ) == pytest.approx(1_000_000 * fallback_rate)++ tracker = SavingsTracker(path=str(tmp_path / "proxy_savings.json"))+ tracker.record_request(+ model="claude-sonnet-4-6",+ input_tokens=1_000,+ tokens_saved=0,+ cache_read_tokens=1_000_000,+ timestamp="2026-07-02T00:00:00Z",+ )+ snapshot = tracker.snapshot()+ assert snapshot["lifetime"]["cache_read_tokens"] == 1_000_000+ assert snapshot["lifetime"]["cache_savings_usd"] > 0.0+ assert snapshot["lifetime"]["cache_savings_usd"] == pytest.approx(1_000_000 * fallback_rate)++def test_non_finite_state_values_coerce_to_defaults(tmp_path):path = tmp_path / "proxy_savings.json"# json accepts bare Infinity/NaN literals; a corrupted file must not crash