Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
feat(savings): carry exact cache-read cost in history rollups (#3734)
gglucass1cb779e1bcae
diff --git a/headroom/proxy/savings_tracker.py b/headroom/proxy/savings_tracker.pyindex 207a2b72f..400c18bf0 100644--- a/headroom/proxy/savings_tracker.py+++ b/headroom/proxy/savings_tracker.py@@ -208,6 +208,19 @@ def _normalize_provider(value: Any) -> str:MODEL_UNKNOWN = "unknown"+def _empty_cache_delta() -> dict[str, Any]:+ """Zeroed cache fields for a rollup bucket or one of its breakdowns.++ ``cache_read_cost_usd_delta`` is None when any contributing checkpoint's+ reads could not be priced (see ``_build_rollup``).+ """+ return {+ "cache_read_tokens_delta": 0,+ "cache_savings_usd_delta": 0.0,+ "cache_read_cost_usd_delta": 0.0,+ }++def _normalize_model(value: Any) -> str:"""Normalize a model label, falling back to a stable sentinel.@@ -1918,6 +1931,45 @@ class SavingsTracker:prev_total_input_cost_usd = 0.0prev_output_tokens = 0prev_output_usd = 0.0+ prev_cache_read_tokens = 0+ prev_cache_savings_usd = 0.0+ # What the bucket's cache reads actually COST, priced per checkpoint+ # with the same function that put them into ``total_input_cost_usd``.+ # Consumers need it to take reads out of the input bill, and cannot+ # derive it from ``cache_savings_usd``: the read discount is not a+ # fixed multiple of the read cost (reads bill at 0.1x on most models,+ # 0.05x or 0.025x on others), so "discount / 9" misprices exactly the+ # models with the steepest cache discount.+ read_cost_per_token: dict[str, float] = {}++ def _read_cost(model: str, reads: int) -> float | None:+ if reads <= 0:+ return 0.0+ # Checkpoints written before per-model attribution carry no model,+ # so their reads cannot be priced the way the request was. Report+ # the bucket's read cost as unknown rather than guess.+ if model == MODEL_UNKNOWN:+ return None+ if model not in read_cost_per_token:+ read_cost_per_token[model] = (+ _estimate_input_cost_usd(model, 1_000_000, cache_read_tokens=1_000_000)+ / 1_000_000+ )+ return reads * read_cost_per_token[model]++ def _add_cache(+ target: dict[str, Any], reads: int, discount: float, cost: float | None+ ) -> None:+ target["cache_read_tokens_delta"] += reads+ target["cache_savings_usd_delta"] = round(+ target["cache_savings_usd_delta"] + discount, 6+ )+ if cost is None or target["cache_read_cost_usd_delta"] is None:+ target["cache_read_cost_usd_delta"] = None+ else:+ target["cache_read_cost_usd_delta"] = round(+ target["cache_read_cost_usd_delta"] + cost, 6+ )for point in history:timestamp = _parse_timestamp(point["timestamp"])@@ -1944,6 +1996,15 @@ class SavingsTracker:delta_output_tokens = max(total_output_tokens - prev_output_tokens, 0)delta_output_usd = max(total_output_usd - prev_output_usd, 0.0)+ total_cache_read_tokens = _coerce_int(point.get("cache_read_tokens"))+ total_cache_savings_usd = _coerce_float(point.get("cache_savings_usd"))+ delta_cache_read_tokens = max(total_cache_read_tokens - prev_cache_read_tokens, 0)+ delta_cache_savings_usd = max(total_cache_savings_usd - prev_cache_savings_usd, 0.0)+ prev_cache_read_tokens = total_cache_read_tokens+ prev_cache_savings_usd = total_cache_savings_usd+ model = _normalize_model(point.get("model"))+ delta_cache_read_cost_usd = _read_cost(model, delta_cache_read_tokens)+prev_total_tokens = total_tokens_savedprev_total_usd = total_usdprev_total_input_tokens = total_input_tokens@@ -1965,6 +2026,7 @@ class SavingsTracker:"total_input_cost_usd": total_input_cost_usd,"output_tokens_saved_delta": 0,"output_savings_usd_delta": 0.0,+ **_empty_cache_delta(),"by_provider": {},"by_model": {},},@@ -1988,12 +2050,21 @@ class SavingsTracker:entry["output_savings_usd_delta"] + delta_output_usd,6,)+ _add_cache(+ entry, delta_cache_read_tokens, delta_cache_savings_usd, delta_cache_read_cost_usd+ )# Attribute this checkpoint's delta to the provider that produced# it. Each checkpoint comes from a single request, so its delta is# wholly owned by one provider. Skip no-op checkpoints so providers# only appear in a bucket where they actually moved a counter.- if delta_tokens or delta_usd or delta_input_tokens or delta_input_cost_usd:+ if (+ delta_tokens+ or delta_usd+ or delta_input_tokens+ or delta_input_cost_usd+ or delta_cache_read_tokens+ ):provider = _normalize_provider(point.get("provider"))prov = entry["by_provider"].setdefault(provider,@@ -2002,8 +2073,15 @@ class SavingsTracker:"compression_savings_usd_delta": 0.0,"total_input_tokens_delta": 0,"total_input_cost_usd_delta": 0.0,+ **_empty_cache_delta(),},)+ _add_cache(+ prov,+ delta_cache_read_tokens,+ delta_cache_savings_usd,+ delta_cache_read_cost_usd,+ )prov["tokens_saved"] += delta_tokensprov["compression_savings_usd_delta"] = round(prov["compression_savings_usd_delta"] + delta_usd,@@ -2015,7 +2093,6 @@ class SavingsTracker:6,)- model = _normalize_model(point.get("model"))mod = entry["by_model"].setdefault(model,{@@ -2023,8 +2100,12 @@ class SavingsTracker:"compression_savings_usd_delta": 0.0,"total_input_tokens_delta": 0,"total_input_cost_usd_delta": 0.0,+ **_empty_cache_delta(),},)+ _add_cache(+ mod, delta_cache_read_tokens, delta_cache_savings_usd, delta_cache_read_cost_usd+ )mod["tokens_saved"] += delta_tokensmod["compression_savings_usd_delta"] = round(mod["compression_savings_usd_delta"] + delta_usd,diff --git a/tests/test_proxy_savings_history.py b/tests/test_proxy_savings_history.pyindex 0a91626c6..d6a7fdcc0 100644--- a/tests/test_proxy_savings_history.py+++ b/tests/test_proxy_savings_history.py@@ -917,6 +917,106 @@ def test_savings_tracker_rollup_attributes_savings_per_provider(tmp_path, monkeyassert third["by_provider"]["unknown"]["tokens_saved"] == 15+def test_savings_tracker_rollup_carries_exact_cache_read_cost_per_provider(tmp_path, monkeypatch):+ # Reads on "steep-model" bill at 0.025x list (Fable-5.1-shaped pricing),+ # on "flat-model" at the usual 0.1x. A consumer that backs the read cost+ # out of the discount as "discount / 9" gets flat-model right and+ # steep-model 4.3x too high, so the rollup has to carry the real figure.+ fake_litellm = SimpleNamespace(+ cost_per_token=lambda **_: (_ for _ in ()).throw(RuntimeError("unused")),+ model_cost={+ "steep-model": {+ "input_cost_per_token": 1e-5,+ "cache_read_input_token_cost": 2.5e-7,+ },+ "flat-model": {+ "input_cost_per_token": 2e-6,+ "cache_read_input_token_cost": 2e-7,+ },+ },+ )+ monkeypatch.setattr(savings_tracker_module, "LITELLM_AVAILABLE", True)+ monkeypatch.setattr(savings_tracker_module, "litellm", fake_litellm)+ tracker = SavingsTracker(+ path=str(tmp_path / "proxy_savings.json"),+ max_history_points=100,+ max_history_age_days=30,+ )++ tracker.record_request(+ model="steep-model",+ provider="anthropic",+ input_tokens=1_010_000,+ tokens_saved=5_000,+ cache_read_tokens=1_000_000,+ uncached_input_tokens=10_000,+ timestamp="2026-03-27T09:10:00Z",+ )+ tracker.record_request(+ model="flat-model",+ provider="openai",+ input_tokens=150_000,+ tokens_saved=2_000,+ cache_read_tokens=100_000,+ uncached_input_tokens=50_000,+ timestamp="2026-03-27T09:20:00Z",+ )++ bucket = tracker.history_response()["series"]["hourly"][0]+ anthropic = bucket["by_provider"]["anthropic"]+ openai = bucket["by_provider"]["openai"]++ assert anthropic["cache_read_tokens_delta"] == 1_000_000+ assert anthropic["cache_read_cost_usd_delta"] == pytest.approx(0.25)+ assert anthropic["cache_savings_usd_delta"] == pytest.approx(9.75)+ # The read cost is exactly the read component of the recorded input cost,+ # so input cost minus read cost is the uncached spend.+ assert anthropic["total_input_cost_usd_delta"] - anthropic[+ "cache_read_cost_usd_delta"+ ] == pytest.approx(10_000 * 1e-5)++ assert openai["cache_read_tokens_delta"] == 100_000+ assert openai["cache_read_cost_usd_delta"] == pytest.approx(0.02)+ assert openai["cache_savings_usd_delta"] == pytest.approx(0.18)++ # Bucket totals are the sum of the providers.+ assert bucket["cache_read_tokens_delta"] == 1_100_000+ assert bucket["cache_read_cost_usd_delta"] == pytest.approx(0.27)+ assert bucket["cache_savings_usd_delta"] == pytest.approx(9.93)+ assert bucket["by_model"]["steep-model"]["cache_read_cost_usd_delta"] == pytest.approx(0.25)+++def test_savings_tracker_rollup_read_cost_unknown_for_model_less_checkpoints(tmp_path):+ tracker = SavingsTracker(+ path=str(tmp_path / "proxy_savings.json"),+ max_history_points=100,+ max_history_age_days=30,+ )+ history = [+ {+ "timestamp": "2026-03-27T09:00:00Z",+ "provider": "anthropic",+ "total_tokens_saved": 0,+ "cache_read_tokens": 0,+ },+ # Written before per-model attribution: reads, but no model to price them.+ {+ "timestamp": "2026-03-27T09:30:00Z",+ "provider": "anthropic",+ "total_tokens_saved": 10,+ "cache_read_tokens": 500,+ "cache_savings_usd": 0.9,+ },+ ]++ bucket = tracker._build_rollup(history, "hour")[0]++ assert bucket["cache_read_tokens_delta"] == 500+ assert bucket["cache_savings_usd_delta"] == pytest.approx(0.9)+ assert bucket["cache_read_cost_usd_delta"] is None+ assert bucket["by_provider"]["anthropic"]["cache_read_cost_usd_delta"] is None++def test_savings_tracker_rollup_attributes_savings_per_model(tmp_path, monkeypatch):path = tmp_path / "proxy_savings.json"tracker = SavingsTracker(