Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

feat(savings): carry exact cache-read cost in history rollups (#3734)

gglucass1cb779e1bcae
diff --git a/headroom/proxy/savings_tracker.py b/headroom/proxy/savings_tracker.py
index 207a2b72f..400c18bf0 100644
--- a/headroom/proxy/savings_tracker.py
+++ b/headroom/proxy/savings_tracker.py
@@ -208,6 +208,19 @@ def _normalize_provider(value: Any) -> str:
MODEL_UNKNOWN = "unknown"
+def _empty_cache_delta() -> dict[str, Any]:
+ """Zeroed cache fields for a rollup bucket or one of its breakdowns.
+
+ ``cache_read_cost_usd_delta`` is None when any contributing checkpoint's
+ reads could not be priced (see ``_build_rollup``).
+ """
+ return {
+ "cache_read_tokens_delta": 0,
+ "cache_savings_usd_delta": 0.0,
+ "cache_read_cost_usd_delta": 0.0,
+ }
+
+
def _normalize_model(value: Any) -> str:
"""Normalize a model label, falling back to a stable sentinel.
@@ -1918,6 +1931,45 @@ class SavingsTracker:
prev_total_input_cost_usd = 0.0
prev_output_tokens = 0
prev_output_usd = 0.0
+ prev_cache_read_tokens = 0
+ prev_cache_savings_usd = 0.0
+ # What the bucket's cache reads actually COST, priced per checkpoint
+ # with the same function that put them into ``total_input_cost_usd``.
+ # Consumers need it to take reads out of the input bill, and cannot
+ # derive it from ``cache_savings_usd``: the read discount is not a
+ # fixed multiple of the read cost (reads bill at 0.1x on most models,
+ # 0.05x or 0.025x on others), so "discount / 9" misprices exactly the
+ # models with the steepest cache discount.
+ read_cost_per_token: dict[str, float] = {}
+
+ def _read_cost(model: str, reads: int) -> float | None:
+ if reads <= 0:
+ return 0.0
+ # Checkpoints written before per-model attribution carry no model,
+ # so their reads cannot be priced the way the request was. Report
+ # the bucket's read cost as unknown rather than guess.
+ if model == MODEL_UNKNOWN:
+ return None
+ if model not in read_cost_per_token:
+ read_cost_per_token[model] = (
+ _estimate_input_cost_usd(model, 1_000_000, cache_read_tokens=1_000_000)
+ / 1_000_000
+ )
+ return reads * read_cost_per_token[model]
+
+ def _add_cache(
+ target: dict[str, Any], reads: int, discount: float, cost: float | None
+ ) -> None:
+ target["cache_read_tokens_delta"] += reads
+ target["cache_savings_usd_delta"] = round(
+ target["cache_savings_usd_delta"] + discount, 6
+ )
+ if cost is None or target["cache_read_cost_usd_delta"] is None:
+ target["cache_read_cost_usd_delta"] = None
+ else:
+ target["cache_read_cost_usd_delta"] = round(
+ target["cache_read_cost_usd_delta"] + cost, 6
+ )
for point in history:
timestamp = _parse_timestamp(point["timestamp"])
@@ -1944,6 +1996,15 @@ class SavingsTracker:
delta_output_tokens = max(total_output_tokens - prev_output_tokens, 0)
delta_output_usd = max(total_output_usd - prev_output_usd, 0.0)
+ total_cache_read_tokens = _coerce_int(point.get("cache_read_tokens"))
+ total_cache_savings_usd = _coerce_float(point.get("cache_savings_usd"))
+ delta_cache_read_tokens = max(total_cache_read_tokens - prev_cache_read_tokens, 0)
+ delta_cache_savings_usd = max(total_cache_savings_usd - prev_cache_savings_usd, 0.0)
+ prev_cache_read_tokens = total_cache_read_tokens
+ prev_cache_savings_usd = total_cache_savings_usd
+ model = _normalize_model(point.get("model"))
+ delta_cache_read_cost_usd = _read_cost(model, delta_cache_read_tokens)
+
prev_total_tokens = total_tokens_saved
prev_total_usd = total_usd
prev_total_input_tokens = total_input_tokens
@@ -1965,6 +2026,7 @@ class SavingsTracker:
"total_input_cost_usd": total_input_cost_usd,
"output_tokens_saved_delta": 0,
"output_savings_usd_delta": 0.0,
+ **_empty_cache_delta(),
"by_provider": {},
"by_model": {},
},
@@ -1988,12 +2050,21 @@ class SavingsTracker:
entry["output_savings_usd_delta"] + delta_output_usd,
6,
)
+ _add_cache(
+ entry, delta_cache_read_tokens, delta_cache_savings_usd, delta_cache_read_cost_usd
+ )
# Attribute this checkpoint's delta to the provider that produced
# it. Each checkpoint comes from a single request, so its delta is
# wholly owned by one provider. Skip no-op checkpoints so providers
# only appear in a bucket where they actually moved a counter.
- if delta_tokens or delta_usd or delta_input_tokens or delta_input_cost_usd:
+ if (
+ delta_tokens
+ or delta_usd
+ or delta_input_tokens
+ or delta_input_cost_usd
+ or delta_cache_read_tokens
+ ):
provider = _normalize_provider(point.get("provider"))
prov = entry["by_provider"].setdefault(
provider,
@@ -2002,8 +2073,15 @@ class SavingsTracker:
"compression_savings_usd_delta": 0.0,
"total_input_tokens_delta": 0,
"total_input_cost_usd_delta": 0.0,
+ **_empty_cache_delta(),
},
)
+ _add_cache(
+ prov,
+ delta_cache_read_tokens,
+ delta_cache_savings_usd,
+ delta_cache_read_cost_usd,
+ )
prov["tokens_saved"] += delta_tokens
prov["compression_savings_usd_delta"] = round(
prov["compression_savings_usd_delta"] + delta_usd,
@@ -2015,7 +2093,6 @@ class SavingsTracker:
6,
)
- model = _normalize_model(point.get("model"))
mod = entry["by_model"].setdefault(
model,
{
@@ -2023,8 +2100,12 @@ class SavingsTracker:
"compression_savings_usd_delta": 0.0,
"total_input_tokens_delta": 0,
"total_input_cost_usd_delta": 0.0,
+ **_empty_cache_delta(),
},
)
+ _add_cache(
+ mod, delta_cache_read_tokens, delta_cache_savings_usd, delta_cache_read_cost_usd
+ )
mod["tokens_saved"] += delta_tokens
mod["compression_savings_usd_delta"] = round(
mod["compression_savings_usd_delta"] + delta_usd,
diff --git a/tests/test_proxy_savings_history.py b/tests/test_proxy_savings_history.py
index 0a91626c6..d6a7fdcc0 100644
--- a/tests/test_proxy_savings_history.py
+++ b/tests/test_proxy_savings_history.py
@@ -917,6 +917,106 @@ def test_savings_tracker_rollup_attributes_savings_per_provider(tmp_path, monkey
assert third["by_provider"]["unknown"]["tokens_saved"] == 15
+def test_savings_tracker_rollup_carries_exact_cache_read_cost_per_provider(tmp_path, monkeypatch):
+ # Reads on "steep-model" bill at 0.025x list (Fable-5.1-shaped pricing),
+ # on "flat-model" at the usual 0.1x. A consumer that backs the read cost
+ # out of the discount as "discount / 9" gets flat-model right and
+ # steep-model 4.3x too high, so the rollup has to carry the real figure.
+ fake_litellm = SimpleNamespace(
+ cost_per_token=lambda **_: (_ for _ in ()).throw(RuntimeError("unused")),
+ model_cost={
+ "steep-model": {
+ "input_cost_per_token": 1e-5,
+ "cache_read_input_token_cost": 2.5e-7,
+ },
+ "flat-model": {
+ "input_cost_per_token": 2e-6,
+ "cache_read_input_token_cost": 2e-7,
+ },
+ },
+ )
+ monkeypatch.setattr(savings_tracker_module, "LITELLM_AVAILABLE", True)
+ monkeypatch.setattr(savings_tracker_module, "litellm", fake_litellm)
+ tracker = SavingsTracker(
+ path=str(tmp_path / "proxy_savings.json"),
+ max_history_points=100,
+ max_history_age_days=30,
+ )
+
+ tracker.record_request(
+ model="steep-model",
+ provider="anthropic",
+ input_tokens=1_010_000,
+ tokens_saved=5_000,
+ cache_read_tokens=1_000_000,
+ uncached_input_tokens=10_000,
+ timestamp="2026-03-27T09:10:00Z",
+ )
+ tracker.record_request(
+ model="flat-model",
+ provider="openai",
+ input_tokens=150_000,
+ tokens_saved=2_000,
+ cache_read_tokens=100_000,
+ uncached_input_tokens=50_000,
+ timestamp="2026-03-27T09:20:00Z",
+ )
+
+ bucket = tracker.history_response()["series"]["hourly"][0]
+ anthropic = bucket["by_provider"]["anthropic"]
+ openai = bucket["by_provider"]["openai"]
+
+ assert anthropic["cache_read_tokens_delta"] == 1_000_000
+ assert anthropic["cache_read_cost_usd_delta"] == pytest.approx(0.25)
+ assert anthropic["cache_savings_usd_delta"] == pytest.approx(9.75)
+ # The read cost is exactly the read component of the recorded input cost,
+ # so input cost minus read cost is the uncached spend.
+ assert anthropic["total_input_cost_usd_delta"] - anthropic[
+ "cache_read_cost_usd_delta"
+ ] == pytest.approx(10_000 * 1e-5)
+
+ assert openai["cache_read_tokens_delta"] == 100_000
+ assert openai["cache_read_cost_usd_delta"] == pytest.approx(0.02)
+ assert openai["cache_savings_usd_delta"] == pytest.approx(0.18)
+
+ # Bucket totals are the sum of the providers.
+ assert bucket["cache_read_tokens_delta"] == 1_100_000
+ assert bucket["cache_read_cost_usd_delta"] == pytest.approx(0.27)
+ assert bucket["cache_savings_usd_delta"] == pytest.approx(9.93)
+ assert bucket["by_model"]["steep-model"]["cache_read_cost_usd_delta"] == pytest.approx(0.25)
+
+
+def test_savings_tracker_rollup_read_cost_unknown_for_model_less_checkpoints(tmp_path):
+ tracker = SavingsTracker(
+ path=str(tmp_path / "proxy_savings.json"),
+ max_history_points=100,
+ max_history_age_days=30,
+ )
+ history = [
+ {
+ "timestamp": "2026-03-27T09:00:00Z",
+ "provider": "anthropic",
+ "total_tokens_saved": 0,
+ "cache_read_tokens": 0,
+ },
+ # Written before per-model attribution: reads, but no model to price them.
+ {
+ "timestamp": "2026-03-27T09:30:00Z",
+ "provider": "anthropic",
+ "total_tokens_saved": 10,
+ "cache_read_tokens": 500,
+ "cache_savings_usd": 0.9,
+ },
+ ]
+
+ bucket = tracker._build_rollup(history, "hour")[0]
+
+ assert bucket["cache_read_tokens_delta"] == 500
+ assert bucket["cache_savings_usd_delta"] == pytest.approx(0.9)
+ assert bucket["cache_read_cost_usd_delta"] is None
+ assert bucket["by_provider"]["anthropic"]["cache_read_cost_usd_delta"] is None
+
+
def test_savings_tracker_rollup_attributes_savings_per_model(tmp_path, monkeypatch):
path = tmp_path / "proxy_savings.json"
tracker = SavingsTracker(