Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
feat(savings): track lifetime output spend so a bill-share rate has a denominator (#3520)
gglucassc766bdb2b361
diff --git a/headroom/proxy/prometheus_metrics.py b/headroom/proxy/prometheus_metrics.pyindex a0db9ceba..66cfbe7a2 100644--- a/headroom/proxy/prometheus_metrics.py+++ b/headroom/proxy/prometheus_metrics.py@@ -1030,6 +1030,7 @@ class PrometheusMetrics:total_input_tokens=total_input_tokens,total_input_cost_usd=total_input_cost_usd,output_tokens_saved=output_tokens_saved,+ output_tokens=output_tokens,estimated_savings_usd=savings_usd,)diff --git a/headroom/proxy/savings_tracker.py b/headroom/proxy/savings_tracker.pyindex ba7f3609b..57d38d16b 100644--- a/headroom/proxy/savings_tracker.py+++ b/headroom/proxy/savings_tracker.py@@ -389,6 +389,17 @@ def _estimate_output_savings_usd(model: str, tokens_saved: int) -> float:return float(tokens_saved) * float(DEFAULT_FALLBACK_OUTPUT_COST_PER_TOKEN)+def _estimate_output_cost_usd(model: str, output_tokens: int) -> float:+ """Estimate what EMITTED output tokens cost, at the model's output rate.++ Same rate table as ``_estimate_output_savings_usd``, which prices the+ tokens the shaper kept the model from emitting; this prices the ones it+ did emit. Split out so the call sites read as spend and saving rather than+ one function doing double duty.+ """+ return _estimate_output_savings_usd(model, output_tokens)++def _estimate_cache_savings_usd(model: str, cache_read_tokens: int) -> float:"""Estimate cache-read savings in USD — the discount delta vs list price.@@ -601,6 +612,7 @@ def _normalize_history_entry(entry: Any) -> dict[str, Any] | None:total_input_cost_usd = 0.0output_tokens_saved = 0output_savings_usd = 0.0+ total_output_cost_usd = 0.0provider = PROVIDER_UNKNOWNmodel = MODEL_UNKNOWN@@ -617,6 +629,7 @@ def _normalize_history_entry(entry: Any) -> dict[str, Any] | None:total_input_cost_usd = _coerce_float(entry.get("total_input_cost_usd"))output_tokens_saved = _coerce_int(entry.get("output_tokens_saved"))output_savings_usd = _coerce_float(entry.get("output_savings_usd"))+ total_output_cost_usd = _coerce_float(entry.get("total_output_cost_usd"))provider = _normalize_provider(entry.get("provider"))model = _normalize_model(entry.get("model"))elif isinstance(entry, list | tuple) and len(entry) >= 2:@@ -646,6 +659,7 @@ def _normalize_history_entry(entry: Any) -> dict[str, Any] | None:"total_input_cost_usd": round(total_input_cost_usd, 6),"output_tokens_saved": output_tokens_saved,"output_savings_usd": round(output_savings_usd, 6),+ "total_output_cost_usd": round(total_output_cost_usd, 6),}@@ -988,6 +1002,10 @@ class SavingsTracker:# message-only while per-model dollars did not.tool_search_saved: int = 0,output_tokens_saved: int = 0,+ # Tokens the model actually emitted, DISJOINT from+ # ``output_tokens_saved`` (which counts the ones it did not). Priced+ # into ``lifetime["total_output_cost_usd"]``; see the delta below.+ output_tokens: int = 0,provider: str | None = None,project: str | None = None,cache_read_tokens: int = 0,@@ -1079,6 +1097,16 @@ class SavingsTracker:if priced is not Noneelse _estimate_output_savings_usd(model, delta_output_tokens_saved))+ # Completion spend. ``total_input_cost_usd`` has never had an output+ # counterpart, so anything answering "what share of my bill did+ # Headroom remove" had to divide savings that include output shaping by+ # an input-only denominator, which overstates the rate. Estimated from+ # the same rate table as the output SAVINGS beside it, so the two are+ # consistent even when litellm has no price for the model.+ delta_output_cost_usd = _estimate_output_cost_usd(+ model,+ max(_coerce_int(output_tokens), 0),+ )delta_cache_savings_usd = (max(_coerce_float(priced.get("provider_cache")), 0.0)if priced is not None@@ -1149,6 +1177,10 @@ class SavingsTracker:lifetime.get("output_savings_usd", 0.0) + delta_output_savings_usd,6,)+ lifetime["total_output_cost_usd"] = round(+ lifetime.get("total_output_cost_usd", 0.0) + delta_output_cost_usd,+ 6,+ )session = self._state["display_session"]last_activity = _parse_timestamp(session.get("last_activity_at"))@@ -1235,6 +1267,7 @@ class SavingsTracker:"total_input_cost_usd": lifetime["total_input_cost_usd"],"output_tokens_saved": lifetime.get("output_tokens_saved", 0),"output_savings_usd": lifetime.get("output_savings_usd", 0.0),+ "total_output_cost_usd": lifetime.get("total_output_cost_usd", 0.0),})self._trim_history_locked(reference_time=timestamp_dt)@@ -1565,6 +1598,7 @@ class SavingsTracker:"total_input_cost_usd","output_tokens_saved_delta","output_savings_usd_delta",+ "total_output_cost_usd_delta",]buffer = StringIO()@@ -1608,6 +1642,9 @@ class SavingsTracker:"cache_savings_usd": 0.0,"total_input_tokens": 0,"total_input_cost_usd": 0.0,+ "output_tokens_saved": 0,+ "output_savings_usd": 0.0,+ "total_output_cost_usd": 0.0,},"display_session": _empty_display_session(),"history": [],@@ -1674,6 +1711,9 @@ class SavingsTracker:lifetime_basis = BASIS_UNKNOWNmigrated_at = Noneis_v6_lifetime = False+ lifetime_output_tokens_saved = 0+ lifetime_output_savings_usd = 0.0+ lifetime_output_cost_usd = 0.0if isinstance(lifetime_raw, dict):lifetime_requests = _coerce_int(lifetime_raw.get("requests"))lifetime_tokens_saved = _coerce_int(lifetime_raw.get("tokens_saved"))@@ -1689,6 +1729,9 @@ class SavingsTracker:lifetime_raw.get("compression_savings_list_usd"))lifetime_basis = str(lifetime_raw.get("savings_basis") or BASIS_UNKNOWN)+ lifetime_output_tokens_saved = _coerce_int(lifetime_raw.get("output_tokens_saved"))+ lifetime_output_savings_usd = _coerce_float(lifetime_raw.get("output_savings_usd"))+ lifetime_output_cost_usd = _coerce_float(lifetime_raw.get("total_output_cost_usd"))if normalized_history:last = normalized_history[-1]@@ -1708,6 +1751,22 @@ class SavingsTracker:lifetime_input_cost_usd,_coerce_float(last.get("total_input_cost_usd")),)+ # The output side accumulates the same way and is checkpointed the+ # same way, so it recovers the same way. Without this a restart+ # restarted the counters at 0 while history kept the old totals,+ # and the rollup's max(delta, 0) then swallowed the difference.+ lifetime_output_tokens_saved = max(+ lifetime_output_tokens_saved,+ _coerce_int(last.get("output_tokens_saved")),+ )+ lifetime_output_savings_usd = max(+ lifetime_output_savings_usd,+ _coerce_float(last.get("output_savings_usd")),+ )+ lifetime_output_cost_usd = max(+ lifetime_output_cost_usd,+ _coerce_float(last.get("total_output_cost_usd")),+ )# v6 migration seed. Runs HERE, after the history back-fill above, not# beside the lifetime read: a state whose `lifetime` block is missing or@@ -1742,6 +1801,9 @@ class SavingsTracker:"cache_savings_usd": round(lifetime_cache_savings_usd, 6),"total_input_tokens": lifetime_input_tokens,"total_input_cost_usd": round(lifetime_input_cost_usd, 6),+ "output_tokens_saved": lifetime_output_tokens_saved,+ "output_savings_usd": round(lifetime_output_savings_usd, 6),+ "total_output_cost_usd": round(lifetime_output_cost_usd, 6),},"display_session": _normalize_display_session(raw.get("display_session")),"history": normalized_history,@@ -2021,6 +2083,7 @@ class SavingsTracker:prev_total_input_cost_usd = 0.0prev_output_tokens = 0prev_output_usd = 0.0+ prev_output_cost_usd = 0.0prev_cache_read_tokens = 0prev_cache_savings_usd = 0.0# What the bucket's cache reads actually COST, priced per checkpoint@@ -2075,6 +2138,7 @@ class SavingsTracker:total_input_cost_usd = _coerce_float(point.get("total_input_cost_usd"))total_output_tokens = _coerce_int(point.get("output_tokens_saved"))total_output_usd = _coerce_float(point.get("output_savings_usd"))+ total_output_cost_usd = _coerce_float(point.get("total_output_cost_usd"))delta_tokens = max(total_tokens_saved - prev_total_tokens, 0)delta_usd = max(total_usd - prev_total_usd, 0.0)delta_input_tokens = max(total_input_tokens - prev_total_input_tokens, 0)@@ -2085,6 +2149,7 @@ class SavingsTracker:delta_output_tokens = max(total_output_tokens - prev_output_tokens, 0)delta_output_usd = max(total_output_usd - prev_output_usd, 0.0)+ delta_output_cost_usd = max(total_output_cost_usd - prev_output_cost_usd, 0.0)total_cache_read_tokens = _coerce_int(point.get("cache_read_tokens"))total_cache_savings_usd = _coerce_float(point.get("cache_savings_usd"))@@ -2101,6 +2166,7 @@ class SavingsTracker:prev_total_input_cost_usd = total_input_cost_usdprev_output_tokens = total_output_tokensprev_output_usd = total_output_usd+ prev_output_cost_usd = total_output_cost_usdentry = aggregated.setdefault(bucket_key,@@ -2116,6 +2182,8 @@ class SavingsTracker:"total_input_cost_usd": total_input_cost_usd,"output_tokens_saved_delta": 0,"output_savings_usd_delta": 0.0,+ "total_output_cost_usd_delta": 0.0,+ "total_output_cost_usd": total_output_cost_usd,**_empty_cache_delta(),"by_provider": {},"by_model": {},@@ -2140,6 +2208,11 @@ class SavingsTracker:entry["output_savings_usd_delta"] + delta_output_usd,6,)+ entry["total_output_cost_usd_delta"] = round(+ entry["total_output_cost_usd_delta"] + delta_output_cost_usd,+ 6,+ )+ entry["total_output_cost_usd"] = round(total_output_cost_usd, 6)_add_cache(entry, delta_cache_read_tokens, delta_cache_savings_usd, delta_cache_read_cost_usd)diff --git a/tests/test_proxy_savings_history.py b/tests/test_proxy_savings_history.pyindex d6a7fdcc0..834407512 100644--- a/tests/test_proxy_savings_history.py+++ b/tests/test_proxy_savings_history.py@@ -84,6 +84,7 @@ def test_savings_tracker_helpers_normalize_inputs_and_paths(tmp_path, monkeypatc"total_input_cost_usd": 0.0,"output_tokens_saved": 0,"output_savings_usd": 0.0,+ "total_output_cost_usd": 0.0,}assert savings_tracker_module._normalize_history_entry({"timestamp": "bad"}) is Noneassert savings_tracker_module._normalize_history_entry(object()) is None@@ -142,6 +143,9 @@ def test_savings_tracker_sanitizes_legacy_state_and_applies_retention(tmp_path):"cache_savings_usd": 0.0,"total_input_tokens": 0,"total_input_cost_usd": 0.0,+ "output_tokens_saved": 0,+ "output_savings_usd": 0.0,+ "total_output_cost_usd": 0.0,}assert snapshot["display_session"] == savings_tracker_module._empty_display_session()assert snapshot["history"] == [@@ -157,6 +161,7 @@ def test_savings_tracker_sanitizes_legacy_state_and_applies_retention(tmp_path):"total_input_cost_usd": 0.0,"output_tokens_saved": 0,"output_savings_usd": 0.0,+ "total_output_cost_usd": 0.0,}]assert snapshot["retention"] == {@@ -185,6 +190,9 @@ def test_non_dict_savings_state_resets_to_default(tmp_path):"cache_savings_usd": 0.0,"total_input_tokens": 0,"total_input_cost_usd": 0.0,+ "output_tokens_saved": 0,+ "output_savings_usd": 0.0,+ "total_output_cost_usd": 0.0,}assert snapshot["display_session"] == savings_tracker_module._empty_display_session()assert snapshot["history"] == []@@ -1424,7 +1432,8 @@ def test_stats_history_csv_export_is_frontend_friendly(tmp_path, monkeypatch):"timestamp,tokens_saved,compression_savings_usd_delta,total_tokens_saved,""compression_savings_usd,total_input_tokens_delta,total_input_tokens,""total_input_cost_usd_delta,total_input_cost_usd,"- "output_tokens_saved_delta,output_savings_usd_delta"+ "output_tokens_saved_delta,output_savings_usd_delta,"+ "total_output_cost_usd_delta")assert len(lines) >= 2assert "total_tokens_saved" in lines[0]diff --git a/tests/test_savings_tracker_output_cost.py b/tests/test_savings_tracker_output_cost.pynew file mode 100644index 000000000..015b6004b--- /dev/null+++ b/tests/test_savings_tracker_output_cost.py@@ -0,0 +1,164 @@+"""Lifetime output spend, so a bill-share rate has a denominator that fits it.++`total_input_cost_usd` has been tracked since the tracker existed; the tokens+the model EMITTED were priced nowhere durable. Any "what share of my bill did+Headroom remove" figure therefore had to divide savings that include output+shaping by an input-only denominator, which overstates the rate. These pin the+new `total_output_cost_usd` on the lifetime block and its per-bucket delta on+the rollup series, plus the legacy-checkpoint path.+"""++from __future__ import annotations++import types++from headroom.proxy import savings_tracker as st+from headroom.proxy.savings_tracker import SavingsTracker, _normalize_history_entry+++def _priced_litellm() -> types.SimpleNamespace:+ return types.SimpleNamespace(+ model_cost={+ "test-model": {+ "input_cost_per_token": 1e-06,+ "output_cost_per_token": 5e-06,+ }+ },+ cost_per_token=lambda **_kw: (0.0, 0.0),+ )+++def _tracker(tmp_path) -> SavingsTracker:+ return SavingsTracker(path=tmp_path / "savings.json", save_flush_every=1)+++def test_emitted_output_tokens_are_priced_onto_lifetime(monkeypatch, tmp_path):+ monkeypatch.setattr(st, "_get_litellm_module", _priced_litellm)+ tracker = _tracker(tmp_path)++ tracker.record_request(+ model="test-model",+ input_tokens=1_000,+ tokens_saved=100,+ output_tokens=2_000,+ )++ lifetime = tracker.snapshot()["lifetime"]+ # 2,000 emitted tokens at $5/M.+ assert lifetime["total_output_cost_usd"] == 0.01+ # The input side is untouched by the new field.+ assert lifetime["total_input_cost_usd"] > 0+++def test_output_cost_is_disjoint_from_output_savings(monkeypatch, tmp_path):+ """Emitted and not-emitted are separate counts at the same rate."""+ monkeypatch.setattr(st, "_get_litellm_module", _priced_litellm)+ tracker = _tracker(tmp_path)++ tracker.record_request(+ model="test-model",+ input_tokens=1_000,+ tokens_saved=0,+ output_tokens=2_000,+ output_tokens_saved=400,+ )++ lifetime = tracker.snapshot()["lifetime"]+ assert lifetime["total_output_cost_usd"] == 0.01 # 2,000 emitted+ assert lifetime["output_savings_usd"] == 0.002 # 400 not emitted+++def test_bill_share_rate_is_computable_and_below_the_input_only_one(monkeypatch, tmp_path):+ """The reason the field exists: an input-only denominator overstates."""+ monkeypatch.setattr(st, "_get_litellm_module", _priced_litellm)+ tracker = _tracker(tmp_path)++ tracker.record_request(+ model="test-model",+ input_tokens=1_000,+ tokens_saved=100,+ output_tokens=2_000,+ output_tokens_saved=400,+ )++ lifetime = tracker.snapshot()["lifetime"]+ saved = lifetime["compression_savings_usd"] + lifetime["output_savings_usd"]+ input_only = saved / (saved + lifetime["total_input_cost_usd"])+ whole_bill = saved / (+ saved + lifetime["total_input_cost_usd"] + lifetime["total_output_cost_usd"]+ )+ assert whole_bill < input_only+++def test_legacy_checkpoints_default_the_field_rather_than_dropping(tmp_path):+ normalized = _normalize_history_entry(+ {+ "timestamp": "2026-01-01T00:00:00+00:00",+ "total_tokens_saved": 10,+ "compression_savings_usd": 0.5,+ }+ )+ assert normalized is not None+ assert normalized["total_output_cost_usd"] == 0.0+++def test_rollup_series_carries_the_per_bucket_output_spend(monkeypatch, tmp_path):+ monkeypatch.setattr(st, "_get_litellm_module", _priced_litellm)+ tracker = _tracker(tmp_path)++ for _ in range(3):+ tracker.record_request(+ model="test-model",+ input_tokens=1_000,+ tokens_saved=100,+ output_tokens=2_000,+ )++ daily = tracker.history_response()["series"]["daily"]+ assert daily, "expected at least one bucket"+ bucket = daily[-1]+ # Three requests at $0.01 of emitted output each, as a delta and a total.+ assert bucket["total_output_cost_usd_delta"] == 0.03+ assert bucket["total_output_cost_usd"] == 0.03+++def test_lifetime_output_spend_survives_a_restart(monkeypatch, tmp_path):+ """The field is a LIFETIME total, so a reload must not restart it at 0.++ The lifetime block accumulates in memory and is checkpointed into history;+ on load the block has to be recovered from both. Without that, request two+ lands on a zeroed counter and the rollup's ``max(delta, 0)`` clamp hides+ the regression by reporting a flat total instead of a negative one.+ """+ monkeypatch.setattr(st, "_get_litellm_module", _priced_litellm)+ path = tmp_path / "savings.json"++ def record(tracker: SavingsTracker) -> None:+ tracker.record_request(+ model="test-model",+ input_tokens=1_000,+ tokens_saved=100,+ output_tokens=2_000,+ output_tokens_saved=400,+ )++ first = SavingsTracker(path=path, save_flush_every=1)+ record(first)+ assert first.snapshot()["lifetime"]["total_output_cost_usd"] == 0.01++ reloaded = SavingsTracker(path=path, save_flush_every=1)+ restored = reloaded.snapshot()["lifetime"]+ assert restored["total_output_cost_usd"] == 0.01+ assert restored["output_savings_usd"] == 0.002+ assert restored["output_tokens_saved"] == 400++ record(reloaded)+ lifetime = reloaded.snapshot()["lifetime"]+ assert lifetime["total_output_cost_usd"] == 0.02+ assert lifetime["output_savings_usd"] == 0.004+ assert lifetime["output_tokens_saved"] == 800++ bucket = reloaded.history_response()["series"]["daily"][-1]+ assert bucket["total_output_cost_usd"] == 0.02+ assert bucket["total_output_cost_usd_delta"] == 0.02+ assert bucket["output_savings_usd_delta"] == 0.004