Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

feat(savings): track lifetime output spend so a bill-share rate has a denominator (#3520)

gglucassc766bdb2b361
diff --git a/headroom/proxy/prometheus_metrics.py b/headroom/proxy/prometheus_metrics.py
index a0db9ceba..66cfbe7a2 100644
--- a/headroom/proxy/prometheus_metrics.py
+++ b/headroom/proxy/prometheus_metrics.py
@@ -1030,6 +1030,7 @@ class PrometheusMetrics:
total_input_tokens=total_input_tokens,
total_input_cost_usd=total_input_cost_usd,
output_tokens_saved=output_tokens_saved,
+ output_tokens=output_tokens,
estimated_savings_usd=savings_usd,
)
diff --git a/headroom/proxy/savings_tracker.py b/headroom/proxy/savings_tracker.py
index ba7f3609b..57d38d16b 100644
--- a/headroom/proxy/savings_tracker.py
+++ b/headroom/proxy/savings_tracker.py
@@ -389,6 +389,17 @@ def _estimate_output_savings_usd(model: str, tokens_saved: int) -> float:
return float(tokens_saved) * float(DEFAULT_FALLBACK_OUTPUT_COST_PER_TOKEN)
+def _estimate_output_cost_usd(model: str, output_tokens: int) -> float:
+ """Estimate what EMITTED output tokens cost, at the model's output rate.
+
+ Same rate table as ``_estimate_output_savings_usd``, which prices the
+ tokens the shaper kept the model from emitting; this prices the ones it
+ did emit. Split out so the call sites read as spend and saving rather than
+ one function doing double duty.
+ """
+ return _estimate_output_savings_usd(model, output_tokens)
+
+
def _estimate_cache_savings_usd(model: str, cache_read_tokens: int) -> float:
"""Estimate cache-read savings in USD — the discount delta vs list price.
@@ -601,6 +612,7 @@ def _normalize_history_entry(entry: Any) -> dict[str, Any] | None:
total_input_cost_usd = 0.0
output_tokens_saved = 0
output_savings_usd = 0.0
+ total_output_cost_usd = 0.0
provider = PROVIDER_UNKNOWN
model = MODEL_UNKNOWN
@@ -617,6 +629,7 @@ def _normalize_history_entry(entry: Any) -> dict[str, Any] | None:
total_input_cost_usd = _coerce_float(entry.get("total_input_cost_usd"))
output_tokens_saved = _coerce_int(entry.get("output_tokens_saved"))
output_savings_usd = _coerce_float(entry.get("output_savings_usd"))
+ total_output_cost_usd = _coerce_float(entry.get("total_output_cost_usd"))
provider = _normalize_provider(entry.get("provider"))
model = _normalize_model(entry.get("model"))
elif isinstance(entry, list | tuple) and len(entry) >= 2:
@@ -646,6 +659,7 @@ def _normalize_history_entry(entry: Any) -> dict[str, Any] | None:
"total_input_cost_usd": round(total_input_cost_usd, 6),
"output_tokens_saved": output_tokens_saved,
"output_savings_usd": round(output_savings_usd, 6),
+ "total_output_cost_usd": round(total_output_cost_usd, 6),
}
@@ -988,6 +1002,10 @@ class SavingsTracker:
# message-only while per-model dollars did not.
tool_search_saved: int = 0,
output_tokens_saved: int = 0,
+ # Tokens the model actually emitted, DISJOINT from
+ # ``output_tokens_saved`` (which counts the ones it did not). Priced
+ # into ``lifetime["total_output_cost_usd"]``; see the delta below.
+ output_tokens: int = 0,
provider: str | None = None,
project: str | None = None,
cache_read_tokens: int = 0,
@@ -1079,6 +1097,16 @@ class SavingsTracker:
if priced is not None
else _estimate_output_savings_usd(model, delta_output_tokens_saved)
)
+ # Completion spend. ``total_input_cost_usd`` has never had an output
+ # counterpart, so anything answering "what share of my bill did
+ # Headroom remove" had to divide savings that include output shaping by
+ # an input-only denominator, which overstates the rate. Estimated from
+ # the same rate table as the output SAVINGS beside it, so the two are
+ # consistent even when litellm has no price for the model.
+ delta_output_cost_usd = _estimate_output_cost_usd(
+ model,
+ max(_coerce_int(output_tokens), 0),
+ )
delta_cache_savings_usd = (
max(_coerce_float(priced.get("provider_cache")), 0.0)
if priced is not None
@@ -1149,6 +1177,10 @@ class SavingsTracker:
lifetime.get("output_savings_usd", 0.0) + delta_output_savings_usd,
6,
)
+ lifetime["total_output_cost_usd"] = round(
+ lifetime.get("total_output_cost_usd", 0.0) + delta_output_cost_usd,
+ 6,
+ )
session = self._state["display_session"]
last_activity = _parse_timestamp(session.get("last_activity_at"))
@@ -1235,6 +1267,7 @@ class SavingsTracker:
"total_input_cost_usd": lifetime["total_input_cost_usd"],
"output_tokens_saved": lifetime.get("output_tokens_saved", 0),
"output_savings_usd": lifetime.get("output_savings_usd", 0.0),
+ "total_output_cost_usd": lifetime.get("total_output_cost_usd", 0.0),
}
)
self._trim_history_locked(reference_time=timestamp_dt)
@@ -1565,6 +1598,7 @@ class SavingsTracker:
"total_input_cost_usd",
"output_tokens_saved_delta",
"output_savings_usd_delta",
+ "total_output_cost_usd_delta",
]
buffer = StringIO()
@@ -1608,6 +1642,9 @@ class SavingsTracker:
"cache_savings_usd": 0.0,
"total_input_tokens": 0,
"total_input_cost_usd": 0.0,
+ "output_tokens_saved": 0,
+ "output_savings_usd": 0.0,
+ "total_output_cost_usd": 0.0,
},
"display_session": _empty_display_session(),
"history": [],
@@ -1674,6 +1711,9 @@ class SavingsTracker:
lifetime_basis = BASIS_UNKNOWN
migrated_at = None
is_v6_lifetime = False
+ lifetime_output_tokens_saved = 0
+ lifetime_output_savings_usd = 0.0
+ lifetime_output_cost_usd = 0.0
if isinstance(lifetime_raw, dict):
lifetime_requests = _coerce_int(lifetime_raw.get("requests"))
lifetime_tokens_saved = _coerce_int(lifetime_raw.get("tokens_saved"))
@@ -1689,6 +1729,9 @@ class SavingsTracker:
lifetime_raw.get("compression_savings_list_usd")
)
lifetime_basis = str(lifetime_raw.get("savings_basis") or BASIS_UNKNOWN)
+ lifetime_output_tokens_saved = _coerce_int(lifetime_raw.get("output_tokens_saved"))
+ lifetime_output_savings_usd = _coerce_float(lifetime_raw.get("output_savings_usd"))
+ lifetime_output_cost_usd = _coerce_float(lifetime_raw.get("total_output_cost_usd"))
if normalized_history:
last = normalized_history[-1]
@@ -1708,6 +1751,22 @@ class SavingsTracker:
lifetime_input_cost_usd,
_coerce_float(last.get("total_input_cost_usd")),
)
+ # The output side accumulates the same way and is checkpointed the
+ # same way, so it recovers the same way. Without this a restart
+ # restarted the counters at 0 while history kept the old totals,
+ # and the rollup's max(delta, 0) then swallowed the difference.
+ lifetime_output_tokens_saved = max(
+ lifetime_output_tokens_saved,
+ _coerce_int(last.get("output_tokens_saved")),
+ )
+ lifetime_output_savings_usd = max(
+ lifetime_output_savings_usd,
+ _coerce_float(last.get("output_savings_usd")),
+ )
+ lifetime_output_cost_usd = max(
+ lifetime_output_cost_usd,
+ _coerce_float(last.get("total_output_cost_usd")),
+ )
# v6 migration seed. Runs HERE, after the history back-fill above, not
# beside the lifetime read: a state whose `lifetime` block is missing or
@@ -1742,6 +1801,9 @@ class SavingsTracker:
"cache_savings_usd": round(lifetime_cache_savings_usd, 6),
"total_input_tokens": lifetime_input_tokens,
"total_input_cost_usd": round(lifetime_input_cost_usd, 6),
+ "output_tokens_saved": lifetime_output_tokens_saved,
+ "output_savings_usd": round(lifetime_output_savings_usd, 6),
+ "total_output_cost_usd": round(lifetime_output_cost_usd, 6),
},
"display_session": _normalize_display_session(raw.get("display_session")),
"history": normalized_history,
@@ -2021,6 +2083,7 @@ class SavingsTracker:
prev_total_input_cost_usd = 0.0
prev_output_tokens = 0
prev_output_usd = 0.0
+ prev_output_cost_usd = 0.0
prev_cache_read_tokens = 0
prev_cache_savings_usd = 0.0
# What the bucket's cache reads actually COST, priced per checkpoint
@@ -2075,6 +2138,7 @@ class SavingsTracker:
total_input_cost_usd = _coerce_float(point.get("total_input_cost_usd"))
total_output_tokens = _coerce_int(point.get("output_tokens_saved"))
total_output_usd = _coerce_float(point.get("output_savings_usd"))
+ total_output_cost_usd = _coerce_float(point.get("total_output_cost_usd"))
delta_tokens = max(total_tokens_saved - prev_total_tokens, 0)
delta_usd = max(total_usd - prev_total_usd, 0.0)
delta_input_tokens = max(total_input_tokens - prev_total_input_tokens, 0)
@@ -2085,6 +2149,7 @@ class SavingsTracker:
delta_output_tokens = max(total_output_tokens - prev_output_tokens, 0)
delta_output_usd = max(total_output_usd - prev_output_usd, 0.0)
+ delta_output_cost_usd = max(total_output_cost_usd - prev_output_cost_usd, 0.0)
total_cache_read_tokens = _coerce_int(point.get("cache_read_tokens"))
total_cache_savings_usd = _coerce_float(point.get("cache_savings_usd"))
@@ -2101,6 +2166,7 @@ class SavingsTracker:
prev_total_input_cost_usd = total_input_cost_usd
prev_output_tokens = total_output_tokens
prev_output_usd = total_output_usd
+ prev_output_cost_usd = total_output_cost_usd
entry = aggregated.setdefault(
bucket_key,
@@ -2116,6 +2182,8 @@ class SavingsTracker:
"total_input_cost_usd": total_input_cost_usd,
"output_tokens_saved_delta": 0,
"output_savings_usd_delta": 0.0,
+ "total_output_cost_usd_delta": 0.0,
+ "total_output_cost_usd": total_output_cost_usd,
**_empty_cache_delta(),
"by_provider": {},
"by_model": {},
@@ -2140,6 +2208,11 @@ class SavingsTracker:
entry["output_savings_usd_delta"] + delta_output_usd,
6,
)
+ entry["total_output_cost_usd_delta"] = round(
+ entry["total_output_cost_usd_delta"] + delta_output_cost_usd,
+ 6,
+ )
+ entry["total_output_cost_usd"] = round(total_output_cost_usd, 6)
_add_cache(
entry, delta_cache_read_tokens, delta_cache_savings_usd, delta_cache_read_cost_usd
)
diff --git a/tests/test_proxy_savings_history.py b/tests/test_proxy_savings_history.py
index d6a7fdcc0..834407512 100644
--- a/tests/test_proxy_savings_history.py
+++ b/tests/test_proxy_savings_history.py
@@ -84,6 +84,7 @@ def test_savings_tracker_helpers_normalize_inputs_and_paths(tmp_path, monkeypatc
"total_input_cost_usd": 0.0,
"output_tokens_saved": 0,
"output_savings_usd": 0.0,
+ "total_output_cost_usd": 0.0,
}
assert savings_tracker_module._normalize_history_entry({"timestamp": "bad"}) is None
assert savings_tracker_module._normalize_history_entry(object()) is None
@@ -142,6 +143,9 @@ def test_savings_tracker_sanitizes_legacy_state_and_applies_retention(tmp_path):
"cache_savings_usd": 0.0,
"total_input_tokens": 0,
"total_input_cost_usd": 0.0,
+ "output_tokens_saved": 0,
+ "output_savings_usd": 0.0,
+ "total_output_cost_usd": 0.0,
}
assert snapshot["display_session"] == savings_tracker_module._empty_display_session()
assert snapshot["history"] == [
@@ -157,6 +161,7 @@ def test_savings_tracker_sanitizes_legacy_state_and_applies_retention(tmp_path):
"total_input_cost_usd": 0.0,
"output_tokens_saved": 0,
"output_savings_usd": 0.0,
+ "total_output_cost_usd": 0.0,
}
]
assert snapshot["retention"] == {
@@ -185,6 +190,9 @@ def test_non_dict_savings_state_resets_to_default(tmp_path):
"cache_savings_usd": 0.0,
"total_input_tokens": 0,
"total_input_cost_usd": 0.0,
+ "output_tokens_saved": 0,
+ "output_savings_usd": 0.0,
+ "total_output_cost_usd": 0.0,
}
assert snapshot["display_session"] == savings_tracker_module._empty_display_session()
assert snapshot["history"] == []
@@ -1424,7 +1432,8 @@ def test_stats_history_csv_export_is_frontend_friendly(tmp_path, monkeypatch):
"timestamp,tokens_saved,compression_savings_usd_delta,total_tokens_saved,"
"compression_savings_usd,total_input_tokens_delta,total_input_tokens,"
"total_input_cost_usd_delta,total_input_cost_usd,"
- "output_tokens_saved_delta,output_savings_usd_delta"
+ "output_tokens_saved_delta,output_savings_usd_delta,"
+ "total_output_cost_usd_delta"
)
assert len(lines) >= 2
assert "total_tokens_saved" in lines[0]
diff --git a/tests/test_savings_tracker_output_cost.py b/tests/test_savings_tracker_output_cost.py
new file mode 100644
index 000000000..015b6004b
--- /dev/null
+++ b/tests/test_savings_tracker_output_cost.py
@@ -0,0 +1,164 @@
+"""Lifetime output spend, so a bill-share rate has a denominator that fits it.
+
+`total_input_cost_usd` has been tracked since the tracker existed; the tokens
+the model EMITTED were priced nowhere durable. Any "what share of my bill did
+Headroom remove" figure therefore had to divide savings that include output
+shaping by an input-only denominator, which overstates the rate. These pin the
+new `total_output_cost_usd` on the lifetime block and its per-bucket delta on
+the rollup series, plus the legacy-checkpoint path.
+"""
+
+from __future__ import annotations
+
+import types
+
+from headroom.proxy import savings_tracker as st
+from headroom.proxy.savings_tracker import SavingsTracker, _normalize_history_entry
+
+
+def _priced_litellm() -> types.SimpleNamespace:
+ return types.SimpleNamespace(
+ model_cost={
+ "test-model": {
+ "input_cost_per_token": 1e-06,
+ "output_cost_per_token": 5e-06,
+ }
+ },
+ cost_per_token=lambda **_kw: (0.0, 0.0),
+ )
+
+
+def _tracker(tmp_path) -> SavingsTracker:
+ return SavingsTracker(path=tmp_path / "savings.json", save_flush_every=1)
+
+
+def test_emitted_output_tokens_are_priced_onto_lifetime(monkeypatch, tmp_path):
+ monkeypatch.setattr(st, "_get_litellm_module", _priced_litellm)
+ tracker = _tracker(tmp_path)
+
+ tracker.record_request(
+ model="test-model",
+ input_tokens=1_000,
+ tokens_saved=100,
+ output_tokens=2_000,
+ )
+
+ lifetime = tracker.snapshot()["lifetime"]
+ # 2,000 emitted tokens at $5/M.
+ assert lifetime["total_output_cost_usd"] == 0.01
+ # The input side is untouched by the new field.
+ assert lifetime["total_input_cost_usd"] > 0
+
+
+def test_output_cost_is_disjoint_from_output_savings(monkeypatch, tmp_path):
+ """Emitted and not-emitted are separate counts at the same rate."""
+ monkeypatch.setattr(st, "_get_litellm_module", _priced_litellm)
+ tracker = _tracker(tmp_path)
+
+ tracker.record_request(
+ model="test-model",
+ input_tokens=1_000,
+ tokens_saved=0,
+ output_tokens=2_000,
+ output_tokens_saved=400,
+ )
+
+ lifetime = tracker.snapshot()["lifetime"]
+ assert lifetime["total_output_cost_usd"] == 0.01 # 2,000 emitted
+ assert lifetime["output_savings_usd"] == 0.002 # 400 not emitted
+
+
+def test_bill_share_rate_is_computable_and_below_the_input_only_one(monkeypatch, tmp_path):
+ """The reason the field exists: an input-only denominator overstates."""
+ monkeypatch.setattr(st, "_get_litellm_module", _priced_litellm)
+ tracker = _tracker(tmp_path)
+
+ tracker.record_request(
+ model="test-model",
+ input_tokens=1_000,
+ tokens_saved=100,
+ output_tokens=2_000,
+ output_tokens_saved=400,
+ )
+
+ lifetime = tracker.snapshot()["lifetime"]
+ saved = lifetime["compression_savings_usd"] + lifetime["output_savings_usd"]
+ input_only = saved / (saved + lifetime["total_input_cost_usd"])
+ whole_bill = saved / (
+ saved + lifetime["total_input_cost_usd"] + lifetime["total_output_cost_usd"]
+ )
+ assert whole_bill < input_only
+
+
+def test_legacy_checkpoints_default_the_field_rather_than_dropping(tmp_path):
+ normalized = _normalize_history_entry(
+ {
+ "timestamp": "2026-01-01T00:00:00+00:00",
+ "total_tokens_saved": 10,
+ "compression_savings_usd": 0.5,
+ }
+ )
+ assert normalized is not None
+ assert normalized["total_output_cost_usd"] == 0.0
+
+
+def test_rollup_series_carries_the_per_bucket_output_spend(monkeypatch, tmp_path):
+ monkeypatch.setattr(st, "_get_litellm_module", _priced_litellm)
+ tracker = _tracker(tmp_path)
+
+ for _ in range(3):
+ tracker.record_request(
+ model="test-model",
+ input_tokens=1_000,
+ tokens_saved=100,
+ output_tokens=2_000,
+ )
+
+ daily = tracker.history_response()["series"]["daily"]
+ assert daily, "expected at least one bucket"
+ bucket = daily[-1]
+ # Three requests at $0.01 of emitted output each, as a delta and a total.
+ assert bucket["total_output_cost_usd_delta"] == 0.03
+ assert bucket["total_output_cost_usd"] == 0.03
+
+
+def test_lifetime_output_spend_survives_a_restart(monkeypatch, tmp_path):
+ """The field is a LIFETIME total, so a reload must not restart it at 0.
+
+ The lifetime block accumulates in memory and is checkpointed into history;
+ on load the block has to be recovered from both. Without that, request two
+ lands on a zeroed counter and the rollup's ``max(delta, 0)`` clamp hides
+ the regression by reporting a flat total instead of a negative one.
+ """
+ monkeypatch.setattr(st, "_get_litellm_module", _priced_litellm)
+ path = tmp_path / "savings.json"
+
+ def record(tracker: SavingsTracker) -> None:
+ tracker.record_request(
+ model="test-model",
+ input_tokens=1_000,
+ tokens_saved=100,
+ output_tokens=2_000,
+ output_tokens_saved=400,
+ )
+
+ first = SavingsTracker(path=path, save_flush_every=1)
+ record(first)
+ assert first.snapshot()["lifetime"]["total_output_cost_usd"] == 0.01
+
+ reloaded = SavingsTracker(path=path, save_flush_every=1)
+ restored = reloaded.snapshot()["lifetime"]
+ assert restored["total_output_cost_usd"] == 0.01
+ assert restored["output_savings_usd"] == 0.002
+ assert restored["output_tokens_saved"] == 400
+
+ record(reloaded)
+ lifetime = reloaded.snapshot()["lifetime"]
+ assert lifetime["total_output_cost_usd"] == 0.02
+ assert lifetime["output_savings_usd"] == 0.004
+ assert lifetime["output_tokens_saved"] == 800
+
+ bucket = reloaded.history_response()["series"]["daily"][-1]
+ assert bucket["total_output_cost_usd"] == 0.02
+ assert bucket["total_output_cost_usd_delta"] == 0.02
+ assert bucket["output_savings_usd_delta"] == 0.004