Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
feat: add spend and input-token rollups to stats-history
Garmec6753205aa0
diff --git a/headroom/proxy/savings_tracker.py b/headroom/proxy/savings_tracker.pyindex 84b252234..0ac72399c 100644--- a/headroom/proxy/savings_tracker.py+++ b/headroom/proxy/savings_tracker.py@@ -145,16 +145,24 @@ def _normalize_history_entry(entry: Any) -> dict[str, Any] | None:timestamp: datetime | None = Nonetotal_tokens_saved = 0compression_savings_usd = 0.0+ total_input_tokens = 0+ total_input_cost_usd = 0.0if isinstance(entry, dict):timestamp = _parse_timestamp(entry.get("timestamp"))total_tokens_saved = _coerce_int(entry.get("total_tokens_saved"))compression_savings_usd = _coerce_float(entry.get("compression_savings_usd"))+ total_input_tokens = _coerce_int(entry.get("total_input_tokens"))+ total_input_cost_usd = _coerce_float(entry.get("total_input_cost_usd"))elif isinstance(entry, (list, tuple)) and len(entry) >= 2:timestamp = _parse_timestamp(entry[0])total_tokens_saved = _coerce_int(entry[1])if len(entry) >= 3:compression_savings_usd = _coerce_float(entry[2])+ if len(entry) >= 4:+ total_input_tokens = _coerce_int(entry[3])+ if len(entry) >= 5:+ total_input_cost_usd = _coerce_float(entry[4])else:return None@@ -165,6 +173,8 @@ def _normalize_history_entry(entry: Any) -> dict[str, Any] | None:"timestamp": _to_utc_iso(timestamp),"total_tokens_saved": total_tokens_saved,"compression_savings_usd": round(compression_savings_usd, 6),+ "total_input_tokens": total_input_tokens,+ "total_input_cost_usd": round(total_input_cost_usd, 6),}@@ -192,6 +202,8 @@ class SavingsTracker:*,model: str,tokens_saved: int,+ total_input_tokens: int | None = None,+ total_input_cost_usd: float | None = None,timestamp: datetime | str | None = None,) -> bool:"""Persist a cumulative savings checkpoint when compression changed totals."""@@ -217,12 +229,28 @@ class SavingsTracker:lifetime["compression_savings_usd"] = round(lifetime["compression_savings_usd"] + delta_usd, 6)+ lifetime["total_input_tokens"] = max(+ lifetime["total_input_tokens"],+ _coerce_int(total_input_tokens, default=lifetime["total_input_tokens"]),+ )+ lifetime["total_input_cost_usd"] = round(+ max(+ lifetime["total_input_cost_usd"],+ _coerce_float(+ total_input_cost_usd,+ default=lifetime["total_input_cost_usd"],+ ),+ ),+ 6,+ )self._state["history"].append({"timestamp": _to_utc_iso(timestamp_dt),"total_tokens_saved": lifetime["tokens_saved"],"compression_savings_usd": lifetime["compression_savings_usd"],+ "total_input_tokens": lifetime["total_input_tokens"],+ "total_input_cost_usd": lifetime["total_input_cost_usd"],})self._trim_history_locked(reference_time=timestamp_dt)@@ -277,7 +305,13 @@ class SavingsTracker:"""Export history or rollup series as CSV."""rows = self.export_rows(series=series)if series == "history":- fieldnames = ["timestamp", "total_tokens_saved", "compression_savings_usd"]+ fieldnames = [+ "timestamp",+ "total_tokens_saved",+ "compression_savings_usd",+ "total_input_tokens",+ "total_input_cost_usd",+ ]else:fieldnames = ["timestamp",@@ -285,6 +319,10 @@ class SavingsTracker:"compression_savings_usd_delta","total_tokens_saved","compression_savings_usd",+ "total_input_tokens_delta",+ "total_input_tokens",+ "total_input_cost_usd_delta",+ "total_input_cost_usd",]buffer = StringIO()@@ -311,7 +349,12 @@ class SavingsTracker:def _default_state(self) -> dict[str, Any]:return {"schema_version": SCHEMA_VERSION,- "lifetime": {"tokens_saved": 0, "compression_savings_usd": 0.0},+ "lifetime": {+ "tokens_saved": 0,+ "compression_savings_usd": 0.0,+ "total_input_tokens": 0,+ "total_input_cost_usd": 0.0,+ },"history": [],}@@ -345,9 +388,13 @@ class SavingsTracker:lifetime_raw = raw.get("lifetime", {})lifetime_tokens_saved = 0lifetime_savings_usd = 0.0+ lifetime_input_tokens = 0+ lifetime_input_cost_usd = 0.0if isinstance(lifetime_raw, dict):lifetime_tokens_saved = _coerce_int(lifetime_raw.get("tokens_saved"))lifetime_savings_usd = _coerce_float(lifetime_raw.get("compression_savings_usd"))+ lifetime_input_tokens = _coerce_int(lifetime_raw.get("total_input_tokens"))+ lifetime_input_cost_usd = _coerce_float(lifetime_raw.get("total_input_cost_usd"))if normalized_history:last = normalized_history[-1]@@ -356,12 +403,22 @@ class SavingsTracker:lifetime_savings_usd,_coerce_float(last["compression_savings_usd"]),)+ lifetime_input_tokens = max(+ lifetime_input_tokens,+ _coerce_int(last.get("total_input_tokens")),+ )+ lifetime_input_cost_usd = max(+ lifetime_input_cost_usd,+ _coerce_float(last.get("total_input_cost_usd")),+ )state = {"schema_version": SCHEMA_VERSION,"lifetime": {"tokens_saved": lifetime_tokens_saved,"compression_savings_usd": round(lifetime_savings_usd, 6),+ "total_input_tokens": lifetime_input_tokens,+ "total_input_cost_usd": round(lifetime_input_cost_usd, 6),},"history": normalized_history,}@@ -437,6 +494,8 @@ class SavingsTracker:aggregated: dict[str, dict[str, Any]] = {}prev_total_tokens = 0prev_total_usd = 0.0+ prev_total_input_tokens = 0+ prev_total_input_cost_usd = 0.0for point in history:timestamp = _parse_timestamp(point["timestamp"])@@ -448,11 +507,17 @@ class SavingsTracker:bucket_key = _to_utc_iso(bucket_start)total_tokens_saved = _coerce_int(point.get("total_tokens_saved"))total_usd = _coerce_float(point.get("compression_savings_usd"))+ total_input_tokens = _coerce_int(point.get("total_input_tokens"))+ total_input_cost_usd = _coerce_float(point.get("total_input_cost_usd"))delta_tokens = max(total_tokens_saved - prev_total_tokens, 0)delta_usd = max(total_usd - prev_total_usd, 0.0)+ delta_input_tokens = max(total_input_tokens - prev_total_input_tokens, 0)+ delta_input_cost_usd = max(total_input_cost_usd - prev_total_input_cost_usd, 0.0)prev_total_tokens = total_tokens_savedprev_total_usd = total_usd+ prev_total_input_tokens = total_input_tokens+ prev_total_input_cost_usd = total_input_cost_usdentry = aggregated.setdefault(bucket_key,@@ -462,6 +527,10 @@ class SavingsTracker:"compression_savings_usd_delta": 0.0,"total_tokens_saved": total_tokens_saved,"compression_savings_usd": total_usd,+ "total_input_tokens_delta": 0,+ "total_input_tokens": total_input_tokens,+ "total_input_cost_usd_delta": 0.0,+ "total_input_cost_usd": total_input_cost_usd,},)entry["tokens_saved"] += delta_tokens@@ -469,7 +538,14 @@ class SavingsTracker:entry["compression_savings_usd_delta"] + delta_usd,6,)+ entry["total_input_tokens_delta"] += delta_input_tokens+ entry["total_input_cost_usd_delta"] = round(+ entry["total_input_cost_usd_delta"] + delta_input_cost_usd,+ 6,+ )entry["total_tokens_saved"] = total_tokens_savedentry["compression_savings_usd"] = round(total_usd, 6)+ entry["total_input_tokens"] = total_input_tokens+ entry["total_input_cost_usd"] = round(total_input_cost_usd, 6)return list(aggregated.values())diff --git a/headroom/proxy/server.py b/headroom/proxy/server.pyindex 8e4c76f78..71c56804d 100644--- a/headroom/proxy/server.py+++ b/headroom/proxy/server.py@@ -1137,7 +1137,11 @@ class CostTracker:class PrometheusMetrics:"""Prometheus-compatible metrics."""- def __init__(self, savings_tracker: SavingsTracker | None = None):+ def __init__(+ self,+ savings_tracker: SavingsTracker | None = None,+ cost_tracker: CostTracker | None = None,+ ):self.requests_total = 0self.requests_by_provider: dict[str, int] = defaultdict(int)self.requests_by_model: dict[str, int] = defaultdict(int)@@ -1201,9 +1205,55 @@ class PrometheusMetrics:# Cumulative savings history (timestamp → cumulative tokens saved)self.savings_history: list[tuple[str, int]] = []self.savings_tracker = savings_tracker or SavingsTracker()+ self.cost_tracker = cost_tracker+ tracker_lifetime = self.savings_tracker.snapshot()["lifetime"]+ self._savings_tracker_input_tokens_offset = max(+ int(tracker_lifetime.get("total_input_tokens", 0) or 0),+ 0,+ )+ self._savings_tracker_input_cost_usd_offset = max(+ float(tracker_lifetime.get("total_input_cost_usd", 0.0) or 0.0),+ 0.0,+ )self._lock = asyncio.Lock()+ def _current_savings_tracker_totals(self) -> tuple[int, float]:+ total_input_tokens = self._savings_tracker_input_tokens_offset + self.tokens_input_total+ total_input_cost_usd = self._savings_tracker_input_cost_usd_offset++ if self.cost_tracker is None:+ return total_input_tokens, total_input_cost_usd++ try:+ cost_stats = self.cost_tracker.stats()+ except Exception:+ logger.debug("Failed to read cost tracker totals for savings history", exc_info=True)+ return total_input_tokens, total_input_cost_usd++ tracked_input_tokens = cost_stats.get("total_input_tokens")+ tracked_input_cost_usd = cost_stats.get("total_input_cost_usd")++ if tracked_input_tokens is not None:+ try:+ total_input_tokens = self._savings_tracker_input_tokens_offset + max(+ int(tracked_input_tokens),+ 0,+ )+ except (TypeError, ValueError):+ pass++ if tracked_input_cost_usd is not None:+ try:+ total_input_cost_usd = self._savings_tracker_input_cost_usd_offset + max(+ float(tracked_input_cost_usd),+ 0.0,+ )+ except (TypeError, ValueError):+ pass++ return total_input_tokens, total_input_cost_usd+async def record_request(self,provider: str,@@ -1293,9 +1343,12 @@ class PrometheusMetrics:self.savings_history = self.savings_history[-500:]if tokens_saved > 0:+ total_input_tokens, total_input_cost_usd = self._current_savings_tracker_totals()self.savings_tracker.record_compression_savings(model=model,tokens_saved=tokens_saved,+ total_input_tokens=total_input_tokens,+ total_input_cost_usd=total_input_cost_usd,)async def record_rate_limited(self):@@ -1603,7 +1656,7 @@ class HeadroomProxy:else None)- self.metrics = PrometheusMetrics()+ self.metrics = PrometheusMetrics(cost_tracker=self.cost_tracker)# Prefix cache tracking: freeze already-cached messages to avoid# invalidating the provider's prefix cache with our transformsdiff --git a/tests/test_proxy_savings_history.py b/tests/test_proxy_savings_history.pyindex 3b6dc0bfe..fe1eac4bf 100644--- a/tests/test_proxy_savings_history.py+++ b/tests/test_proxy_savings_history.py@@ -20,6 +20,8 @@ from headroom.proxy.server import ProxyConfig, create_appdef _record_request(client: TestClient, *, model: str, tokens_saved: int) -> None:proxy = client.app.state.proxy+ if proxy.cost_tracker:+ proxy.cost_tracker.record_tokens(model, tokens_saved, 120)asyncio.run(proxy.metrics.record_request(provider="openai",@@ -58,6 +60,8 @@ def test_savings_tracker_helpers_normalize_inputs_and_paths(tmp_path, monkeypatc"timestamp": "2026-03-27T09:00:00Z","total_tokens_saved": 12,"compression_savings_usd": 0.5,+ "total_input_tokens": 0,+ "total_input_cost_usd": 0.0,}assert savings_tracker_module._normalize_history_entry({"timestamp": "bad"}) is Noneassert savings_tracker_module._normalize_history_entry(object()) is None@@ -99,12 +103,16 @@ def test_savings_tracker_sanitizes_legacy_state_and_applies_retention(tmp_path):assert snapshot["lifetime"] == {"tokens_saved": 30,"compression_savings_usd": pytest.approx(0.03),+ "total_input_tokens": 0,+ "total_input_cost_usd": 0.0,}assert snapshot["history"] == [{"timestamp": "2026-03-27T09:00:00Z","total_tokens_saved": 30,"compression_savings_usd": 0.03,+ "total_input_tokens": 0,+ "total_input_cost_usd": 0.0,}]assert snapshot["retention"] == {@@ -123,6 +131,8 @@ def test_non_dict_savings_state_resets_to_default(tmp_path):assert snapshot["lifetime"] == {"tokens_saved": 0,"compression_savings_usd": 0.0,+ "total_input_tokens": 0,+ "total_input_cost_usd": 0.0,}assert snapshot["history"] == []@@ -145,6 +155,8 @@ def test_record_compression_savings_skips_empty_updates_and_normalizes_timestampassert tracker.record_compression_savings(model="gpt-4o",tokens_saved=10,+ total_input_tokens=120,+ total_input_cost_usd=0.24,timestamp=local_time,)@@ -153,6 +165,8 @@ def test_record_compression_savings_skips_empty_updates_and_normalizes_timestampassert tracker.record_compression_savings(model="gpt-4o",tokens_saved=5,+ total_input_tokens=180,+ total_input_cost_usd=0.36,timestamp="not-a-timestamp",)@@ -162,16 +176,22 @@ def test_record_compression_savings_skips_empty_updates_and_normalizes_timestamp"timestamp": "2026-03-27T08:00:00Z","total_tokens_saved": 10,"compression_savings_usd": 0.01,+ "total_input_tokens": 120,+ "total_input_cost_usd": 0.24,},{"timestamp": "2026-03-27T12:34:00Z","total_tokens_saved": 15,"compression_savings_usd": 0.015,+ "total_input_tokens": 180,+ "total_input_cost_usd": 0.36,},]persisted = json.loads(path.read_text(encoding="utf-8"))assert persisted["lifetime"]["tokens_saved"] == 15+ assert persisted["lifetime"]["total_input_tokens"] == 180+ assert persisted["lifetime"]["total_input_cost_usd"] == pytest.approx(0.36)assert persisted["history"][-1]["timestamp"] == "2026-03-27T12:34:00Z"@@ -219,7 +239,7 @@ def test_litellm_resolution_and_savings_estimation_fallbacks(monkeypatch):assert savings_tracker_module._estimate_compression_savings_usd("gpt-4o", 100) == 0.0-def test_savings_tracker_rollups_are_chart_friendly(tmp_path, monkeypatch):+def test_savings_tracker_rollups_preserve_spend_and_input_history(tmp_path, monkeypatch):path = tmp_path / "proxy_savings.json"tracker = SavingsTracker(path=str(path), max_history_points=100, max_history_age_days=30)monkeypatch.setattr(@@ -230,26 +250,36 @@ def test_savings_tracker_rollups_are_chart_friendly(tmp_path, monkeypatch):tracker.record_compression_savings(model="gpt-4o",tokens_saved=100,+ total_input_tokens=120,+ total_input_cost_usd=0.24,timestamp="2026-03-27T09:10:00Z",)tracker.record_compression_savings(model="gpt-4o",tokens_saved=50,+ total_input_tokens=210,+ total_input_cost_usd=0.42,timestamp="2026-03-27T09:40:00Z",)tracker.record_compression_savings(model="gpt-4o",tokens_saved=25,+ total_input_tokens=300,+ total_input_cost_usd=0.63,timestamp="2026-03-27T10:05:00Z",)tracker.record_compression_savings(model="gpt-4o",tokens_saved=10,+ total_input_tokens=360,+ total_input_cost_usd=0.75,timestamp="2026-03-28T08:00:00Z",)tracker.record_compression_savings(model="gpt-4o",tokens_saved=20,+ total_input_tokens=450,+ total_input_cost_usd=0.93,timestamp="2026-04-02T14:00:00Z",)@@ -257,6 +287,8 @@ def test_savings_tracker_rollups_are_chart_friendly(tmp_path, monkeypatch):assert response["lifetime"]["tokens_saved"] == 205assert response["lifetime"]["compression_savings_usd"] == pytest.approx(0.205)+ assert response["lifetime"]["total_input_tokens"] == 450+ assert response["lifetime"]["total_input_cost_usd"] == pytest.approx(0.93)assert len(response["history"]) == 5hourly = response["series"]["hourly"]@@ -268,12 +300,28 @@ def test_savings_tracker_rollups_are_chart_friendly(tmp_path, monkeypatch):]assert hourly[0]["tokens_saved"] == 150assert hourly[0]["total_tokens_saved"] == 150+ assert hourly[0]["total_input_tokens_delta"] == 210+ assert hourly[0]["total_input_tokens"] == 210+ assert hourly[0]["total_input_cost_usd_delta"] == pytest.approx(0.42)+ assert hourly[0]["total_input_cost_usd"] == pytest.approx(0.42)assert hourly[1]["tokens_saved"] == 25assert hourly[1]["total_tokens_saved"] == 175+ assert hourly[1]["total_input_tokens_delta"] == 90+ assert hourly[1]["total_input_tokens"] == 300+ assert hourly[1]["total_input_cost_usd_delta"] == pytest.approx(0.21)+ assert hourly[1]["total_input_cost_usd"] == pytest.approx(0.63)assert hourly[2]["tokens_saved"] == 10assert hourly[2]["total_tokens_saved"] == 185+ assert hourly[2]["total_input_tokens_delta"] == 60+ assert hourly[2]["total_input_tokens"] == 360+ assert hourly[2]["total_input_cost_usd_delta"] == pytest.approx(0.12)+ assert hourly[2]["total_input_cost_usd"] == pytest.approx(0.75)assert hourly[3]["tokens_saved"] == 20assert hourly[3]["total_tokens_saved"] == 205+ assert hourly[3]["total_input_tokens_delta"] == 90+ assert hourly[3]["total_input_tokens"] == 450+ assert hourly[3]["total_input_cost_usd_delta"] == pytest.approx(0.18)+ assert hourly[3]["total_input_cost_usd"] == pytest.approx(0.93)daily = response["series"]["daily"]assert [point["timestamp"] for point in daily] == [@@ -283,10 +331,22 @@ def test_savings_tracker_rollups_are_chart_friendly(tmp_path, monkeypatch):]assert daily[0]["tokens_saved"] == 175assert daily[0]["total_tokens_saved"] == 175+ assert daily[0]["total_input_tokens_delta"] == 300+ assert daily[0]["total_input_tokens"] == 300+ assert daily[0]["total_input_cost_usd_delta"] == pytest.approx(0.63)+ assert daily[0]["total_input_cost_usd"] == pytest.approx(0.63)assert daily[1]["tokens_saved"] == 10assert daily[1]["total_tokens_saved"] == 185+ assert daily[1]["total_input_tokens_delta"] == 60+ assert daily[1]["total_input_tokens"] == 360+ assert daily[1]["total_input_cost_usd_delta"] == pytest.approx(0.12)+ assert daily[1]["total_input_cost_usd"] == pytest.approx(0.75)assert daily[2]["tokens_saved"] == 20assert daily[2]["total_tokens_saved"] == 205+ assert daily[2]["total_input_tokens_delta"] == 90+ assert daily[2]["total_input_tokens"] == 450+ assert daily[2]["total_input_cost_usd_delta"] == pytest.approx(0.18)+ assert daily[2]["total_input_cost_usd"] == pytest.approx(0.93)weekly = response["series"]["weekly"]assert [point["timestamp"] for point in weekly] == [@@ -321,6 +381,10 @@ def test_savings_tracker_rollups_are_chart_friendly(tmp_path, monkeypatch):def test_stats_history_persists_across_restarts_and_stats_stays_compatible(tmp_path, monkeypatch):savings_path = tmp_path / "proxy_savings.json"monkeypatch.setenv("HEADROOM_SAVINGS_PATH", str(savings_path))+ monkeypatch.setattr(+ "headroom.proxy.server.CostTracker._get_cache_prices",+ lambda self, model: (0.001, 0.0015, 0.002),+ )config = ProxyConfig(cache_enabled=False,@@ -346,8 +410,14 @@ def test_stats_history_persists_across_restarts_and_stats_stays_compatible(tmp_passert history_data["schema_version"] == 1assert history_data["storage_path"] == str(savings_path)assert history_data["lifetime"]["tokens_saved"] == 40+ assert history_data["lifetime"]["total_input_tokens"] == 120+ assert history_data["lifetime"]["total_input_cost_usd"] == pytest.approx(0.24)assert list(history_data["series"].keys()) == ["hourly", "daily", "weekly", "monthly"]assert history_data["exports"]["available_series"][-2:] == ["weekly", "monthly"]+ assert history_data["series"]["hourly"][0]["total_input_tokens_delta"] == 120+ assert history_data["series"]["hourly"][0]["total_input_cost_usd_delta"] == pytest.approx(+ 0.24+ )with TestClient(create_app(config)) as client:history = client.get("/stats-history")@@ -358,15 +428,25 @@ def test_stats_history_persists_across_restarts_and_stats_stays_compatible(tmp_pupdated = client.get("/stats-history").json()assert updated["lifetime"]["tokens_saved"] == 55+ assert updated["lifetime"]["total_input_tokens"] == 240+ assert updated["lifetime"]["total_input_cost_usd"] == pytest.approx(0.48)assert len(updated["history"]) == 2+ assert updated["series"]["daily"][0]["total_input_tokens_delta"] == 240+ assert updated["series"]["daily"][0]["total_input_cost_usd_delta"] == pytest.approx(0.48)persisted = json.loads(savings_path.read_text())assert persisted["lifetime"]["tokens_saved"] == 55+ assert persisted["lifetime"]["total_input_tokens"] == 240+ assert persisted["lifetime"]["total_input_cost_usd"] == pytest.approx(0.48)def test_stats_history_csv_export_is_frontend_friendly(tmp_path, monkeypatch):savings_path = tmp_path / "proxy_savings.json"monkeypatch.setenv("HEADROOM_SAVINGS_PATH", str(savings_path))+ monkeypatch.setattr(+ "headroom.proxy.server.CostTracker._get_cache_prices",+ lambda self, model: (0.001, 0.0015, 0.002),+ )config = ProxyConfig(cache_enabled=False,@@ -388,10 +468,12 @@ def test_stats_history_csv_export_is_frontend_friendly(tmp_path, monkeypatch):lines = response.text.strip().splitlines()assert lines[0] == ("timestamp,tokens_saved,compression_savings_usd_delta,total_tokens_saved,"- "compression_savings_usd"+ "compression_savings_usd,total_input_tokens_delta,total_input_tokens,"+ "total_input_cost_usd_delta,total_input_cost_usd")assert len(lines) >= 2assert "total_tokens_saved" in lines[0]+ assert "total_input_cost_usd" in lines[0]def test_malformed_savings_state_is_ignored_safely(tmp_path, monkeypatch):