Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(proxy): expose persistent savings metrics (#1647)

Vinay Gupta5fe4e7b19530
diff --git a/docs/content/docs/metrics.mdx b/docs/content/docs/metrics.mdx
index 9d07b0d25..0318d9273 100644
--- a/docs/content/docs/metrics.mdx
+++ b/docs/content/docs/metrics.mdx
@@ -72,6 +72,9 @@ headroom_requests_total{mode="optimize"} 1234
# HELP headroom_tokens_saved_total Total tokens saved
headroom_tokens_saved_total 5678900
+# HELP headroom_persistent_savings_tokens_saved_total Durable lifetime input tokens saved by proxy compression
+headroom_persistent_savings_tokens_saved_total 5678900
+
# HELP headroom_compression_ratio Compression ratio histogram
headroom_compression_ratio_bucket{le="0.5"} 890
headroom_compression_ratio_bucket{le="0.7"} 1100
@@ -253,7 +256,8 @@ When the budget is exceeded, requests return a budget exceeded error, the `/stat
| Metric | What It Tells You | Target |
|--------|-------------------|--------|
-| `tokens_saved_total` | Total cost savings | Higher is better |
+| `headroom_tokens_saved_total` | Runtime tokens saved since this proxy process started | Higher is better |
+| `headroom_persistent_savings_tokens_saved_total` | Durable lifetime tokens saved from `/stats.persistent_savings` | Higher is better |
| `compression_ratio_avg` | Efficiency | 0.7--0.9 typical |
| `cache_hit_rate` | Cache effectiveness | >20% is good |
| `latency_p99` | Performance impact | <10ms |
@@ -265,7 +269,8 @@ Example Prometheus queries for a Grafana dashboard:
| Panel | PromQL |
|-------|--------|
-| Tokens Saved | `headroom_tokens_saved_total` |
+| Runtime Tokens Saved | `headroom_tokens_saved_total` |
+| Lifetime Tokens Saved | `headroom_persistent_savings_tokens_saved_total` |
| Compression Ratio (median) | `histogram_quantile(0.5, headroom_compression_ratio_bucket)` |
| Request Latency (p99) | `histogram_quantile(0.99, headroom_latency_seconds_bucket)` |
| Cache Hit Rate | `headroom_cache_hits_total / (headroom_cache_hits_total + headroom_cache_misses_total)` |
diff --git a/docs/content/docs/proxy.mdx b/docs/content/docs/proxy.mdx
index 0eb856f24..4bd675c1a 100644
--- a/docs/content/docs/proxy.mdx
+++ b/docs/content/docs/proxy.mdx
@@ -151,11 +151,14 @@ curl http://localhost:8787/metrics
```
headroom_requests_total{mode="optimize"} 1234
headroom_tokens_saved_total 5678900
+headroom_persistent_savings_tokens_saved_total 5678900
headroom_compression_ratio_bucket{le="0.5"} 890
headroom_latency_seconds_bucket{le="0.01"} 800
headroom_cache_hits_total 456
```
+`headroom_tokens_saved_total` is the runtime counter for the current proxy process. Use `headroom_persistent_savings_tokens_saved_total` for durable lifetime savings that match `/stats.persistent_savings`.
+
### `POST /v1/messages`
Anthropic API format. The proxy compresses messages, forwards to Anthropic, and returns the response.
diff --git a/headroom/proxy/prometheus_metrics.py b/headroom/proxy/prometheus_metrics.py
index dca95ddb0..bf593d404 100644
--- a/headroom/proxy/prometheus_metrics.py
+++ b/headroom/proxy/prometheus_metrics.py
@@ -825,6 +825,7 @@ class PrometheusMetrics:
stage_timing_count_snapshot = dict(self.stage_timing_count)
stage_timing_max_snapshot = dict(self.stage_timing_max)
async with self._lock:
+ lifetime_savings = self.savings_tracker.snapshot()["lifetime"]
lines: list[str] = []
_append_metric(
lines,
@@ -896,6 +897,44 @@ class PrometheusMetrics:
help_text="Tokens saved by optimization",
value=self.tokens_saved_total,
)
+ _append_metric(
+ lines,
+ name="headroom_persistent_savings_requests_total",
+ metric_type="counter",
+ help_text="Durable lifetime requests recorded by the proxy savings tracker",
+ value=lifetime_savings["requests"],
+ )
+ _append_metric(
+ lines,
+ name="headroom_persistent_savings_tokens_saved_total",
+ metric_type="counter",
+ help_text="Durable lifetime input tokens saved by proxy compression",
+ value=lifetime_savings["tokens_saved"],
+ )
+ _append_metric(
+ lines,
+ name="headroom_persistent_savings_input_tokens_total",
+ metric_type="counter",
+ help_text="Durable lifetime input tokens recorded by the proxy savings tracker",
+ value=lifetime_savings["total_input_tokens"],
+ )
+ _append_metric(
+ lines,
+ name="headroom_persistent_savings_input_cost_usd_total",
+ metric_type="counter",
+ help_text="Durable lifetime input spend in USD estimated by the proxy savings tracker",
+ value=lifetime_savings["total_input_cost_usd"],
+ )
+ _append_metric(
+ lines,
+ name="headroom_persistent_savings_compression_savings_usd_total",
+ metric_type="counter",
+ help_text=(
+ "Durable lifetime compression savings in USD estimated by the "
+ "proxy savings tracker"
+ ),
+ value=lifetime_savings["compression_savings_usd"],
+ )
# NOTE: per-strategy compression breakdown is tracked
# internally on `self.compressions_by_strategy` and
# `self.tokens_saved_by_strategy` (populated by
diff --git a/tests/test_proxy_cache_ttl_metrics.py b/tests/test_proxy_cache_ttl_metrics.py
index c68fa8ad5..e603ddb5b 100644
--- a/tests/test_proxy_cache_ttl_metrics.py
+++ b/tests/test_proxy_cache_ttl_metrics.py
@@ -9,6 +9,7 @@ import pytest
from headroom.observability import reset_headroom_tracing, reset_otel_metrics
from headroom.proxy.cost import CostTracker, build_prefix_cache_stats
from headroom.proxy.prometheus_metrics import PrometheusMetrics
+from headroom.proxy.savings_tracker import SavingsTracker
def test_prometheus_metrics_tracks_observed_ttl_buckets() -> None:
@@ -79,8 +80,10 @@ def test_prefix_cache_stats_include_observed_ttl_mix() -> None:
assert stats["totals"]["observed_ttl_buckets"]["1h"]["tokens"] == 45
-def test_prometheus_metrics_export_includes_extended_fields() -> None:
- metrics = PrometheusMetrics()
+def test_prometheus_metrics_export_includes_extended_fields(tmp_path) -> None:
+ metrics = PrometheusMetrics(
+ savings_tracker=SavingsTracker(path=str(tmp_path / "proxy_savings.json"))
+ )
asyncio.run(
metrics.record_request(
@@ -105,6 +108,11 @@ def test_prometheus_metrics_export_includes_extended_fields() -> None:
exported = asyncio.run(metrics.export())
+ assert "headroom_requests_total 1" in exported
+ assert "headroom_tokens_saved_total 5" in exported
+ assert "headroom_persistent_savings_requests_total 1" in exported
+ assert "headroom_persistent_savings_tokens_saved_total 5" in exported
+ assert "headroom_persistent_savings_input_tokens_total 100" in exported
assert "headroom_latency_ms_count 1" in exported
assert 'headroom_transform_timing_ms_sum{transform="router"} 4.5' in exported
assert 'headroom_waste_signal_tokens_total{signal="json_bloat"} 7' in exported
@@ -113,6 +121,31 @@ def test_prometheus_metrics_export_includes_extended_fields() -> None:
assert "headroom_cache_bust_tokens_lost_total 11" in exported
+def test_prometheus_export_includes_persistent_savings_after_restart(tmp_path) -> None:
+ savings_path = tmp_path / "proxy_savings.json"
+ metrics = PrometheusMetrics(savings_tracker=SavingsTracker(path=str(savings_path)))
+
+ asyncio.run(
+ metrics.record_request(
+ provider="openai",
+ model="gpt-4o",
+ input_tokens=120,
+ output_tokens=20,
+ tokens_saved=40,
+ latency_ms=12.5,
+ )
+ )
+
+ reloaded = PrometheusMetrics(savings_tracker=SavingsTracker(path=str(savings_path)))
+ exported = asyncio.run(reloaded.export())
+
+ assert "headroom_tokens_saved_total 0" in exported
+ assert "headroom_requests_total 0" in exported
+ assert "headroom_persistent_savings_requests_total 1" in exported
+ assert "headroom_persistent_savings_tokens_saved_total 40" in exported
+ assert "headroom_persistent_savings_input_tokens_total 120" in exported
+
+
def test_streaming_parser_extracts_anthropic_ttl_bucket_usage() -> None:
from headroom.proxy.server import HeadroomProxy, ProxyConfig
diff --git a/tests/test_proxy_savings_history.py b/tests/test_proxy_savings_history.py
index 6bcc56bbd..f6b258171 100644
--- a/tests/test_proxy_savings_history.py
+++ b/tests/test_proxy_savings_history.py
@@ -866,6 +866,12 @@ def test_stats_history_persists_across_restarts_and_stats_stays_compatible(tmp_p
assert stats_data["persistent_savings"]["lifetime"]["tokens_saved"] == 40
assert stats_data["persistent_savings"]["storage_path"] == str(savings_path)
+ metrics = client.get("/metrics")
+ assert metrics.status_code == 200
+ assert "headroom_tokens_saved_total 40" in metrics.text
+ assert "headroom_persistent_savings_tokens_saved_total 40" in metrics.text
+ assert "headroom_persistent_savings_requests_total 1" in metrics.text
+
history = client.get("/stats-history")
assert history.status_code == 200
history_data = history.json()
@@ -907,6 +913,12 @@ def test_stats_history_persists_across_restarts_and_stats_stays_compatible(tmp_p
assert history.json()["lifetime"]["tokens_saved"] == 40
assert history.json()["display_session"]["requests"] == 1
+ metrics = client.get("/metrics")
+ assert metrics.status_code == 200
+ assert "headroom_tokens_saved_total 0" in metrics.text
+ assert "headroom_persistent_savings_tokens_saved_total 40" in metrics.text
+ assert "headroom_persistent_savings_requests_total 1" in metrics.text
+
_record_request(client, model="gpt-4o", tokens_saved=15)
updated = client.get("/stats-history").json()
@@ -922,6 +934,12 @@ def test_stats_history_persists_across_restarts_and_stats_stays_compatible(tmp_p
assert updated["series"]["daily"][0]["total_input_tokens_delta"] == 240
assert updated["series"]["daily"][0]["total_input_cost_usd_delta"] == pytest.approx(0.48)
+ metrics = client.get("/metrics")
+ assert metrics.status_code == 200
+ assert "headroom_tokens_saved_total 15" in metrics.text
+ assert "headroom_persistent_savings_tokens_saved_total 55" in metrics.text
+ assert "headroom_persistent_savings_requests_total 2" in metrics.text
+
full = client.get("/stats-history?history_mode=full").json()
assert full["history_summary"]["mode"] == "full"
assert full["history_summary"]["stored_points"] == 2