Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(proxy): expose persistent savings metrics (#1647)
Vinay Gupta5fe4e7b19530
diff --git a/docs/content/docs/metrics.mdx b/docs/content/docs/metrics.mdxindex 9d07b0d25..0318d9273 100644--- a/docs/content/docs/metrics.mdx+++ b/docs/content/docs/metrics.mdx@@ -72,6 +72,9 @@ headroom_requests_total{mode="optimize"} 1234# HELP headroom_tokens_saved_total Total tokens savedheadroom_tokens_saved_total 5678900+# HELP headroom_persistent_savings_tokens_saved_total Durable lifetime input tokens saved by proxy compression+headroom_persistent_savings_tokens_saved_total 5678900+# HELP headroom_compression_ratio Compression ratio histogramheadroom_compression_ratio_bucket{le="0.5"} 890headroom_compression_ratio_bucket{le="0.7"} 1100@@ -253,7 +256,8 @@ When the budget is exceeded, requests return a budget exceeded error, the `/stat| Metric | What It Tells You | Target ||--------|-------------------|--------|-| `tokens_saved_total` | Total cost savings | Higher is better |+| `headroom_tokens_saved_total` | Runtime tokens saved since this proxy process started | Higher is better |+| `headroom_persistent_savings_tokens_saved_total` | Durable lifetime tokens saved from `/stats.persistent_savings` | Higher is better || `compression_ratio_avg` | Efficiency | 0.7--0.9 typical || `cache_hit_rate` | Cache effectiveness | >20% is good || `latency_p99` | Performance impact | <10ms |@@ -265,7 +269,8 @@ Example Prometheus queries for a Grafana dashboard:| Panel | PromQL ||-------|--------|-| Tokens Saved | `headroom_tokens_saved_total` |+| Runtime Tokens Saved | `headroom_tokens_saved_total` |+| Lifetime Tokens Saved | `headroom_persistent_savings_tokens_saved_total` || Compression Ratio (median) | `histogram_quantile(0.5, headroom_compression_ratio_bucket)` || Request Latency (p99) | `histogram_quantile(0.99, headroom_latency_seconds_bucket)` || Cache Hit Rate | `headroom_cache_hits_total / (headroom_cache_hits_total + headroom_cache_misses_total)` |diff --git a/docs/content/docs/proxy.mdx b/docs/content/docs/proxy.mdxindex 0eb856f24..4bd675c1a 100644--- a/docs/content/docs/proxy.mdx+++ b/docs/content/docs/proxy.mdx@@ -151,11 +151,14 @@ curl http://localhost:8787/metrics```headroom_requests_total{mode="optimize"} 1234headroom_tokens_saved_total 5678900+headroom_persistent_savings_tokens_saved_total 5678900headroom_compression_ratio_bucket{le="0.5"} 890headroom_latency_seconds_bucket{le="0.01"} 800headroom_cache_hits_total 456```+`headroom_tokens_saved_total` is the runtime counter for the current proxy process. Use `headroom_persistent_savings_tokens_saved_total` for durable lifetime savings that match `/stats.persistent_savings`.+### `POST /v1/messages`Anthropic API format. The proxy compresses messages, forwards to Anthropic, and returns the response.diff --git a/headroom/proxy/prometheus_metrics.py b/headroom/proxy/prometheus_metrics.pyindex dca95ddb0..bf593d404 100644--- a/headroom/proxy/prometheus_metrics.py+++ b/headroom/proxy/prometheus_metrics.py@@ -825,6 +825,7 @@ class PrometheusMetrics:stage_timing_count_snapshot = dict(self.stage_timing_count)stage_timing_max_snapshot = dict(self.stage_timing_max)async with self._lock:+ lifetime_savings = self.savings_tracker.snapshot()["lifetime"]lines: list[str] = []_append_metric(lines,@@ -896,6 +897,44 @@ class PrometheusMetrics:help_text="Tokens saved by optimization",value=self.tokens_saved_total,)+ _append_metric(+ lines,+ name="headroom_persistent_savings_requests_total",+ metric_type="counter",+ help_text="Durable lifetime requests recorded by the proxy savings tracker",+ value=lifetime_savings["requests"],+ )+ _append_metric(+ lines,+ name="headroom_persistent_savings_tokens_saved_total",+ metric_type="counter",+ help_text="Durable lifetime input tokens saved by proxy compression",+ value=lifetime_savings["tokens_saved"],+ )+ _append_metric(+ lines,+ name="headroom_persistent_savings_input_tokens_total",+ metric_type="counter",+ help_text="Durable lifetime input tokens recorded by the proxy savings tracker",+ value=lifetime_savings["total_input_tokens"],+ )+ _append_metric(+ lines,+ name="headroom_persistent_savings_input_cost_usd_total",+ metric_type="counter",+ help_text="Durable lifetime input spend in USD estimated by the proxy savings tracker",+ value=lifetime_savings["total_input_cost_usd"],+ )+ _append_metric(+ lines,+ name="headroom_persistent_savings_compression_savings_usd_total",+ metric_type="counter",+ help_text=(+ "Durable lifetime compression savings in USD estimated by the "+ "proxy savings tracker"+ ),+ value=lifetime_savings["compression_savings_usd"],+ )# NOTE: per-strategy compression breakdown is tracked# internally on `self.compressions_by_strategy` and# `self.tokens_saved_by_strategy` (populated bydiff --git a/tests/test_proxy_cache_ttl_metrics.py b/tests/test_proxy_cache_ttl_metrics.pyindex c68fa8ad5..e603ddb5b 100644--- a/tests/test_proxy_cache_ttl_metrics.py+++ b/tests/test_proxy_cache_ttl_metrics.py@@ -9,6 +9,7 @@ import pytestfrom headroom.observability import reset_headroom_tracing, reset_otel_metricsfrom headroom.proxy.cost import CostTracker, build_prefix_cache_statsfrom headroom.proxy.prometheus_metrics import PrometheusMetrics+from headroom.proxy.savings_tracker import SavingsTrackerdef test_prometheus_metrics_tracks_observed_ttl_buckets() -> None:@@ -79,8 +80,10 @@ def test_prefix_cache_stats_include_observed_ttl_mix() -> None:assert stats["totals"]["observed_ttl_buckets"]["1h"]["tokens"] == 45-def test_prometheus_metrics_export_includes_extended_fields() -> None:- metrics = PrometheusMetrics()+def test_prometheus_metrics_export_includes_extended_fields(tmp_path) -> None:+ metrics = PrometheusMetrics(+ savings_tracker=SavingsTracker(path=str(tmp_path / "proxy_savings.json"))+ )asyncio.run(metrics.record_request(@@ -105,6 +108,11 @@ def test_prometheus_metrics_export_includes_extended_fields() -> None:exported = asyncio.run(metrics.export())+ assert "headroom_requests_total 1" in exported+ assert "headroom_tokens_saved_total 5" in exported+ assert "headroom_persistent_savings_requests_total 1" in exported+ assert "headroom_persistent_savings_tokens_saved_total 5" in exported+ assert "headroom_persistent_savings_input_tokens_total 100" in exportedassert "headroom_latency_ms_count 1" in exportedassert 'headroom_transform_timing_ms_sum{transform="router"} 4.5' in exportedassert 'headroom_waste_signal_tokens_total{signal="json_bloat"} 7' in exported@@ -113,6 +121,31 @@ def test_prometheus_metrics_export_includes_extended_fields() -> None:assert "headroom_cache_bust_tokens_lost_total 11" in exported+def test_prometheus_export_includes_persistent_savings_after_restart(tmp_path) -> None:+ savings_path = tmp_path / "proxy_savings.json"+ metrics = PrometheusMetrics(savings_tracker=SavingsTracker(path=str(savings_path)))++ asyncio.run(+ metrics.record_request(+ provider="openai",+ model="gpt-4o",+ input_tokens=120,+ output_tokens=20,+ tokens_saved=40,+ latency_ms=12.5,+ )+ )++ reloaded = PrometheusMetrics(savings_tracker=SavingsTracker(path=str(savings_path)))+ exported = asyncio.run(reloaded.export())++ assert "headroom_tokens_saved_total 0" in exported+ assert "headroom_requests_total 0" in exported+ assert "headroom_persistent_savings_requests_total 1" in exported+ assert "headroom_persistent_savings_tokens_saved_total 40" in exported+ assert "headroom_persistent_savings_input_tokens_total 120" in exported++def test_streaming_parser_extracts_anthropic_ttl_bucket_usage() -> None:from headroom.proxy.server import HeadroomProxy, ProxyConfigdiff --git a/tests/test_proxy_savings_history.py b/tests/test_proxy_savings_history.pyindex 6bcc56bbd..f6b258171 100644--- a/tests/test_proxy_savings_history.py+++ b/tests/test_proxy_savings_history.py@@ -866,6 +866,12 @@ def test_stats_history_persists_across_restarts_and_stats_stays_compatible(tmp_passert stats_data["persistent_savings"]["lifetime"]["tokens_saved"] == 40assert stats_data["persistent_savings"]["storage_path"] == str(savings_path)+ metrics = client.get("/metrics")+ assert metrics.status_code == 200+ assert "headroom_tokens_saved_total 40" in metrics.text+ assert "headroom_persistent_savings_tokens_saved_total 40" in metrics.text+ assert "headroom_persistent_savings_requests_total 1" in metrics.text+history = client.get("/stats-history")assert history.status_code == 200history_data = history.json()@@ -907,6 +913,12 @@ def test_stats_history_persists_across_restarts_and_stats_stays_compatible(tmp_passert history.json()["lifetime"]["tokens_saved"] == 40assert history.json()["display_session"]["requests"] == 1+ metrics = client.get("/metrics")+ assert metrics.status_code == 200+ assert "headroom_tokens_saved_total 0" in metrics.text+ assert "headroom_persistent_savings_tokens_saved_total 40" in metrics.text+ assert "headroom_persistent_savings_requests_total 1" in metrics.text+_record_request(client, model="gpt-4o", tokens_saved=15)updated = client.get("/stats-history").json()@@ -922,6 +934,12 @@ def test_stats_history_persists_across_restarts_and_stats_stays_compatible(tmp_passert updated["series"]["daily"][0]["total_input_tokens_delta"] == 240assert updated["series"]["daily"][0]["total_input_cost_usd_delta"] == pytest.approx(0.48)+ metrics = client.get("/metrics")+ assert metrics.status_code == 200+ assert "headroom_tokens_saved_total 15" in metrics.text+ assert "headroom_persistent_savings_tokens_saved_total 55" in metrics.text+ assert "headroom_persistent_savings_requests_total 2" in metrics.text+full = client.get("/stats-history?history_mode=full").json()assert full["history_summary"]["mode"] == "full"assert full["history_summary"]["stored_points"] == 2