Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
feat(metrics): export compression-failed and kompress size-gate counters (#1569)
Rocker Zhangd7283387ac23
diff --git a/headroom/proxy/handlers/anthropic.py b/headroom/proxy/handlers/anthropic.pyindex 4ee7b5958..149fc76b3 100644--- a/headroom/proxy/handlers/anthropic.py+++ b/headroom/proxy/handlers/anthropic.py@@ -1568,6 +1568,14 @@ class AnthropicHandlerMixin:logger.warning(f"[{request_id}] Optimization failed: {type(e).__name__}: {e}")# Flag compression failure for observability_compression_failed = True+ # Split timeout from other errors: a timeout means the+ # compression budget was too tight, not a code bug.+ reason = (+ "timeout"+ if isinstance(e, (asyncio.TimeoutError, TimeoutError))+ else "error"+ )+ self.metrics.record_compression_failed(reason)# Cache-safety (ALL modes): forward the previously-cached (compressed)# prefix byte-identical. The freeze path can emit the agent's ORIGINAL@@ -3649,6 +3657,12 @@ class AnthropicHandlerMixin:logger.warning(f"[{request_id}] Optimization failed for batch request '{custom_id}': {e}")+ # Same fail-open accounting as the single-message path: a+ # timeout means the budget was too tight, not a code bug.+ reason = (+ "timeout" if isinstance(e, (asyncio.TimeoutError, TimeoutError)) else "error"+ )+ self.metrics.record_compression_failed(reason)# Pass through unchanged on failurecompressed_requests.append(batch_req)total_optimized_tokens += original_tokensdiff --git a/headroom/proxy/prometheus_metrics.py b/headroom/proxy/prometheus_metrics.pyindex a137956c2..56e20e0ab 100644--- a/headroom/proxy/prometheus_metrics.py+++ b/headroom/proxy/prometheus_metrics.py@@ -117,6 +117,29 @@ class PrometheusMetrics:self.compressions_by_strategy: dict[str, int] = defaultdict(int)self.tokens_saved_by_strategy: dict[str, int] = defaultdict(int)+ # Fail-open compression failures, keyed by reason ("timeout",+ # "error"). The proxy fails open on any optimization error so the+ # request still succeeds; without this counter the failure is only+ # visible as a log line. Splitting timeout from other errors tells us+ # whether the compression budget is too tight vs. a real bug.+ self.compression_failed_by_reason: dict[str, int] = defaultdict(int)++ # Kompress size-gate outcomes, keyed by outcome ("within",+ # "exceeded"). The gate routes oversized blocks away from ML+ # compression (ContentRouter._kompress_max_tokens, #1171). This+ # counter proves whether the gate ever fires on live traffic.+ self.kompress_size_gate_by_outcome: dict[str, int] = defaultdict(int)++ # These two counters are mutated from the compression thread-pool+ # worker (record_kompress_size_gate runs inside ContentRouter, which+ # the proxy runs in an executor), while export()/reset_runtime() read+ # and clear them from the event-loop thread. asyncio.Lock can't be+ # taken off-loop, so guard them with a plain threading.Lock, mirroring+ # _stage_timing_lock below. Without it a first-time key insert can race+ # an export iteration ("dictionary changed size during iteration") and+ # concurrent increments can be lost.+ self._obs_counter_lock = threading.Lock()+# Codex WebSocket compression observability. These are intentionally# aggregate counters/sums, not per-unit storage, so /stats can answer# routing questions without growing with traffic volume.@@ -294,6 +317,9 @@ class PrometheusMetrics:self.compressions_by_strategy.clear()self.tokens_saved_by_strategy.clear()+ with self._obs_counter_lock:+ self.compression_failed_by_reason.clear()+ self.kompress_size_gate_by_outcome.clear()self.codex_ws_units_total = 0self.codex_ws_units_modified_total = 0@@ -451,6 +477,30 @@ class PrometheusMetrics:if saved > 0:self.tokens_saved_by_strategy[strategy] += saved+ def record_compression_failed(self, reason: str) -> None:+ """Record one fail-open compression failure, bucketed by ``reason``.++ Called from the optimization fail-open site (handlers/anthropic.py)+ with ``reason`` in ``{"timeout", "error"}``. Guarded by+ ``_obs_counter_lock`` so it stays consistent with the off-thread+ ``record_kompress_size_gate`` writer and the export/reset readers.+ """+ with self._obs_counter_lock:+ self.compression_failed_by_reason[reason or "error"] += 1++ def record_kompress_size_gate(self, outcome: str) -> None:+ """Record one kompress size-gate decision, bucketed by ``outcome``.++ Called from ContentRouter via the observer hook with ``outcome`` in+ ``{"within", "exceeded"}`` — "exceeded" when an eligible block is too+ large and routed off ML, "within" when it passes the gate. Proves+ whether the size gate (#1171) ever fires on live traffic. Runs on the+ compression executor thread, so the increment is guarded by+ ``_obs_counter_lock`` to stay safe against the export/reset readers.+ """+ with self._obs_counter_lock:+ self.kompress_size_gate_by_outcome[outcome or "within"] += 1+def record_router_route_counts(self, counts: dict[str, int]) -> None:"""Accumulate ContentRouter routing-category counts for a singlepass. The router emits a dict like ``{"user_msg": 12,@@ -1087,6 +1137,38 @@ class PrometheusMetrics:)lines.append("")+ # Snapshot the off-thread observer counters under their own lock,+ # then format outside it (see _obs_counter_lock).+ with self._obs_counter_lock:+ compression_failed = dict(self.compression_failed_by_reason)+ kompress_size_gate = dict(self.kompress_size_gate_by_outcome)++ if compression_failed:+ lines.extend(+ [+ "# HELP headroom_compression_failed_total Fail-open compression failures by reason",+ "# TYPE headroom_compression_failed_total counter",+ ]+ )+ for reason, count in compression_failed.items():+ lines.append(+ f'headroom_compression_failed_total{{reason="{_escape_label_value(reason)}"}} {count}'+ )+ lines.append("")++ if kompress_size_gate:+ lines.extend(+ [+ "# HELP headroom_kompress_size_gate_total Kompress size-gate decisions by outcome; within counts a gate pass, not whether ML compression then ran",+ "# TYPE headroom_kompress_size_gate_total counter",+ ]+ )+ for outcome, count in kompress_size_gate.items():+ lines.append(+ f'headroom_kompress_size_gate_total{{outcome="{_escape_label_value(outcome)}"}} {count}'+ )+ lines.append("")+lines.extend(["# HELP headroom_requests_by_provider Requests by provider",diff --git a/headroom/transforms/content_router.py b/headroom/transforms/content_router.pyindex 5a697446e..ab8e9e273 100644--- a/headroom/transforms/content_router.py+++ b/headroom/transforms/content_router.py@@ -1636,6 +1636,23 @@ class ContentRouter(Transform):except Exception as e: # pragma: no cover - defensivelogger.debug("CompressionObserver raised (non-fatal): %s", e)+ def _observe_kompress_size_gate(self, outcome: str) -> None:+ """Forward one kompress size-gate decision to the observer.++ ``outcome`` is "exceeded" when an eligible block trips the size+ ceiling and is routed off ML, "within" when it passes the gate.+ Goes through the same observer hook as ``_observe`` so the metric+ reaches the PrometheusMetrics singleton without ``content_router``+ importing ``headroom.proxy`` (no cycle). Defensive: a missing+ method or a buggy observer must not break the compression.+ """+ if self._observer is None:+ return+ try:+ self._observer.record_kompress_size_gate(outcome)+ except Exception as e: # pragma: no cover - defensive+ logger.debug("CompressionObserver raised (non-fatal): %s", e)+def _determine_strategy(self, content: str) -> CompressionStrategy:"""Determine the compression strategy from content analysis.@@ -2367,6 +2384,7 @@ class ContentRouter(Transform):# rather than ModernBERT, keeping the request path bounded.if self._kompress_max_tokens > 0 and len(text_to_compress) > self._kompress_max_tokens * 4:self._kompress_gate_fires += 1+ self._observe_kompress_size_gate("exceeded")logger.info("kompress size-gate fired: ~%d tok (>%d) routed off ML (fire #%d)",len(text_to_compress) // 4,@@ -2397,6 +2415,12 @@ class ContentRouter(Transform):out = restore_tags(out, protected)return out, _estimate_tokens(out)+ # Reached only when the gate is enabled and this eligible block is+ # under the ceiling — the counterpart "within" outcome to the+ # "exceeded" branch above. Lets /metrics prove the gate's hit rate.+ if self._kompress_max_tokens > 0:+ self._observe_kompress_size_gate("within")+# Primary: Kompress. On a cold cache the model is fetched once in the# background (ensure_background_load) instead of blocking this request# thread on a 274MB download that races the compression timeout anddiff --git a/tests/test_anthropic_pre_upstream_backpressure.py b/tests/test_anthropic_pre_upstream_backpressure.pyindex c2fc9db72..805bb9ec0 100644--- a/tests/test_anthropic_pre_upstream_backpressure.py+++ b/tests/test_anthropic_pre_upstream_backpressure.py@@ -74,6 +74,9 @@ class _DummyMetrics:async def record_failed(self, **kwargs) -> None:return None+ def record_compression_failed(self, reason: str) -> None:+ return None+class _ResponseStub:def __init__(self, status_code: int = 200) -> None:diff --git a/tests/test_prometheus_obs_counters.py b/tests/test_prometheus_obs_counters.pynew file mode 100644index 000000000..88a8f1287--- /dev/null+++ b/tests/test_prometheus_obs_counters.py@@ -0,0 +1,125 @@+"""Unit tests for the fail-open + kompress-size-gate observability counters.++Covers the two counters added to ``PrometheusMetrics``:++* ``headroom_compression_failed_total{reason}`` — recorded at the proxy's+ optimization fail-open site, split into "timeout" vs "error".+* ``headroom_kompress_size_gate_total{outcome}`` — recorded by ContentRouter+ via the observer hook, split into "exceeded" vs "within".++Imports only the metrics module so the test stays free of heavy ML deps.+"""++from __future__ import annotations++import asyncio+import threading++import pytest++from headroom.proxy.prometheus_metrics import PrometheusMetrics+++def test_record_compression_failed_buckets_by_reason() -> None:+ metrics = PrometheusMetrics()++ metrics.record_compression_failed("timeout")+ metrics.record_compression_failed("error")+ metrics.record_compression_failed("error")++ assert metrics.compression_failed_by_reason["timeout"] == 1+ assert metrics.compression_failed_by_reason["error"] == 2+++def test_record_compression_failed_empty_reason_defaults_to_error() -> None:+ metrics = PrometheusMetrics()++ metrics.record_compression_failed("")++ assert metrics.compression_failed_by_reason["error"] == 1+++def test_record_kompress_size_gate_buckets_by_outcome() -> None:+ metrics = PrometheusMetrics()++ metrics.record_kompress_size_gate("exceeded")+ metrics.record_kompress_size_gate("within")+ metrics.record_kompress_size_gate("within")++ assert metrics.kompress_size_gate_by_outcome["exceeded"] == 1+ assert metrics.kompress_size_gate_by_outcome["within"] == 2+++async def test_counters_exported_in_prometheus_text() -> None:+ metrics = PrometheusMetrics()++ metrics.record_compression_failed("timeout")+ metrics.record_compression_failed("error")+ metrics.record_kompress_size_gate("exceeded")+ metrics.record_kompress_size_gate("within")++ text = await metrics.export()++ assert "# TYPE headroom_compression_failed_total counter" in text+ assert 'headroom_compression_failed_total{reason="timeout"} 1' in text+ assert 'headroom_compression_failed_total{reason="error"} 1' in text++ assert "# TYPE headroom_kompress_size_gate_total counter" in text+ assert 'headroom_kompress_size_gate_total{outcome="exceeded"} 1' in text+ assert 'headroom_kompress_size_gate_total{outcome="within"} 1' in text+++async def test_counters_absent_from_export_until_recorded() -> None:+ metrics = PrometheusMetrics()++ text = await metrics.export()++ # Conditional emission: the families only appear once a sample exists,+ # matching the other labelled-counter blocks in export().+ assert "headroom_compression_failed_total" not in text+ assert "headroom_kompress_size_gate_total" not in text+++async def test_reset_runtime_clears_both_counters() -> None:+ metrics = PrometheusMetrics()++ metrics.record_compression_failed("timeout")+ metrics.record_kompress_size_gate("exceeded")++ await metrics.reset_runtime()++ assert dict(metrics.compression_failed_by_reason) == {}+ assert dict(metrics.kompress_size_gate_by_outcome) == {}+++async def test_gate_counter_is_thread_safe_under_concurrent_export() -> None:+ # record_kompress_size_gate runs on the compression executor thread while+ # export() reads from the event loop. Concurrent unguarded access would+ # lose increments or raise "dictionary changed size during iteration".+ metrics = PrometheusMetrics()+ n_threads, per_thread = 8, 4000+ errors: list[str] = []++ def hammer() -> None:+ for i in range(per_thread):+ metrics.record_kompress_size_gate("within" if i % 2 else "exceeded")++ threads = [threading.Thread(target=hammer) for _ in range(n_threads)]+ for t in threads:+ t.start()+ while any(t.is_alive() for t in threads):+ try:+ await metrics.export()+ except Exception as exc: # pragma: no cover - failure path+ errors.append(repr(exc))+ await asyncio.sleep(0)+ for t in threads:+ t.join()++ assert not errors, f"export() raced the writer: {errors[:3]}"+ totals = dict(metrics.kompress_size_gate_by_outcome)+ assert sum(totals.values()) == n_threads * per_thread