Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

feat(metrics): export compression-failed and kompress size-gate counters (#1569)

Rocker Zhangd7283387ac23
diff --git a/headroom/proxy/handlers/anthropic.py b/headroom/proxy/handlers/anthropic.py
index 4ee7b5958..149fc76b3 100644
--- a/headroom/proxy/handlers/anthropic.py
+++ b/headroom/proxy/handlers/anthropic.py
@@ -1568,6 +1568,14 @@ class AnthropicHandlerMixin:
logger.warning(f"[{request_id}] Optimization failed: {type(e).__name__}: {e}")
# Flag compression failure for observability
_compression_failed = True
+ # Split timeout from other errors: a timeout means the
+ # compression budget was too tight, not a code bug.
+ reason = (
+ "timeout"
+ if isinstance(e, (asyncio.TimeoutError, TimeoutError))
+ else "error"
+ )
+ self.metrics.record_compression_failed(reason)
# Cache-safety (ALL modes): forward the previously-cached (compressed)
# prefix byte-identical. The freeze path can emit the agent's ORIGINAL
@@ -3649,6 +3657,12 @@ class AnthropicHandlerMixin:
logger.warning(
f"[{request_id}] Optimization failed for batch request '{custom_id}': {e}"
)
+ # Same fail-open accounting as the single-message path: a
+ # timeout means the budget was too tight, not a code bug.
+ reason = (
+ "timeout" if isinstance(e, (asyncio.TimeoutError, TimeoutError)) else "error"
+ )
+ self.metrics.record_compression_failed(reason)
# Pass through unchanged on failure
compressed_requests.append(batch_req)
total_optimized_tokens += original_tokens
diff --git a/headroom/proxy/prometheus_metrics.py b/headroom/proxy/prometheus_metrics.py
index a137956c2..56e20e0ab 100644
--- a/headroom/proxy/prometheus_metrics.py
+++ b/headroom/proxy/prometheus_metrics.py
@@ -117,6 +117,29 @@ class PrometheusMetrics:
self.compressions_by_strategy: dict[str, int] = defaultdict(int)
self.tokens_saved_by_strategy: dict[str, int] = defaultdict(int)
+ # Fail-open compression failures, keyed by reason ("timeout",
+ # "error"). The proxy fails open on any optimization error so the
+ # request still succeeds; without this counter the failure is only
+ # visible as a log line. Splitting timeout from other errors tells us
+ # whether the compression budget is too tight vs. a real bug.
+ self.compression_failed_by_reason: dict[str, int] = defaultdict(int)
+
+ # Kompress size-gate outcomes, keyed by outcome ("within",
+ # "exceeded"). The gate routes oversized blocks away from ML
+ # compression (ContentRouter._kompress_max_tokens, #1171). This
+ # counter proves whether the gate ever fires on live traffic.
+ self.kompress_size_gate_by_outcome: dict[str, int] = defaultdict(int)
+
+ # These two counters are mutated from the compression thread-pool
+ # worker (record_kompress_size_gate runs inside ContentRouter, which
+ # the proxy runs in an executor), while export()/reset_runtime() read
+ # and clear them from the event-loop thread. asyncio.Lock can't be
+ # taken off-loop, so guard them with a plain threading.Lock, mirroring
+ # _stage_timing_lock below. Without it a first-time key insert can race
+ # an export iteration ("dictionary changed size during iteration") and
+ # concurrent increments can be lost.
+ self._obs_counter_lock = threading.Lock()
+
# Codex WebSocket compression observability. These are intentionally
# aggregate counters/sums, not per-unit storage, so /stats can answer
# routing questions without growing with traffic volume.
@@ -294,6 +317,9 @@ class PrometheusMetrics:
self.compressions_by_strategy.clear()
self.tokens_saved_by_strategy.clear()
+ with self._obs_counter_lock:
+ self.compression_failed_by_reason.clear()
+ self.kompress_size_gate_by_outcome.clear()
self.codex_ws_units_total = 0
self.codex_ws_units_modified_total = 0
@@ -451,6 +477,30 @@ class PrometheusMetrics:
if saved > 0:
self.tokens_saved_by_strategy[strategy] += saved
+ def record_compression_failed(self, reason: str) -> None:
+ """Record one fail-open compression failure, bucketed by ``reason``.
+
+ Called from the optimization fail-open site (handlers/anthropic.py)
+ with ``reason`` in ``{"timeout", "error"}``. Guarded by
+ ``_obs_counter_lock`` so it stays consistent with the off-thread
+ ``record_kompress_size_gate`` writer and the export/reset readers.
+ """
+ with self._obs_counter_lock:
+ self.compression_failed_by_reason[reason or "error"] += 1
+
+ def record_kompress_size_gate(self, outcome: str) -> None:
+ """Record one kompress size-gate decision, bucketed by ``outcome``.
+
+ Called from ContentRouter via the observer hook with ``outcome`` in
+ ``{"within", "exceeded"}`` — "exceeded" when an eligible block is too
+ large and routed off ML, "within" when it passes the gate. Proves
+ whether the size gate (#1171) ever fires on live traffic. Runs on the
+ compression executor thread, so the increment is guarded by
+ ``_obs_counter_lock`` to stay safe against the export/reset readers.
+ """
+ with self._obs_counter_lock:
+ self.kompress_size_gate_by_outcome[outcome or "within"] += 1
+
def record_router_route_counts(self, counts: dict[str, int]) -> None:
"""Accumulate ContentRouter routing-category counts for a single
pass. The router emits a dict like ``{"user_msg": 12,
@@ -1087,6 +1137,38 @@ class PrometheusMetrics:
)
lines.append("")
+ # Snapshot the off-thread observer counters under their own lock,
+ # then format outside it (see _obs_counter_lock).
+ with self._obs_counter_lock:
+ compression_failed = dict(self.compression_failed_by_reason)
+ kompress_size_gate = dict(self.kompress_size_gate_by_outcome)
+
+ if compression_failed:
+ lines.extend(
+ [
+ "# HELP headroom_compression_failed_total Fail-open compression failures by reason",
+ "# TYPE headroom_compression_failed_total counter",
+ ]
+ )
+ for reason, count in compression_failed.items():
+ lines.append(
+ f'headroom_compression_failed_total{{reason="{_escape_label_value(reason)}"}} {count}'
+ )
+ lines.append("")
+
+ if kompress_size_gate:
+ lines.extend(
+ [
+ "# HELP headroom_kompress_size_gate_total Kompress size-gate decisions by outcome; within counts a gate pass, not whether ML compression then ran",
+ "# TYPE headroom_kompress_size_gate_total counter",
+ ]
+ )
+ for outcome, count in kompress_size_gate.items():
+ lines.append(
+ f'headroom_kompress_size_gate_total{{outcome="{_escape_label_value(outcome)}"}} {count}'
+ )
+ lines.append("")
+
lines.extend(
[
"# HELP headroom_requests_by_provider Requests by provider",
diff --git a/headroom/transforms/content_router.py b/headroom/transforms/content_router.py
index 5a697446e..ab8e9e273 100644
--- a/headroom/transforms/content_router.py
+++ b/headroom/transforms/content_router.py
@@ -1636,6 +1636,23 @@ class ContentRouter(Transform):
except Exception as e: # pragma: no cover - defensive
logger.debug("CompressionObserver raised (non-fatal): %s", e)
+ def _observe_kompress_size_gate(self, outcome: str) -> None:
+ """Forward one kompress size-gate decision to the observer.
+
+ ``outcome`` is "exceeded" when an eligible block trips the size
+ ceiling and is routed off ML, "within" when it passes the gate.
+ Goes through the same observer hook as ``_observe`` so the metric
+ reaches the PrometheusMetrics singleton without ``content_router``
+ importing ``headroom.proxy`` (no cycle). Defensive: a missing
+ method or a buggy observer must not break the compression.
+ """
+ if self._observer is None:
+ return
+ try:
+ self._observer.record_kompress_size_gate(outcome)
+ except Exception as e: # pragma: no cover - defensive
+ logger.debug("CompressionObserver raised (non-fatal): %s", e)
+
def _determine_strategy(self, content: str) -> CompressionStrategy:
"""Determine the compression strategy from content analysis.
@@ -2367,6 +2384,7 @@ class ContentRouter(Transform):
# rather than ModernBERT, keeping the request path bounded.
if self._kompress_max_tokens > 0 and len(text_to_compress) > self._kompress_max_tokens * 4:
self._kompress_gate_fires += 1
+ self._observe_kompress_size_gate("exceeded")
logger.info(
"kompress size-gate fired: ~%d tok (>%d) routed off ML (fire #%d)",
len(text_to_compress) // 4,
@@ -2397,6 +2415,12 @@ class ContentRouter(Transform):
out = restore_tags(out, protected)
return out, _estimate_tokens(out)
+ # Reached only when the gate is enabled and this eligible block is
+ # under the ceiling — the counterpart "within" outcome to the
+ # "exceeded" branch above. Lets /metrics prove the gate's hit rate.
+ if self._kompress_max_tokens > 0:
+ self._observe_kompress_size_gate("within")
+
# Primary: Kompress. On a cold cache the model is fetched once in the
# background (ensure_background_load) instead of blocking this request
# thread on a 274MB download that races the compression timeout and
diff --git a/tests/test_anthropic_pre_upstream_backpressure.py b/tests/test_anthropic_pre_upstream_backpressure.py
index c2fc9db72..805bb9ec0 100644
--- a/tests/test_anthropic_pre_upstream_backpressure.py
+++ b/tests/test_anthropic_pre_upstream_backpressure.py
@@ -74,6 +74,9 @@ class _DummyMetrics:
async def record_failed(self, **kwargs) -> None:
return None
+ def record_compression_failed(self, reason: str) -> None:
+ return None
+
class _ResponseStub:
def __init__(self, status_code: int = 200) -> None:
diff --git a/tests/test_prometheus_obs_counters.py b/tests/test_prometheus_obs_counters.py
new file mode 100644
index 000000000..88a8f1287
--- /dev/null
+++ b/tests/test_prometheus_obs_counters.py
@@ -0,0 +1,125 @@
+"""Unit tests for the fail-open + kompress-size-gate observability counters.
+
+Covers the two counters added to ``PrometheusMetrics``:
+
+* ``headroom_compression_failed_total{reason}`` — recorded at the proxy's
+ optimization fail-open site, split into "timeout" vs "error".
+* ``headroom_kompress_size_gate_total{outcome}`` — recorded by ContentRouter
+ via the observer hook, split into "exceeded" vs "within".
+
+Imports only the metrics module so the test stays free of heavy ML deps.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import threading
+
+import pytest
+
+from headroom.proxy.prometheus_metrics import PrometheusMetrics
+
+
+def test_record_compression_failed_buckets_by_reason() -> None:
+ metrics = PrometheusMetrics()
+
+ metrics.record_compression_failed("timeout")
+ metrics.record_compression_failed("error")
+ metrics.record_compression_failed("error")
+
+ assert metrics.compression_failed_by_reason["timeout"] == 1
+ assert metrics.compression_failed_by_reason["error"] == 2
+
+
+def test_record_compression_failed_empty_reason_defaults_to_error() -> None:
+ metrics = PrometheusMetrics()
+
+ metrics.record_compression_failed("")
+
+ assert metrics.compression_failed_by_reason["error"] == 1
+
+
+def test_record_kompress_size_gate_buckets_by_outcome() -> None:
+ metrics = PrometheusMetrics()
+
+ metrics.record_kompress_size_gate("exceeded")
+ metrics.record_kompress_size_gate("within")
+ metrics.record_kompress_size_gate("within")
+
+ assert metrics.kompress_size_gate_by_outcome["exceeded"] == 1
+ assert metrics.kompress_size_gate_by_outcome["within"] == 2
+
+
+async def test_counters_exported_in_prometheus_text() -> None:
+ metrics = PrometheusMetrics()
+
+ metrics.record_compression_failed("timeout")
+ metrics.record_compression_failed("error")
+ metrics.record_kompress_size_gate("exceeded")
+ metrics.record_kompress_size_gate("within")
+
+ text = await metrics.export()
+
+ assert "# TYPE headroom_compression_failed_total counter" in text
+ assert 'headroom_compression_failed_total{reason="timeout"} 1' in text
+ assert 'headroom_compression_failed_total{reason="error"} 1' in text
+
+ assert "# TYPE headroom_kompress_size_gate_total counter" in text
+ assert 'headroom_kompress_size_gate_total{outcome="exceeded"} 1' in text
+ assert 'headroom_kompress_size_gate_total{outcome="within"} 1' in text
+
+
+async def test_counters_absent_from_export_until_recorded() -> None:
+ metrics = PrometheusMetrics()
+
+ text = await metrics.export()
+
+ # Conditional emission: the families only appear once a sample exists,
+ # matching the other labelled-counter blocks in export().
+ assert "headroom_compression_failed_total" not in text
+ assert "headroom_kompress_size_gate_total" not in text
+
+
+async def test_reset_runtime_clears_both_counters() -> None:
+ metrics = PrometheusMetrics()
+
+ metrics.record_compression_failed("timeout")
+ metrics.record_kompress_size_gate("exceeded")
+
+ await metrics.reset_runtime()
+
+ assert dict(metrics.compression_failed_by_reason) == {}
+ assert dict(metrics.kompress_size_gate_by_outcome) == {}
+
+
+async def test_gate_counter_is_thread_safe_under_concurrent_export() -> None:
+ # record_kompress_size_gate runs on the compression executor thread while
+ # export() reads from the event loop. Concurrent unguarded access would
+ # lose increments or raise "dictionary changed size during iteration".
+ metrics = PrometheusMetrics()
+ n_threads, per_thread = 8, 4000
+ errors: list[str] = []
+
+ def hammer() -> None:
+ for i in range(per_thread):
+ metrics.record_kompress_size_gate("within" if i % 2 else "exceeded")
+
+ threads = [threading.Thread(target=hammer) for _ in range(n_threads)]
+ for t in threads:
+ t.start()
+ while any(t.is_alive() for t in threads):
+ try:
+ await metrics.export()
+ except Exception as exc: # pragma: no cover - failure path
+ errors.append(repr(exc))
+ await asyncio.sleep(0)
+ for t in threads:
+ t.join()
+
+ assert not errors, f"export() raced the writer: {errors[:3]}"
+ totals = dict(metrics.kompress_size_gate_by_outcome)
+ assert sum(totals.values()) == n_threads * per_thread