Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(proxy): quarantine compression while timed-out workers run (#2292)

Sebastian Schkudlara517bf992cfb0
diff --git a/headroom/proxy/prometheus_metrics.py b/headroom/proxy/prometheus_metrics.py
index 94f005fe3..97006376c 100644
--- a/headroom/proxy/prometheus_metrics.py
+++ b/headroom/proxy/prometheus_metrics.py
@@ -130,14 +130,18 @@ class PrometheusMetrics:
# counter proves whether the gate ever fires on live traffic.
self.kompress_size_gate_by_outcome: dict[str, int] = defaultdict(int)
- # These two counters are mutated from the compression thread-pool
- # worker (record_kompress_size_gate runs inside ContentRouter, which
- # the proxy runs in an executor), while export()/reset_runtime() read
- # and clear them from the event-loop thread. asyncio.Lock can't be
- # taken off-loop, so guard them with a plain threading.Lock, mirroring
- # _stage_timing_lock below. Without it a first-time key insert can race
- # an export iteration ("dictionary changed size during iteration") and
- # concurrent increments can be lost.
+ # Timeout-debt quarantine events. ``activated`` means a request timed
+ # out while its worker kept running; ``skipped`` means later work was
+ # rejected before executor admission behind that worker.
+ self.compression_quarantine_by_event: dict[str, int] = defaultdict(int)
+
+ # These counters are mutated from compression worker threads and the
+ # event-loop thread, while export()/reset_runtime() read and clear them.
+ # asyncio.Lock can't be taken off-loop, so guard them with a plain
+ # threading.Lock, mirroring _stage_timing_lock below. Without it a
+ # first-time key insert can race an export iteration ("dictionary
+ # changed size during iteration") and concurrent increments can be
+ # lost.
self._obs_counter_lock = threading.Lock()
# Codex WebSocket compression observability. These are intentionally
@@ -320,6 +324,7 @@ class PrometheusMetrics:
with self._obs_counter_lock:
self.compression_failed_by_reason.clear()
self.kompress_size_gate_by_outcome.clear()
+ self.compression_quarantine_by_event.clear()
self.codex_ws_units_total = 0
self.codex_ws_units_modified_total = 0
@@ -502,6 +507,18 @@ class PrometheusMetrics:
with self._obs_counter_lock:
self.kompress_size_gate_by_outcome[outcome or "within"] += 1
+ def record_compression_quarantine(self, event: str) -> None:
+ """Record one timeout-debt quarantine event.
+
+ ``event`` is ``"activated"`` when a timed-out worker is confirmed to
+ still be running, or ``"skipped"`` when new compression is rejected
+ before executor admission while that worker remains. Both worker and
+ event-loop threads call this method, so updates share the
+ observer-counter lock.
+ """
+ with self._obs_counter_lock:
+ self.compression_quarantine_by_event[event or "skipped"] += 1
+
def record_router_route_counts(self, counts: dict[str, int]) -> None:
"""Accumulate ContentRouter routing-category counts for a single
pass. The router emits a dict like ``{"user_msg": 12,
@@ -1167,6 +1184,7 @@ class PrometheusMetrics:
with self._obs_counter_lock:
compression_failed = dict(self.compression_failed_by_reason)
kompress_size_gate = dict(self.kompress_size_gate_by_outcome)
+ compression_quarantine = dict(self.compression_quarantine_by_event)
if compression_failed:
lines.extend(
@@ -1194,6 +1212,19 @@ class PrometheusMetrics:
)
lines.append("")
+ if compression_quarantine:
+ lines.extend(
+ [
+ "# HELP headroom_compression_quarantine_total Timeout-debt quarantine events by type",
+ "# TYPE headroom_compression_quarantine_total counter",
+ ]
+ )
+ for event, count in compression_quarantine.items():
+ lines.append(
+ f'headroom_compression_quarantine_total{{event="{_escape_label_value(event)}"}} {count}'
+ )
+ lines.append("")
+
lines.extend(
[
"# HELP headroom_requests_by_provider Requests by provider",
diff --git a/headroom/proxy/server.py b/headroom/proxy/server.py
index 58d2dbdac..abecfe245 100644
--- a/headroom/proxy/server.py
+++ b/headroom/proxy/server.py
@@ -486,6 +486,15 @@ class LoopHealthState(TypedDict):
last_known_failure: LoopFailureDetails | None
+class CompressionQuarantinedError(asyncio.TimeoutError):
+ """Compression was skipped while a timed-out worker was still running.
+
+ Inherit from ``asyncio.TimeoutError`` rather than the builtin directly so
+ Python 3.10 callers classify quarantine bypasses exactly like executor
+ timeouts. The two exception classes only became aliases in Python 3.11.
+ """
+
+
_MULTI_WORKER_CONFIG_ENV = "HEADROOM_PROXY_CONFIG_JSON"
# Env var that opts out of the Rust core deployment smoke test (Hotfix-A0).
@@ -1011,6 +1020,16 @@ class HeadroomProxy(
# Counter: threads that finished AFTER their asyncio future hit the
# timeout. Stuck-thread leak indicator.
self._compression_leaked_threads: int = 0
+ # Timeout-debt quarantine. Python cannot preempt a worker after its
+ # asyncio waiter times out, so accepting more compression while that
+ # worker is still running can multiply one slow call into a saturated
+ # executor. New work raises immediately until every known post-timeout
+ # worker has genuinely exited, then follows the caller's existing
+ # compression-failure policy.
+ self._compression_timed_out_in_flight: int = 0
+ self._compression_timed_out_in_flight_max: int = 0
+ self._compression_quarantine_activations: int = 0
+ self._compression_quarantine_skips: int = 0
self._compression_metrics_lock = threading.Lock()
# Backend for Anthropic API (direct, LiteLLM, or any-llm)
@@ -1240,10 +1259,14 @@ class HeadroomProxy(
worker keeps running to completion, ignored. We detect this by
marking the call timed out on the asyncio side and incrementing
``_compression_leaked_threads`` from the worker's ``finally``
- block after it eventually finishes. Jobs that time out before a
- worker starts are removed from the queued gauge instead. Operators
- can see leaked-thread rate and queue pressure climbing in
- ``/stats`` before the pool fills up.
+ block after it eventually finishes. While any such worker remains,
+ new calls raise :class:`CompressionQuarantinedError` immediately so
+ callers apply the existing compression-failure policy instead of
+ filling the rest of the pool with the same timeout debt. Jobs that are
+ successfully cancelled before a worker starts are removed from the
+ queued gauge and do not activate the quarantine. If cancellation races
+ with worker startup, the now-running job is tracked as timeout debt and
+ does activate it.
Args:
fn: A no-arg sync callable that runs the compression. Must not
@@ -1260,20 +1283,71 @@ class HeadroomProxy(
Raises:
``asyncio.TimeoutError`` if the callable doesn't return within
``timeout``. Any exception raised by ``fn`` propagates
- unchanged.
+ unchanged. :class:`CompressionQuarantinedError` (an
+ ``asyncio.TimeoutError`` subclass) if a prior timed-out worker is
+ still running.
"""
+ with self._compression_metrics_lock:
+ timed_out_in_flight = self._compression_timed_out_in_flight
+ if timed_out_in_flight > 0:
+ self._compression_quarantine_skips += 1
+
+ if timed_out_in_flight > 0:
+ self.metrics.record_compression_quarantine("skipped")
+ raise CompressionQuarantinedError(
+ f"compression quarantined: {timed_out_in_flight} timed-out worker(s) still running"
+ )
+
loop = asyncio.get_running_loop()
queued_at = time.monotonic()
- state = {"queued": True, "timed_out": False}
+ state = {
+ "queued": True,
+ "started": False,
+ "finished": False,
+ "timed_out": False,
+ "timeout_debt_recorded": False,
+ }
with self._compression_metrics_lock:
self._compression_queued += 1
if self._compression_queued > self._compression_queued_max:
self._compression_queued_max = self._compression_queued
+ def _record_timeout_debt_locked() -> bool:
+ """Record a still-running post-timeout worker; lock must be held.
+
+ Returns ``True`` only when this worker transitions the executor
+ from clear to quarantined.
+ """
+ if (
+ not state["timed_out"]
+ or not state["started"]
+ or state["finished"]
+ or state["timeout_debt_recorded"]
+ ):
+ return False
+ was_clear = self._compression_timed_out_in_flight == 0
+ self._compression_timed_out_in_flight += 1
+ self._compression_timed_out_in_flight_max = max(
+ self._compression_timed_out_in_flight_max,
+ self._compression_timed_out_in_flight,
+ )
+ state["timeout_debt_recorded"] = True
+ if was_clear:
+ self._compression_quarantine_activations += 1
+ return was_clear
+
+ def _announce_quarantine() -> None:
+ self.metrics.record_compression_quarantine("activated")
+ logger.warning(
+ "Compression worker exceeded its request deadline and is still running; "
+ "new compression is quarantined until timed-out workers exit"
+ )
+
def _wrapped(): # noqa: ANN202
started_at = time.monotonic()
queue_wait = started_at - queued_at
with self._compression_metrics_lock:
+ state["started"] = True
if state["queued"]:
self._compression_queued -= 1
state["queued"] = False
@@ -1283,17 +1357,29 @@ class HeadroomProxy(
self._compression_in_flight += 1
if self._compression_in_flight > self._compression_in_flight_max:
self._compression_in_flight_max = self._compression_in_flight
+ quarantine_activated = _record_timeout_debt_locked()
+ if quarantine_activated:
+ _announce_quarantine()
try:
return fn()
finally:
elapsed = time.monotonic() - started_at
with self._compression_metrics_lock:
+ state["finished"] = True
self._compression_in_flight -= 1
self._compression_run_seconds_total += elapsed
if elapsed > self._compression_run_seconds_max:
self._compression_run_seconds_max = elapsed
if state["timed_out"]:
self._compression_leaked_threads += 1
+ if state["timeout_debt_recorded"]:
+ self._compression_timed_out_in_flight -= 1
+ state["timeout_debt_recorded"] = False
+ quarantine_cleared = self._compression_timed_out_in_flight == 0
+ else:
+ quarantine_cleared = False
+ if quarantine_cleared:
+ logger.info("Compression quarantine cleared after all timed-out workers exited")
future = loop.run_in_executor(self._compression_executor, _wrapped)
try:
@@ -1305,6 +1391,9 @@ class HeadroomProxy(
self._compression_queued -= 1
state["queued"] = False
self._compression_queue_timeouts += 1
+ quarantine_activated = _record_timeout_debt_locked()
+ if quarantine_activated:
+ _announce_quarantine()
raise
async def _run_compression_background(self, fn): # noqa: ANN001, ANN201
@@ -2461,6 +2550,10 @@ def create_app(config: ProxyConfig | None = None) -> FastAPI:
_comp_run_total = proxy._compression_run_seconds_total
_comp_run_max = proxy._compression_run_seconds_max
_comp_leaked = proxy._compression_leaked_threads
+ _comp_timed_out_in_flight = proxy._compression_timed_out_in_flight
+ _comp_timed_out_in_flight_max = proxy._compression_timed_out_in_flight_max
+ _comp_quarantine_activations = proxy._compression_quarantine_activations
+ _comp_quarantine_skips = proxy._compression_quarantine_skips
return {
"anthropic_pre_upstream": {
"enabled": proxy.anthropic_pre_upstream_sem is not None,
@@ -2488,6 +2581,11 @@ def create_app(config: ProxyConfig | None = None) -> FastAPI:
"run_seconds_total": _comp_run_total,
"run_seconds_max": _comp_run_max,
"leaked_threads_total": _comp_leaked,
+ "quarantine_active": _comp_timed_out_in_flight > 0,
+ "timed_out_workers": _comp_timed_out_in_flight,
+ "timed_out_workers_max": _comp_timed_out_in_flight_max,
+ "quarantine_activations_total": _comp_quarantine_activations,
+ "quarantine_skips_total": _comp_quarantine_skips,
"source": ("auto" if config.compression_max_workers is None else "explicit"),
},
"websocket_sessions": {
diff --git a/tests/test_prometheus_obs_counters.py b/tests/test_prometheus_obs_counters.py
index 88a8f1287..258cb23e1 100644
--- a/tests/test_prometheus_obs_counters.py
+++ b/tests/test_prometheus_obs_counters.py
@@ -1,11 +1,13 @@
-"""Unit tests for the fail-open + kompress-size-gate observability counters.
+"""Unit tests for fail-open compression observability counters.
-Covers the two counters added to ``PrometheusMetrics``:
+Covers the related counters added to ``PrometheusMetrics``:
* ``headroom_compression_failed_total{reason}`` — recorded at the proxy's
optimization fail-open site, split into "timeout" vs "error".
* ``headroom_kompress_size_gate_total{outcome}`` — recorded by ContentRouter
via the observer hook, split into "exceeded" vs "within".
+* ``headroom_compression_quarantine_total{event}`` — records quarantine
+ activation and immediate executor skips while a timed-out worker remains.
Imports only the metrics module so the test stays free of heavy ML deps.
"""
@@ -50,6 +52,17 @@ def test_record_kompress_size_gate_buckets_by_outcome() -> None:
assert metrics.kompress_size_gate_by_outcome["within"] == 2
+def test_record_compression_quarantine_buckets_by_event() -> None:
+ metrics = PrometheusMetrics()
+
+ metrics.record_compression_quarantine("activated")
+ metrics.record_compression_quarantine("skipped")
+ metrics.record_compression_quarantine("skipped")
+
+ assert metrics.compression_quarantine_by_event["activated"] == 1
+ assert metrics.compression_quarantine_by_event["skipped"] == 2
+
+
@pytest.mark.asyncio
async def test_counters_exported_in_prometheus_text() -> None:
metrics = PrometheusMetrics()
@@ -58,6 +71,8 @@ async def test_counters_exported_in_prometheus_text() -> None:
metrics.record_compression_failed("error")
metrics.record_kompress_size_gate("exceeded")
metrics.record_kompress_size_gate("within")
+ metrics.record_compression_quarantine("activated")
+ metrics.record_compression_quarantine("skipped")
text = await metrics.export()
@@ -69,6 +84,10 @@ async def test_counters_exported_in_prometheus_text() -> None:
assert 'headroom_kompress_size_gate_total{outcome="exceeded"} 1' in text
assert 'headroom_kompress_size_gate_total{outcome="within"} 1' in text
+ assert "# TYPE headroom_compression_quarantine_total counter" in text
+ assert 'headroom_compression_quarantine_total{event="activated"} 1' in text
+ assert 'headroom_compression_quarantine_total{event="skipped"} 1' in text
+
@pytest.mark.asyncio
async def test_counters_absent_from_export_until_recorded() -> None:
@@ -80,19 +99,22 @@ async def test_counters_absent_from_export_until_recorded() -> None:
# matching the other labelled-counter blocks in export().
assert "headroom_compression_failed_total" not in text
assert "headroom_kompress_size_gate_total" not in text
+ assert "headroom_compression_quarantine_total" not in text
@pytest.mark.asyncio
-async def test_reset_runtime_clears_both_counters() -> None:
+async def test_reset_runtime_clears_observability_counters() -> None:
metrics = PrometheusMetrics()
metrics.record_compression_failed("timeout")
metrics.record_kompress_size_gate("exceeded")
+ metrics.record_compression_quarantine("activated")
await metrics.reset_runtime()
assert dict(metrics.compression_failed_by_reason) == {}
assert dict(metrics.kompress_size_gate_by_outcome) == {}
+ assert dict(metrics.compression_quarantine_by_event) == {}
@pytest.mark.asyncio
diff --git a/tests/test_proxy_compression_executor.py b/tests/test_proxy_compression_executor.py
index 5077d2c91..4a05f9b72 100644
--- a/tests/test_proxy_compression_executor.py
+++ b/tests/test_proxy_compression_executor.py
@@ -16,6 +16,9 @@ Locks the following invariants:
4. Jobs that time out while still queued do not leak the running gauge.
5. ``/stats runtime.compression_executor`` surfaces the gauges + counters so
operators can see leaked-thread rate and queue pressure.
+6. Once a timed-out worker is known to still be running, new compression work
+ raises an asyncio timeout immediately until that worker exits instead of
+ multiplying the timeout debt across the executor.
These tests also serve as documentation: anyone reading them sees that
"timeout fired" does not mean "compression was cancelled" — it means "we
@@ -206,6 +209,73 @@ def test_timeout_fires_and_leaked_thread_is_counted() -> None:
assert proxy._compression_in_flight == 0
+def test_timeout_quarantines_new_work_until_timed_out_worker_finishes() -> None:
+ """One post-timeout worker must not admit more compression work.
+
+ This is the production failure mode behind the executor cascade: the
+ asyncio waiter times out, but its thread continues running. Without a
+ quarantine, every subsequent request can occupy another worker and repeat
+ the same timeout until the pool is exhausted.
+ """
+ proxy = _make_proxy(compression_max_workers=2)
+ first_started = threading.Event()
+ release_first = threading.Event()
+ second_started = threading.Event()
+
+ def _timed_out_compression():
+ first_started.set()
+ release_first.wait(timeout=5.0)
+ return "late"
+
+ def _second_compression():
+ second_started.set()
+ return "second"
+
+ async def _drive():
+ with pytest.raises(asyncio.TimeoutError):
+ await proxy._run_compression_in_executor(_timed_out_compression, timeout=0.05)
+ assert first_started.is_set()
+
+ bypass_started = time.monotonic()
+ try:
+ # asyncio.TimeoutError is distinct from builtin TimeoutError on
+ # Python 3.10. The quarantine signal must follow the former so the
+ # existing handler failure policy classifies it as a timeout.
+ with pytest.raises(asyncio.TimeoutError, match="quarantin"):
+ await proxy._run_compression_in_executor(_second_compression, timeout=1.0)
+ bypass_elapsed = time.monotonic() - bypass_started
+ assert bypass_elapsed < 0.2
+ assert not second_started.is_set()
+
+ with proxy._compression_metrics_lock:
+ assert proxy._compression_timed_out_in_flight == 1
+ assert proxy._compression_quarantine_skips == 1
+ assert proxy._compression_quarantine_activations == 1
+ finally:
+ release_first.set()
+
+ for _ in range(100):
+ with proxy._compression_metrics_lock:
+ if proxy._compression_timed_out_in_flight == 0:
+ break
+ await asyncio.sleep(0.01)
+
+ with proxy._compression_metrics_lock:
+ assert proxy._compression_timed_out_in_flight == 0
+ assert proxy._compression_leaked_threads == 1
+
+ # Quarantine is self-clearing: normal compression resumes after the
+ # timed-out worker has genuinely left the executor.
+ assert (
+ await proxy._run_compression_in_executor(_second_compression, timeout=1.0) == "second"
+ )
+ return await proxy.metrics.export()
+
+ prometheus_text = asyncio.run(_drive())
+ assert 'headroom_compression_quarantine_total{event="activated"} 1' in prometheus_text
+ assert 'headroom_compression_quarantine_total{event="skipped"} 1' in prometheus_text
+
+
def test_timeout_before_worker_start_does_not_leak_in_flight() -> None:
"""If a queued job times out before a worker starts, queued accounting
is cleaned up without touching the running gauge.
@@ -256,6 +326,9 @@ def test_timeout_before_worker_start_does_not_leak_in_flight() -> None:
assert proxy._compression_queued == 0
assert proxy._compression_in_flight == 0
assert proxy._compression_leaked_threads == 0
+ assert proxy._compression_timed_out_in_flight == 0
+ assert proxy._compression_quarantine_activations == 0
+ assert proxy._compression_quarantine_skips == 0
def test_compression_executor_skip_signal_remains_visible() -> None:
@@ -353,6 +426,11 @@ def test_compression_executor_metrics_appear_in_runtime_payload() -> None:
assert ce["queue_wait_seconds_total"] == 0.0
assert ce["run_seconds_total"] == 0.0
assert ce["leaked_threads_total"] == 0
+ assert ce["quarantine_active"] is False
+ assert ce["timed_out_workers"] == 0
+ assert ce["timed_out_workers_max"] == 0
+ assert ce["quarantine_activations_total"] == 0
+ assert ce["quarantine_skips_total"] == 0
assert ce["source"] == "explicit"