Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(proxy): quarantine compression while timed-out workers run (#2292)
Sebastian Schkudlara517bf992cfb0
diff --git a/headroom/proxy/prometheus_metrics.py b/headroom/proxy/prometheus_metrics.pyindex 94f005fe3..97006376c 100644--- a/headroom/proxy/prometheus_metrics.py+++ b/headroom/proxy/prometheus_metrics.py@@ -130,14 +130,18 @@ class PrometheusMetrics:# counter proves whether the gate ever fires on live traffic.self.kompress_size_gate_by_outcome: dict[str, int] = defaultdict(int)- # These two counters are mutated from the compression thread-pool- # worker (record_kompress_size_gate runs inside ContentRouter, which- # the proxy runs in an executor), while export()/reset_runtime() read- # and clear them from the event-loop thread. asyncio.Lock can't be- # taken off-loop, so guard them with a plain threading.Lock, mirroring- # _stage_timing_lock below. Without it a first-time key insert can race- # an export iteration ("dictionary changed size during iteration") and- # concurrent increments can be lost.+ # Timeout-debt quarantine events. ``activated`` means a request timed+ # out while its worker kept running; ``skipped`` means later work was+ # rejected before executor admission behind that worker.+ self.compression_quarantine_by_event: dict[str, int] = defaultdict(int)++ # These counters are mutated from compression worker threads and the+ # event-loop thread, while export()/reset_runtime() read and clear them.+ # asyncio.Lock can't be taken off-loop, so guard them with a plain+ # threading.Lock, mirroring _stage_timing_lock below. Without it a+ # first-time key insert can race an export iteration ("dictionary+ # changed size during iteration") and concurrent increments can be+ # lost.self._obs_counter_lock = threading.Lock()# Codex WebSocket compression observability. These are intentionally@@ -320,6 +324,7 @@ class PrometheusMetrics:with self._obs_counter_lock:self.compression_failed_by_reason.clear()self.kompress_size_gate_by_outcome.clear()+ self.compression_quarantine_by_event.clear()self.codex_ws_units_total = 0self.codex_ws_units_modified_total = 0@@ -502,6 +507,18 @@ class PrometheusMetrics:with self._obs_counter_lock:self.kompress_size_gate_by_outcome[outcome or "within"] += 1+ def record_compression_quarantine(self, event: str) -> None:+ """Record one timeout-debt quarantine event.++ ``event`` is ``"activated"`` when a timed-out worker is confirmed to+ still be running, or ``"skipped"`` when new compression is rejected+ before executor admission while that worker remains. Both worker and+ event-loop threads call this method, so updates share the+ observer-counter lock.+ """+ with self._obs_counter_lock:+ self.compression_quarantine_by_event[event or "skipped"] += 1+def record_router_route_counts(self, counts: dict[str, int]) -> None:"""Accumulate ContentRouter routing-category counts for a singlepass. The router emits a dict like ``{"user_msg": 12,@@ -1167,6 +1184,7 @@ class PrometheusMetrics:with self._obs_counter_lock:compression_failed = dict(self.compression_failed_by_reason)kompress_size_gate = dict(self.kompress_size_gate_by_outcome)+ compression_quarantine = dict(self.compression_quarantine_by_event)if compression_failed:lines.extend(@@ -1194,6 +1212,19 @@ class PrometheusMetrics:)lines.append("")+ if compression_quarantine:+ lines.extend(+ [+ "# HELP headroom_compression_quarantine_total Timeout-debt quarantine events by type",+ "# TYPE headroom_compression_quarantine_total counter",+ ]+ )+ for event, count in compression_quarantine.items():+ lines.append(+ f'headroom_compression_quarantine_total{{event="{_escape_label_value(event)}"}} {count}'+ )+ lines.append("")+lines.extend(["# HELP headroom_requests_by_provider Requests by provider",diff --git a/headroom/proxy/server.py b/headroom/proxy/server.pyindex 58d2dbdac..abecfe245 100644--- a/headroom/proxy/server.py+++ b/headroom/proxy/server.py@@ -486,6 +486,15 @@ class LoopHealthState(TypedDict):last_known_failure: LoopFailureDetails | None+class CompressionQuarantinedError(asyncio.TimeoutError):+ """Compression was skipped while a timed-out worker was still running.++ Inherit from ``asyncio.TimeoutError`` rather than the builtin directly so+ Python 3.10 callers classify quarantine bypasses exactly like executor+ timeouts. The two exception classes only became aliases in Python 3.11.+ """++_MULTI_WORKER_CONFIG_ENV = "HEADROOM_PROXY_CONFIG_JSON"# Env var that opts out of the Rust core deployment smoke test (Hotfix-A0).@@ -1011,6 +1020,16 @@ class HeadroomProxy(# Counter: threads that finished AFTER their asyncio future hit the# timeout. Stuck-thread leak indicator.self._compression_leaked_threads: int = 0+ # Timeout-debt quarantine. Python cannot preempt a worker after its+ # asyncio waiter times out, so accepting more compression while that+ # worker is still running can multiply one slow call into a saturated+ # executor. New work raises immediately until every known post-timeout+ # worker has genuinely exited, then follows the caller's existing+ # compression-failure policy.+ self._compression_timed_out_in_flight: int = 0+ self._compression_timed_out_in_flight_max: int = 0+ self._compression_quarantine_activations: int = 0+ self._compression_quarantine_skips: int = 0self._compression_metrics_lock = threading.Lock()# Backend for Anthropic API (direct, LiteLLM, or any-llm)@@ -1240,10 +1259,14 @@ class HeadroomProxy(worker keeps running to completion, ignored. We detect this bymarking the call timed out on the asyncio side and incrementing``_compression_leaked_threads`` from the worker's ``finally``- block after it eventually finishes. Jobs that time out before a- worker starts are removed from the queued gauge instead. Operators- can see leaked-thread rate and queue pressure climbing in- ``/stats`` before the pool fills up.+ block after it eventually finishes. While any such worker remains,+ new calls raise :class:`CompressionQuarantinedError` immediately so+ callers apply the existing compression-failure policy instead of+ filling the rest of the pool with the same timeout debt. Jobs that are+ successfully cancelled before a worker starts are removed from the+ queued gauge and do not activate the quarantine. If cancellation races+ with worker startup, the now-running job is tracked as timeout debt and+ does activate it.Args:fn: A no-arg sync callable that runs the compression. Must not@@ -1260,20 +1283,71 @@ class HeadroomProxy(Raises:``asyncio.TimeoutError`` if the callable doesn't return within``timeout``. Any exception raised by ``fn`` propagates- unchanged.+ unchanged. :class:`CompressionQuarantinedError` (an+ ``asyncio.TimeoutError`` subclass) if a prior timed-out worker is+ still running."""+ with self._compression_metrics_lock:+ timed_out_in_flight = self._compression_timed_out_in_flight+ if timed_out_in_flight > 0:+ self._compression_quarantine_skips += 1++ if timed_out_in_flight > 0:+ self.metrics.record_compression_quarantine("skipped")+ raise CompressionQuarantinedError(+ f"compression quarantined: {timed_out_in_flight} timed-out worker(s) still running"+ )+loop = asyncio.get_running_loop()queued_at = time.monotonic()- state = {"queued": True, "timed_out": False}+ state = {+ "queued": True,+ "started": False,+ "finished": False,+ "timed_out": False,+ "timeout_debt_recorded": False,+ }with self._compression_metrics_lock:self._compression_queued += 1if self._compression_queued > self._compression_queued_max:self._compression_queued_max = self._compression_queued+ def _record_timeout_debt_locked() -> bool:+ """Record a still-running post-timeout worker; lock must be held.++ Returns ``True`` only when this worker transitions the executor+ from clear to quarantined.+ """+ if (+ not state["timed_out"]+ or not state["started"]+ or state["finished"]+ or state["timeout_debt_recorded"]+ ):+ return False+ was_clear = self._compression_timed_out_in_flight == 0+ self._compression_timed_out_in_flight += 1+ self._compression_timed_out_in_flight_max = max(+ self._compression_timed_out_in_flight_max,+ self._compression_timed_out_in_flight,+ )+ state["timeout_debt_recorded"] = True+ if was_clear:+ self._compression_quarantine_activations += 1+ return was_clear++ def _announce_quarantine() -> None:+ self.metrics.record_compression_quarantine("activated")+ logger.warning(+ "Compression worker exceeded its request deadline and is still running; "+ "new compression is quarantined until timed-out workers exit"+ )+def _wrapped(): # noqa: ANN202started_at = time.monotonic()queue_wait = started_at - queued_atwith self._compression_metrics_lock:+ state["started"] = Trueif state["queued"]:self._compression_queued -= 1state["queued"] = False@@ -1283,17 +1357,29 @@ class HeadroomProxy(self._compression_in_flight += 1if self._compression_in_flight > self._compression_in_flight_max:self._compression_in_flight_max = self._compression_in_flight+ quarantine_activated = _record_timeout_debt_locked()+ if quarantine_activated:+ _announce_quarantine()try:return fn()finally:elapsed = time.monotonic() - started_atwith self._compression_metrics_lock:+ state["finished"] = Trueself._compression_in_flight -= 1self._compression_run_seconds_total += elapsedif elapsed > self._compression_run_seconds_max:self._compression_run_seconds_max = elapsedif state["timed_out"]:self._compression_leaked_threads += 1+ if state["timeout_debt_recorded"]:+ self._compression_timed_out_in_flight -= 1+ state["timeout_debt_recorded"] = False+ quarantine_cleared = self._compression_timed_out_in_flight == 0+ else:+ quarantine_cleared = False+ if quarantine_cleared:+ logger.info("Compression quarantine cleared after all timed-out workers exited")future = loop.run_in_executor(self._compression_executor, _wrapped)try:@@ -1305,6 +1391,9 @@ class HeadroomProxy(self._compression_queued -= 1state["queued"] = Falseself._compression_queue_timeouts += 1+ quarantine_activated = _record_timeout_debt_locked()+ if quarantine_activated:+ _announce_quarantine()raiseasync def _run_compression_background(self, fn): # noqa: ANN001, ANN201@@ -2461,6 +2550,10 @@ def create_app(config: ProxyConfig | None = None) -> FastAPI:_comp_run_total = proxy._compression_run_seconds_total_comp_run_max = proxy._compression_run_seconds_max_comp_leaked = proxy._compression_leaked_threads+ _comp_timed_out_in_flight = proxy._compression_timed_out_in_flight+ _comp_timed_out_in_flight_max = proxy._compression_timed_out_in_flight_max+ _comp_quarantine_activations = proxy._compression_quarantine_activations+ _comp_quarantine_skips = proxy._compression_quarantine_skipsreturn {"anthropic_pre_upstream": {"enabled": proxy.anthropic_pre_upstream_sem is not None,@@ -2488,6 +2581,11 @@ def create_app(config: ProxyConfig | None = None) -> FastAPI:"run_seconds_total": _comp_run_total,"run_seconds_max": _comp_run_max,"leaked_threads_total": _comp_leaked,+ "quarantine_active": _comp_timed_out_in_flight > 0,+ "timed_out_workers": _comp_timed_out_in_flight,+ "timed_out_workers_max": _comp_timed_out_in_flight_max,+ "quarantine_activations_total": _comp_quarantine_activations,+ "quarantine_skips_total": _comp_quarantine_skips,"source": ("auto" if config.compression_max_workers is None else "explicit"),},"websocket_sessions": {diff --git a/tests/test_prometheus_obs_counters.py b/tests/test_prometheus_obs_counters.pyindex 88a8f1287..258cb23e1 100644--- a/tests/test_prometheus_obs_counters.py+++ b/tests/test_prometheus_obs_counters.py@@ -1,11 +1,13 @@-"""Unit tests for the fail-open + kompress-size-gate observability counters.+"""Unit tests for fail-open compression observability counters.-Covers the two counters added to ``PrometheusMetrics``:+Covers the related counters added to ``PrometheusMetrics``:* ``headroom_compression_failed_total{reason}`` — recorded at the proxy'soptimization fail-open site, split into "timeout" vs "error".* ``headroom_kompress_size_gate_total{outcome}`` — recorded by ContentRoutervia the observer hook, split into "exceeded" vs "within".+* ``headroom_compression_quarantine_total{event}`` — records quarantine+ activation and immediate executor skips while a timed-out worker remains.Imports only the metrics module so the test stays free of heavy ML deps."""@@ -50,6 +52,17 @@ def test_record_kompress_size_gate_buckets_by_outcome() -> None:assert metrics.kompress_size_gate_by_outcome["within"] == 2+def test_record_compression_quarantine_buckets_by_event() -> None:+ metrics = PrometheusMetrics()++ metrics.record_compression_quarantine("activated")+ metrics.record_compression_quarantine("skipped")+ metrics.record_compression_quarantine("skipped")++ assert metrics.compression_quarantine_by_event["activated"] == 1+ assert metrics.compression_quarantine_by_event["skipped"] == 2++@pytest.mark.asyncioasync def test_counters_exported_in_prometheus_text() -> None:metrics = PrometheusMetrics()@@ -58,6 +71,8 @@ async def test_counters_exported_in_prometheus_text() -> None:metrics.record_compression_failed("error")metrics.record_kompress_size_gate("exceeded")metrics.record_kompress_size_gate("within")+ metrics.record_compression_quarantine("activated")+ metrics.record_compression_quarantine("skipped")text = await metrics.export()@@ -69,6 +84,10 @@ async def test_counters_exported_in_prometheus_text() -> None:assert 'headroom_kompress_size_gate_total{outcome="exceeded"} 1' in textassert 'headroom_kompress_size_gate_total{outcome="within"} 1' in text+ assert "# TYPE headroom_compression_quarantine_total counter" in text+ assert 'headroom_compression_quarantine_total{event="activated"} 1' in text+ assert 'headroom_compression_quarantine_total{event="skipped"} 1' in text+@pytest.mark.asyncioasync def test_counters_absent_from_export_until_recorded() -> None:@@ -80,19 +99,22 @@ async def test_counters_absent_from_export_until_recorded() -> None:# matching the other labelled-counter blocks in export().assert "headroom_compression_failed_total" not in textassert "headroom_kompress_size_gate_total" not in text+ assert "headroom_compression_quarantine_total" not in text@pytest.mark.asyncio-async def test_reset_runtime_clears_both_counters() -> None:+async def test_reset_runtime_clears_observability_counters() -> None:metrics = PrometheusMetrics()metrics.record_compression_failed("timeout")metrics.record_kompress_size_gate("exceeded")+ metrics.record_compression_quarantine("activated")await metrics.reset_runtime()assert dict(metrics.compression_failed_by_reason) == {}assert dict(metrics.kompress_size_gate_by_outcome) == {}+ assert dict(metrics.compression_quarantine_by_event) == {}@pytest.mark.asynciodiff --git a/tests/test_proxy_compression_executor.py b/tests/test_proxy_compression_executor.pyindex 5077d2c91..4a05f9b72 100644--- a/tests/test_proxy_compression_executor.py+++ b/tests/test_proxy_compression_executor.py@@ -16,6 +16,9 @@ Locks the following invariants:4. Jobs that time out while still queued do not leak the running gauge.5. ``/stats runtime.compression_executor`` surfaces the gauges + counters sooperators can see leaked-thread rate and queue pressure.+6. Once a timed-out worker is known to still be running, new compression work+ raises an asyncio timeout immediately until that worker exits instead of+ multiplying the timeout debt across the executor.These tests also serve as documentation: anyone reading them sees that"timeout fired" does not mean "compression was cancelled" — it means "we@@ -206,6 +209,73 @@ def test_timeout_fires_and_leaked_thread_is_counted() -> None:assert proxy._compression_in_flight == 0+def test_timeout_quarantines_new_work_until_timed_out_worker_finishes() -> None:+ """One post-timeout worker must not admit more compression work.++ This is the production failure mode behind the executor cascade: the+ asyncio waiter times out, but its thread continues running. Without a+ quarantine, every subsequent request can occupy another worker and repeat+ the same timeout until the pool is exhausted.+ """+ proxy = _make_proxy(compression_max_workers=2)+ first_started = threading.Event()+ release_first = threading.Event()+ second_started = threading.Event()++ def _timed_out_compression():+ first_started.set()+ release_first.wait(timeout=5.0)+ return "late"++ def _second_compression():+ second_started.set()+ return "second"++ async def _drive():+ with pytest.raises(asyncio.TimeoutError):+ await proxy._run_compression_in_executor(_timed_out_compression, timeout=0.05)+ assert first_started.is_set()++ bypass_started = time.monotonic()+ try:+ # asyncio.TimeoutError is distinct from builtin TimeoutError on+ # Python 3.10. The quarantine signal must follow the former so the+ # existing handler failure policy classifies it as a timeout.+ with pytest.raises(asyncio.TimeoutError, match="quarantin"):+ await proxy._run_compression_in_executor(_second_compression, timeout=1.0)+ bypass_elapsed = time.monotonic() - bypass_started+ assert bypass_elapsed < 0.2+ assert not second_started.is_set()++ with proxy._compression_metrics_lock:+ assert proxy._compression_timed_out_in_flight == 1+ assert proxy._compression_quarantine_skips == 1+ assert proxy._compression_quarantine_activations == 1+ finally:+ release_first.set()++ for _ in range(100):+ with proxy._compression_metrics_lock:+ if proxy._compression_timed_out_in_flight == 0:+ break+ await asyncio.sleep(0.01)++ with proxy._compression_metrics_lock:+ assert proxy._compression_timed_out_in_flight == 0+ assert proxy._compression_leaked_threads == 1++ # Quarantine is self-clearing: normal compression resumes after the+ # timed-out worker has genuinely left the executor.+ assert (+ await proxy._run_compression_in_executor(_second_compression, timeout=1.0) == "second"+ )+ return await proxy.metrics.export()++ prometheus_text = asyncio.run(_drive())+ assert 'headroom_compression_quarantine_total{event="activated"} 1' in prometheus_text+ assert 'headroom_compression_quarantine_total{event="skipped"} 1' in prometheus_text++def test_timeout_before_worker_start_does_not_leak_in_flight() -> None:"""If a queued job times out before a worker starts, queued accountingis cleaned up without touching the running gauge.@@ -256,6 +326,9 @@ def test_timeout_before_worker_start_does_not_leak_in_flight() -> None:assert proxy._compression_queued == 0assert proxy._compression_in_flight == 0assert proxy._compression_leaked_threads == 0+ assert proxy._compression_timed_out_in_flight == 0+ assert proxy._compression_quarantine_activations == 0+ assert proxy._compression_quarantine_skips == 0def test_compression_executor_skip_signal_remains_visible() -> None:@@ -353,6 +426,11 @@ def test_compression_executor_metrics_appear_in_runtime_payload() -> None:assert ce["queue_wait_seconds_total"] == 0.0assert ce["run_seconds_total"] == 0.0assert ce["leaked_threads_total"] == 0+ assert ce["quarantine_active"] is False+ assert ce["timed_out_workers"] == 0+ assert ce["timed_out_workers_max"] == 0+ assert ce["quarantine_activations_total"] == 0+ assert ce["quarantine_skips_total"] == 0assert ce["source"] == "explicit"