Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(proxy): quarantine compression only once half the pool is stuck (#3932)
gglucassbe2b205c5143
diff --git a/headroom/proxy/server.py b/headroom/proxy/server.pyindex ca82ea3ad..ce8ccfd91 100644--- a/headroom/proxy/server.py+++ b/headroom/proxy/server.py@@ -1276,11 +1276,19 @@ class HeadroomProxy(# timeout. Stuck-thread leak indicator.self._compression_leaked_threads: int = 0# Timeout-debt quarantine. Python cannot preempt a worker after its- # asyncio waiter times out, so accepting more compression while that- # worker is still running can multiply one slow call into a saturated- # executor. New work raises immediately until every known post-timeout- # worker has genuinely exited, then follows the caller's existing- # compression-failure policy.+ # asyncio waiter times out, so accepting more compression while those+ # workers are still running can multiply slow calls into a saturated+ # executor. Once half the pool is held by post-timeout workers, new work+ # raises immediately until the debt drops below that again, and callers+ # follow their existing compression-failure policy. One straggler on a+ # mostly idle pool does not saturate it, so it does not refuse work;+ # pools of 1-3 workers still quarantine on the first timeout.+ # HEADROOM_COMPRESSION_QUARANTINE_THRESHOLD=1 restores that everywhere.+ self._compression_quarantine_threshold: int = _get_env_int(+ "HEADROOM_COMPRESSION_QUARANTINE_THRESHOLD",+ max(1, _compression_max // 2),+ min_value=1,+ )self._compression_timed_out_in_flight: int = 0self._compression_timed_out_in_flight_max: int = 0self._compression_quarantine_activations: int = 0@@ -1531,10 +1539,11 @@ class HeadroomProxy(worker keeps running to completion, ignored. We detect this bymarking the call timed out on the asyncio side and incrementing``_compression_leaked_threads`` from the worker's ``finally``- block after it eventually finishes. While any such worker remains,- new calls raise :class:`CompressionQuarantinedError` immediately so- callers apply the existing compression-failure policy instead of- filling the rest of the pool with the same timeout debt. Jobs that are+ block after it eventually finishes. While such workers hold half the+ pool (``_compression_quarantine_threshold``), new calls raise+ :class:`CompressionQuarantinedError` immediately so callers apply the+ existing compression-failure policy instead of filling the rest of the+ pool with the same timeout debt. Jobs that aresuccessfully cancelled before a worker starts are removed from thequeued gauge and do not activate the quarantine. If cancellation raceswith worker startup, the now-running job is tracked as timeout debt and@@ -1556,20 +1565,19 @@ class HeadroomProxy(``asyncio.TimeoutError`` if the callable doesn't return within``timeout``. Any exception raised by ``fn`` propagatesunchanged. :class:`CompressionQuarantinedError` (an- ``asyncio.TimeoutError`` subclass) if a prior timed-out worker is- still running.+ ``asyncio.TimeoutError`` subclass) if prior timed-out workers still+ hold half the pool."""now = time.monotonic()with self._compression_metrics_lock:timed_out_in_flight = self._compression_timed_out_in_flight- quarantined = timed_out_in_flight > 0 and now < self._compression_quarantine_deadline+ saturating = timed_out_in_flight >= self._compression_quarantine_threshold+ quarantined = saturating and now < self._compression_quarantine_deadline# Debt outlived the cap: presume the worker leaked/hung and stop# blocking on it. Count the release once per lapse (while debt stands# and the deadline has passed) so operators can see it happened.released = (- timed_out_in_flight > 0- and not quarantined- and self._compression_quarantine_deadline > 0.0+ saturating and not quarantined and self._compression_quarantine_deadline > 0.0)if quarantined:self._compression_quarantine_skips += 1@@ -1613,7 +1621,7 @@ class HeadroomProxy("""Record a still-running post-timeout worker; lock must be held.Returns ``True`` only when this worker transitions the executor- from clear to quarantined.+ from clear (or released by the time cap) to quarantined."""if (not state["timed_out"]@@ -1622,7 +1630,12 @@ class HeadroomProxy(or state["timeout_debt_recorded"]):return False- was_clear = self._compression_timed_out_in_flight == 0+ now = time.monotonic()+ threshold = self._compression_quarantine_threshold+ was_quarantined = (+ self._compression_timed_out_in_flight >= threshold+ and now < self._compression_quarantine_deadline+ )self._compression_timed_out_in_flight += 1self._compression_timed_out_in_flight_max = max(self._compression_timed_out_in_flight_max,@@ -1631,19 +1644,19 @@ class HeadroomProxy(# (Re)arm the quarantine time cap on every fresh timeout, so ongoing# slowness keeps quarantining while a single leaked worker cannot# hold it past the cap (#2360).- self._compression_quarantine_deadline = (- time.monotonic() + self._compression_quarantine_max_seconds- )+ self._compression_quarantine_deadline = now + self._compression_quarantine_max_secondsstate["timeout_debt_recorded"] = True- if was_clear:+ activated = not was_quarantined and self._compression_timed_out_in_flight >= threshold+ if activated:self._compression_quarantine_activations += 1- return was_clear+ return activateddef _announce_quarantine() -> None:self.metrics.record_compression_quarantine("activated")logger.warning("Compression worker exceeded its request deadline and is still running; "- "new compression is quarantined until timed-out workers exit"+ "new compression is quarantined until fewer than %d timed-out workers remain",+ self._compression_quarantine_threshold,)def _wrapped(): # noqa: ANN202@@ -1678,11 +1691,15 @@ class HeadroomProxy(if state["timeout_debt_recorded"]:self._compression_timed_out_in_flight -= 1state["timeout_debt_recorded"] = False- quarantine_cleared = self._compression_timed_out_in_flight == 0+ remaining = self._compression_timed_out_in_flight+ quarantine_cleared = remaining == self._compression_quarantine_threshold - 1else:quarantine_cleared = Falseif quarantine_cleared:- logger.info("Compression quarantine cleared after all timed-out workers exited")+ logger.info(+ "Compression quarantine cleared; %d timed-out worker(s) still running",+ remaining,+ )future = loop.run_in_executor(self._compression_executor, _wrapped)try:@@ -3542,6 +3559,10 @@ def create_app(config: ProxyConfig | None = None) -> FastAPI:_comp_run_max = proxy._compression_run_seconds_max_comp_leaked = proxy._compression_leaked_threads_comp_timed_out_in_flight = proxy._compression_timed_out_in_flight+ _comp_quarantine_active = (+ _comp_timed_out_in_flight >= proxy._compression_quarantine_threshold+ and time.monotonic() < proxy._compression_quarantine_deadline+ )_comp_timed_out_in_flight_max = proxy._compression_timed_out_in_flight_max_comp_quarantine_activations = proxy._compression_quarantine_activations_comp_quarantine_skips = proxy._compression_quarantine_skips@@ -3572,7 +3593,7 @@ def create_app(config: ProxyConfig | None = None) -> FastAPI:"run_seconds_total": _comp_run_total,"run_seconds_max": _comp_run_max,"leaked_threads_total": _comp_leaked,- "quarantine_active": _comp_timed_out_in_flight > 0,+ "quarantine_active": _comp_quarantine_active,"timed_out_workers": _comp_timed_out_in_flight,"timed_out_workers_max": _comp_timed_out_in_flight_max,"quarantine_activations_total": _comp_quarantine_activations,diff --git a/tests/test_proxy_compression_executor.py b/tests/test_proxy_compression_executor.pyindex 159ff7952..7659c4eeb 100644--- a/tests/test_proxy_compression_executor.py+++ b/tests/test_proxy_compression_executor.py@@ -294,6 +294,93 @@ def test_timeout_quarantines_new_work_until_timed_out_worker_finishes() -> None:assert 'headroom_compression_quarantine_total{event="skipped"} 1' in prometheus_text+def test_quarantine_waits_for_half_the_pool_to_be_stuck() -> None:+ """One straggler must not refuse compression while most workers are idle.++ The quarantine exists so timed-out work cannot saturate the executor. On+ an 8-worker pool, one or three stuck workers leave the pool mostly idle,+ so compression keeps running; at four (half the pool) the quarantine+ engages, is counted once, and ``/health`` reports it. A timeout after the+ time cap released it counts as a fresh activation.+ """+ from fastapi.testclient import TestClient++ from headroom.proxy.server import CompressionQuarantinedError++ config = ProxyConfig(+ optimize=False,+ cache_enabled=False,+ rate_limit_enabled=False,+ cost_tracking_enabled=False,+ log_requests=False,+ ccr_inject_tool=False,+ ccr_handle_responses=False,+ ccr_context_tracking=False,+ image_optimize=False,+ compression_max_workers=8,+ )+ app = create_app(config)+ proxy = app.state.proxy+ release = threading.Event()++ async def _strand(count: int) -> None:+ for _ in range(count):+ with pytest.raises(asyncio.TimeoutError):+ await proxy._run_compression_in_executor(release.wait, timeout=0.05)++ def _executor_health() -> dict:+ with TestClient(app) as client:+ return client.get("/health").json()["runtime"]["compression_executor"]++ try:+ asyncio.run(_strand(1))+ assert asyncio.run(proxy._run_compression_in_executor(lambda: "ran", timeout=1.0)) == "ran"+ health = _executor_health()+ assert health["timed_out_workers"] == 1+ assert health["quarantine_active"] is False++ asyncio.run(_strand(2))+ assert asyncio.run(proxy._run_compression_in_executor(lambda: "ran", timeout=1.0)) == "ran"+ with proxy._compression_metrics_lock:+ assert proxy._compression_quarantine_activations == 0+ assert proxy._compression_quarantine_skips == 0+ assert proxy._compression_quarantine_releases == 0++ asyncio.run(_strand(1))+ with pytest.raises(CompressionQuarantinedError):+ asyncio.run(proxy._run_compression_in_executor(lambda: "unused", timeout=1.0))+ health = _executor_health()+ assert health["timed_out_workers"] == 4+ assert health["quarantine_active"] is True+ assert health["quarantine_activations_total"] == 1+ assert health["quarantine_skips_total"] == 1++ proxy._compression_quarantine_deadline = time.monotonic() - 1.0+ assert asyncio.run(proxy._run_compression_in_executor(lambda: "ran", timeout=1.0)) == "ran"+ assert proxy._compression_quarantine_releases == 1+ asyncio.run(_strand(1))+ with pytest.raises(CompressionQuarantinedError):+ asyncio.run(proxy._run_compression_in_executor(lambda: "unused", timeout=1.0))+ assert proxy._compression_quarantine_activations == 2+ finally:+ release.set()+++ ("workers", "env", "expected"),+ [(8, None, 4), (3, None, 1), (1, None, 1), (8, "1", 1), (8, "0", 4)],+)+def test_quarantine_threshold_is_half_the_pool_unless_overridden(+ monkeypatch, workers: int, env: str | None, expected: int+) -> None:+ if env is None:+ monkeypatch.delenv("HEADROOM_COMPRESSION_QUARANTINE_THRESHOLD", raising=False)+ else:+ monkeypatch.setenv("HEADROOM_COMPRESSION_QUARANTINE_THRESHOLD", env)+ proxy = _make_proxy(compression_max_workers=workers)+ assert proxy._compression_quarantine_threshold == expected++def test_timeout_before_worker_start_does_not_leak_in_flight() -> None:"""If a queued job times out before a worker starts, queued accountingis cleaned up without touching the running gauge.diff --git a/tests/test_tokenizer_count_offload.py b/tests/test_tokenizer_count_offload.pyindex e59b001a4..77db5a4d8 100644--- a/tests/test_tokenizer_count_offload.py+++ b/tests/test_tokenizer_count_offload.py@@ -179,8 +179,9 @@ async def test_count_tokens_offloaded_fails_open_on_executor_quarantine() -> Non# quarantines the next call — no mock of the helper itself. Since the# quarantine became time-capped (#2412), standing debt alone no longer# quarantines: the deadline armed by the fresh timeout must still be in- # the future, so arm it the way a real timeout would.- proxy._compression_timed_out_in_flight = 1+ # the future, so arm it the way a real timeout would. The debt must reach+ # the quarantine threshold (half the auto-sized pool).+ proxy._compression_timed_out_in_flight = proxy._compression_quarantine_thresholdproxy._compression_quarantine_deadline = time.monotonic() + 60.0tokenizer, tokens = await proxy._count_tokens_offloaded(@@ -198,8 +199,8 @@ async def test_count_tokens_offloaded_returns_count_text_capable_tokenizer() ->proxy = _make_proxy()# Quarantine forces the fail-open branch (an EstimatingTokenCounter).# Post-#2412 the quarantine is time-capped, so the deadline must be armed- # alongside the standing debt.- proxy._compression_timed_out_in_flight = 1+ # alongside the standing debt, which must reach the quarantine threshold.+ proxy._compression_timed_out_in_flight = proxy._compression_quarantine_thresholdproxy._compression_quarantine_deadline = time.monotonic() + 60.0# The empty-messages count is intentionally discarded by that handler