Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(proxy): rename leaked_threads metric to timed_out_workers_total (#3908)
Omarbd164a2c3ba3
diff --git a/headroom/proxy/server.py b/headroom/proxy/server.pyindex 121379f51..8289ddaef 100644--- a/headroom/proxy/server.py+++ b/headroom/proxy/server.py@@ -1314,9 +1314,10 @@ class HeadroomProxy(# sharing the loop's default executor (file IO, etc.).# 2. Tasks that exceed ``COMPRESSION_TIMEOUT_SECONDS`` and complete# *after* the asyncio future was cancelled are counted in the- # ``compression_leaked_threads`` gauge — Python cannot preempt- # the worker, so this is the only signal that some pool slots- # are sitting on stuck work.+ # ``compression_timed_out_workers_total`` counter — Python cannot+ # preempt the worker, so this is the only signal that some pool+ # slots sat on work past its deadline. Cumulative: it counts+ # workers that finished late, not threads still stuck (#3653)._compression_max_cfg = config.compression_max_workersif _compression_max_cfg is None:_compression_max = max(1, os.cpu_count() or 1)@@ -1362,9 +1363,12 @@ class HeadroomProxy(self._compression_in_flight_max: int = 0self._compression_run_seconds_total: float = 0.0self._compression_run_seconds_max: float = 0.0- # Counter: threads that finished AFTER their asyncio future hit the- # timeout. Stuck-thread leak indicator.- self._compression_leaked_threads: int = 0+ # Counter: workers that finished AFTER their asyncio future hit the+ # timeout. Cumulative total, never decremented: a late-finishing+ # worker is counted once when it exits. It does NOT report live+ # stuck threads — a steady value means past timeouts, not a+ # growing leak (#3653).+ self._compression_timed_out_workers_total: int = 0# Timeout-debt quarantine. Python cannot preempt a worker after its# asyncio waiter times out, so accepting more compression while those# workers are still running can multiply slow calls into a saturated@@ -1619,7 +1623,7 @@ class HeadroomProxy(preempt running CPython bytecode or in-flight Rust calls. Theworker keeps running to completion, ignored. We detect this bymarking the call timed out on the asyncio side and incrementing- ``_compression_leaked_threads`` from the worker's ``finally``+ ``_compression_timed_out_workers_total`` from the worker's ``finally``block after it eventually finishes. While such workers hold half thepool (``_compression_quarantine_threshold``), new calls raise:class:`CompressionQuarantinedError` immediately so callers apply the@@ -1633,11 +1637,11 @@ class HeadroomProxy(Args:fn: A no-arg sync callable that runs the compression. Must notraise asyncio Cancellation; if it does, the wrapper still- decrements the in-flight gauge but the leaked-thread+ decrements the in-flight gauge but the timed-out-workercounter may double-count.timeout: Wall-clock timeout for the asyncio side. The- executor worker keeps running past this (Python limitation- — see above), but at least the awaiter unblocks.+ executor worker keeps running past this (Python limitation+ — see above), but at least the awaiter unblocks.Returns:Whatever ``fn()`` returns.@@ -1768,7 +1772,7 @@ class HeadroomProxy(if elapsed > self._compression_run_seconds_max:self._compression_run_seconds_max = elapsedif state["timed_out"]:- self._compression_leaked_threads += 1+ self._compression_timed_out_workers_total += 1if state["timeout_debt_recorded"]:self._compression_timed_out_in_flight -= 1state["timeout_debt_recorded"] = False@@ -3858,7 +3862,7 @@ def create_app(config: ProxyConfig | None = None) -> FastAPI:_comp_in_flight_max = proxy._compression_in_flight_max_comp_run_total = proxy._compression_run_seconds_total_comp_run_max = proxy._compression_run_seconds_max- _comp_leaked = proxy._compression_leaked_threads+ _comp_leaked = proxy._compression_timed_out_workers_total_comp_timed_out_in_flight = proxy._compression_timed_out_in_flight_comp_quarantine_active = (_comp_timed_out_in_flight >= proxy._compression_quarantine_threshold@@ -3894,6 +3898,11 @@ def create_app(config: ProxyConfig | None = None) -> FastAPI:"run_seconds_total": _comp_run_total,"run_seconds_max": _comp_run_max,"leaked_threads_total": _comp_leaked,+ # Canonical name: this is a cumulative count of workers that+ # finished after their deadline, not live stuck threads.+ # `leaked_threads_total` is kept as a deprecated alias so+ # existing dashboards and watchdogs keep working (#3653).+ "timed_out_workers_total": _comp_leaked,"quarantine_active": _comp_quarantine_active,"timed_out_workers": _comp_timed_out_in_flight,"timed_out_workers_max": _comp_timed_out_in_flight_max,diff --git a/tests/test_anthropic_pre_upstream_backpressure.py b/tests/test_anthropic_pre_upstream_backpressure.pyindex 00c1dbc91..1d76e0a5a 100644--- a/tests/test_anthropic_pre_upstream_backpressure.py+++ b/tests/test_anthropic_pre_upstream_backpressure.py@@ -201,7 +201,7 @@ class _DummyAnthropicHandler(AnthropicHandlerMixin):self.compression_max_workers = 4self._compression_in_flight = 0self._compression_in_flight_max = 0- self._compression_leaked_threads = 0+ self._compression_timed_out_workers_total = 0self._compression_metrics_lock = _threading.Lock()self._upstream_delay_s = upstream_delay_sself._raise_during_critical = raise_during_critical@@ -234,7 +234,7 @@ class _DummyAnthropicHandler(AnthropicHandlerMixin):with self._compression_metrics_lock:self._compression_in_flight -= 1if elapsed > timeout:- self._compression_leaked_threads += 1+ self._compression_timed_out_workers_total += 1future = loop.run_in_executor(self._compression_executor, _wrapped)return await asyncio.wait_for(future, timeout=timeout)diff --git a/tests/test_proxy_compression_executor.py b/tests/test_proxy_compression_executor.pyindex 4d34475f9..e650c0fe8 100644--- a/tests/test_proxy_compression_executor.py+++ b/tests/test_proxy_compression_executor.py@@ -12,7 +12,8 @@ Locks the following invariants:3. When a compression call exceeds its timeout, the awaiter unblocks with``TimeoutError`` — but the worker thread keeps running (Python cannotpreempt running CPython bytecode or in-flight Rust calls), and when the- work eventually completes, ``compression_leaked_threads`` increments.+ work eventually completes, ``_compression_timed_out_workers_total``+ increments.4. Jobs that time out while still queued do not leak the running gauge.5. ``/stats runtime.compression_executor`` surfaces the gauges + counters sooperators can see leaked-thread rate and queue pressure.@@ -23,7 +24,7 @@ Locks the following invariants:These tests also serve as documentation: anyone reading them sees that"timeout fired" does not mean "compression was cancelled" — it means "westopped waiting; the worker is still going". A bounded pool plus the-leaked-thread counter is how we make that visible.+timed-out-worker counter is how we make that visible."""from __future__ import annotations@@ -245,7 +246,7 @@ def test_high_water_mark_persists_after_completion() -> None:def test_timeout_fires_and_leaked_thread_is_counted() -> None:"""When the compression exceeds ``timeout``, the awaiter sees``TimeoutError`` immediately. The worker keeps running; when it finishes,- ``_compression_leaked_threads`` increments by 1.+ ``_compression_timed_out_workers_total`` increments by 1."""proxy = _make_proxy(compression_max_workers=2)finished_event = threading.Event()@@ -269,13 +270,13 @@ def test_timeout_fires_and_leaked_thread_is_counted() -> None:deadline = time.monotonic() + 1.0while time.monotonic() < deadline:with proxy._compression_metrics_lock:- if proxy._compression_leaked_threads >= 1:+ if proxy._compression_timed_out_workers_total >= 1:breaktime.sleep(0.01)with proxy._compression_metrics_lock:- assert proxy._compression_leaked_threads >= 1, (- f"leaked_threads should be ≥ 1; got {proxy._compression_leaked_threads}. "+ assert proxy._compression_timed_out_workers_total >= 1, (+ f"timed-out-worker count should be ≥ 1; got {proxy._compression_timed_out_workers_total}. "f"The worker either didn't finish past the deadline, or the wrapper "f"didn't increment the counter.")@@ -336,7 +337,7 @@ def test_timeout_quarantines_new_work_until_timed_out_worker_finishes() -> None:with proxy._compression_metrics_lock:assert proxy._compression_timed_out_in_flight == 0- assert proxy._compression_leaked_threads == 1+ assert proxy._compression_timed_out_workers_total == 1# Quarantine is self-clearing: normal compression resumes after the# timed-out worker has genuinely left the executor.@@ -486,7 +487,7 @@ def test_timeout_before_worker_start_does_not_leak_in_flight() -> None:with proxy._compression_metrics_lock:assert proxy._compression_queued == 0assert proxy._compression_in_flight == 0- assert proxy._compression_leaked_threads == 0+ assert proxy._compression_timed_out_workers_total == 0assert proxy._compression_timed_out_in_flight == 0assert proxy._compression_quarantine_activations == 0assert proxy._compression_quarantine_skips == 0@@ -587,6 +588,9 @@ def test_compression_executor_metrics_appear_in_runtime_payload() -> None:assert ce["queue_wait_seconds_total"] == 0.0assert ce["run_seconds_total"] == 0.0assert ce["leaked_threads_total"] == 0+ # Deprecated alias must track the canonical counter (#3653).+ assert ce["timed_out_workers_total"] == 0+ assert ce["timed_out_workers_total"] == ce["leaked_threads_total"]assert ce["quarantine_active"] is Falseassert ce["timed_out_workers"] == 0assert ce["timed_out_workers_max"] == 0