Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(proxy): rename leaked_threads metric to timed_out_workers_total (#3908)

Omarbd164a2c3ba3
diff --git a/headroom/proxy/server.py b/headroom/proxy/server.py
index 121379f51..8289ddaef 100644
--- a/headroom/proxy/server.py
+++ b/headroom/proxy/server.py
@@ -1314,9 +1314,10 @@ class HeadroomProxy(
# sharing the loop's default executor (file IO, etc.).
# 2. Tasks that exceed ``COMPRESSION_TIMEOUT_SECONDS`` and complete
# *after* the asyncio future was cancelled are counted in the
- # ``compression_leaked_threads`` gauge — Python cannot preempt
- # the worker, so this is the only signal that some pool slots
- # are sitting on stuck work.
+ # ``compression_timed_out_workers_total`` counter — Python cannot
+ # preempt the worker, so this is the only signal that some pool
+ # slots sat on work past its deadline. Cumulative: it counts
+ # workers that finished late, not threads still stuck (#3653).
_compression_max_cfg = config.compression_max_workers
if _compression_max_cfg is None:
_compression_max = max(1, os.cpu_count() or 1)
@@ -1362,9 +1363,12 @@ class HeadroomProxy(
self._compression_in_flight_max: int = 0
self._compression_run_seconds_total: float = 0.0
self._compression_run_seconds_max: float = 0.0
- # Counter: threads that finished AFTER their asyncio future hit the
- # timeout. Stuck-thread leak indicator.
- self._compression_leaked_threads: int = 0
+ # Counter: workers that finished AFTER their asyncio future hit the
+ # timeout. Cumulative total, never decremented: a late-finishing
+ # worker is counted once when it exits. It does NOT report live
+ # stuck threads — a steady value means past timeouts, not a
+ # growing leak (#3653).
+ self._compression_timed_out_workers_total: int = 0
# Timeout-debt quarantine. Python cannot preempt a worker after its
# asyncio waiter times out, so accepting more compression while those
# workers are still running can multiply slow calls into a saturated
@@ -1619,7 +1623,7 @@ class HeadroomProxy(
preempt running CPython bytecode or in-flight Rust calls. The
worker keeps running to completion, ignored. We detect this by
marking the call timed out on the asyncio side and incrementing
- ``_compression_leaked_threads`` from the worker's ``finally``
+ ``_compression_timed_out_workers_total`` from the worker's ``finally``
block after it eventually finishes. While such workers hold half the
pool (``_compression_quarantine_threshold``), new calls raise
:class:`CompressionQuarantinedError` immediately so callers apply the
@@ -1633,11 +1637,11 @@ class HeadroomProxy(
Args:
fn: A no-arg sync callable that runs the compression. Must not
raise asyncio Cancellation; if it does, the wrapper still
- decrements the in-flight gauge but the leaked-thread
+ decrements the in-flight gauge but the timed-out-worker
counter may double-count.
timeout: Wall-clock timeout for the asyncio side. The
- executor worker keeps running past this (Python limitation
- — see above), but at least the awaiter unblocks.
+ executor worker keeps running past this (Python limitation
+ — see above), but at least the awaiter unblocks.
Returns:
Whatever ``fn()`` returns.
@@ -1768,7 +1772,7 @@ class HeadroomProxy(
if elapsed > self._compression_run_seconds_max:
self._compression_run_seconds_max = elapsed
if state["timed_out"]:
- self._compression_leaked_threads += 1
+ self._compression_timed_out_workers_total += 1
if state["timeout_debt_recorded"]:
self._compression_timed_out_in_flight -= 1
state["timeout_debt_recorded"] = False
@@ -3858,7 +3862,7 @@ def create_app(config: ProxyConfig | None = None) -> FastAPI:
_comp_in_flight_max = proxy._compression_in_flight_max
_comp_run_total = proxy._compression_run_seconds_total
_comp_run_max = proxy._compression_run_seconds_max
- _comp_leaked = proxy._compression_leaked_threads
+ _comp_leaked = proxy._compression_timed_out_workers_total
_comp_timed_out_in_flight = proxy._compression_timed_out_in_flight
_comp_quarantine_active = (
_comp_timed_out_in_flight >= proxy._compression_quarantine_threshold
@@ -3894,6 +3898,11 @@ def create_app(config: ProxyConfig | None = None) -> FastAPI:
"run_seconds_total": _comp_run_total,
"run_seconds_max": _comp_run_max,
"leaked_threads_total": _comp_leaked,
+ # Canonical name: this is a cumulative count of workers that
+ # finished after their deadline, not live stuck threads.
+ # `leaked_threads_total` is kept as a deprecated alias so
+ # existing dashboards and watchdogs keep working (#3653).
+ "timed_out_workers_total": _comp_leaked,
"quarantine_active": _comp_quarantine_active,
"timed_out_workers": _comp_timed_out_in_flight,
"timed_out_workers_max": _comp_timed_out_in_flight_max,
diff --git a/tests/test_anthropic_pre_upstream_backpressure.py b/tests/test_anthropic_pre_upstream_backpressure.py
index 00c1dbc91..1d76e0a5a 100644
--- a/tests/test_anthropic_pre_upstream_backpressure.py
+++ b/tests/test_anthropic_pre_upstream_backpressure.py
@@ -201,7 +201,7 @@ class _DummyAnthropicHandler(AnthropicHandlerMixin):
self.compression_max_workers = 4
self._compression_in_flight = 0
self._compression_in_flight_max = 0
- self._compression_leaked_threads = 0
+ self._compression_timed_out_workers_total = 0
self._compression_metrics_lock = _threading.Lock()
self._upstream_delay_s = upstream_delay_s
self._raise_during_critical = raise_during_critical
@@ -234,7 +234,7 @@ class _DummyAnthropicHandler(AnthropicHandlerMixin):
with self._compression_metrics_lock:
self._compression_in_flight -= 1
if elapsed > timeout:
- self._compression_leaked_threads += 1
+ self._compression_timed_out_workers_total += 1
future = loop.run_in_executor(self._compression_executor, _wrapped)
return await asyncio.wait_for(future, timeout=timeout)
diff --git a/tests/test_proxy_compression_executor.py b/tests/test_proxy_compression_executor.py
index 4d34475f9..e650c0fe8 100644
--- a/tests/test_proxy_compression_executor.py
+++ b/tests/test_proxy_compression_executor.py
@@ -12,7 +12,8 @@ Locks the following invariants:
3. When a compression call exceeds its timeout, the awaiter unblocks with
``TimeoutError`` — but the worker thread keeps running (Python cannot
preempt running CPython bytecode or in-flight Rust calls), and when the
- work eventually completes, ``compression_leaked_threads`` increments.
+ work eventually completes, ``_compression_timed_out_workers_total``
+ increments.
4. Jobs that time out while still queued do not leak the running gauge.
5. ``/stats runtime.compression_executor`` surfaces the gauges + counters so
operators can see leaked-thread rate and queue pressure.
@@ -23,7 +24,7 @@ Locks the following invariants:
These tests also serve as documentation: anyone reading them sees that
"timeout fired" does not mean "compression was cancelled" — it means "we
stopped waiting; the worker is still going". A bounded pool plus the
-leaked-thread counter is how we make that visible.
+timed-out-worker counter is how we make that visible.
"""
from __future__ import annotations
@@ -245,7 +246,7 @@ def test_high_water_mark_persists_after_completion() -> None:
def test_timeout_fires_and_leaked_thread_is_counted() -> None:
"""When the compression exceeds ``timeout``, the awaiter sees
``TimeoutError`` immediately. The worker keeps running; when it finishes,
- ``_compression_leaked_threads`` increments by 1.
+ ``_compression_timed_out_workers_total`` increments by 1.
"""
proxy = _make_proxy(compression_max_workers=2)
finished_event = threading.Event()
@@ -269,13 +270,13 @@ def test_timeout_fires_and_leaked_thread_is_counted() -> None:
deadline = time.monotonic() + 1.0
while time.monotonic() < deadline:
with proxy._compression_metrics_lock:
- if proxy._compression_leaked_threads >= 1:
+ if proxy._compression_timed_out_workers_total >= 1:
break
time.sleep(0.01)
with proxy._compression_metrics_lock:
- assert proxy._compression_leaked_threads >= 1, (
- f"leaked_threads should be ≥ 1; got {proxy._compression_leaked_threads}. "
+ assert proxy._compression_timed_out_workers_total >= 1, (
+ f"timed-out-worker count should be ≥ 1; got {proxy._compression_timed_out_workers_total}. "
f"The worker either didn't finish past the deadline, or the wrapper "
f"didn't increment the counter."
)
@@ -336,7 +337,7 @@ def test_timeout_quarantines_new_work_until_timed_out_worker_finishes() -> None:
with proxy._compression_metrics_lock:
assert proxy._compression_timed_out_in_flight == 0
- assert proxy._compression_leaked_threads == 1
+ assert proxy._compression_timed_out_workers_total == 1
# Quarantine is self-clearing: normal compression resumes after the
# timed-out worker has genuinely left the executor.
@@ -486,7 +487,7 @@ def test_timeout_before_worker_start_does_not_leak_in_flight() -> None:
with proxy._compression_metrics_lock:
assert proxy._compression_queued == 0
assert proxy._compression_in_flight == 0
- assert proxy._compression_leaked_threads == 0
+ assert proxy._compression_timed_out_workers_total == 0
assert proxy._compression_timed_out_in_flight == 0
assert proxy._compression_quarantine_activations == 0
assert proxy._compression_quarantine_skips == 0
@@ -587,6 +588,9 @@ def test_compression_executor_metrics_appear_in_runtime_payload() -> None:
assert ce["queue_wait_seconds_total"] == 0.0
assert ce["run_seconds_total"] == 0.0
assert ce["leaked_threads_total"] == 0
+ # Deprecated alias must track the canonical counter (#3653).
+ assert ce["timed_out_workers_total"] == 0
+ assert ce["timed_out_workers_total"] == ce["leaked_threads_total"]
assert ce["quarantine_active"] is False
assert ce["timed_out_workers"] == 0
assert ce["timed_out_workers_max"] == 0