Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(router): bound ML compression per request, not just per block (#3721)

Tejas Chopraf37980f7f761
diff --git a/headroom/transforms/content_router.py b/headroom/transforms/content_router.py
index 848cb81fe..c9fac792a 100644
--- a/headroom/transforms/content_router.py
+++ b/headroom/transforms/content_router.py
@@ -166,6 +166,68 @@ def _estimate_tokens(text: str) -> int:
return max(1, _TOKEN_ESTIMATOR.count_text(text))
+#: Share of the outer compression timeout the ML stage may consume. The
+#: remainder has to cover this request's non-ML work AND the overrun of the one
+#: ML call that can still be running when the ceiling is reached -- the worker
+#: is not preemptible, so the ceiling can only stop the NEXT call.
+_ML_STAGE_BUDGET_FRACTION = 0.5
+
+
+def _ml_stage_deadline_seconds() -> float:
+ """Wall-clock ceiling for ALL ML (Kompress) work inside one request.
+
+ ``_kompress_max_tokens`` bounds a single block. It cannot bound a request:
+ the mixed-content path splits a payload into sections and calls
+ ``_try_ml_compressor`` once per section (see ``_compress_mixed``), so every
+ section can sit under the per-block ceiling while their sum runs for
+ minutes. #3711 reported ~70s on a 1.4MB tool_result of prose blocks
+ separated by small JSON objects -- with the size gate recording only
+ ``within`` -- which blew the 30s compression budget and then quarantined
+ compression for every following request.
+
+ So the guard has to match the budget's granularity: once this much time
+ has gone into ML for this request, remaining blocks pass through unchanged.
+ Raw rather than a cheaper lossy fallback: the router caches that verdict,
+ so the block keeps the same bytes on later turns and the prompt cache
+ holds.
+
+ The point of the ceiling is to degrade to a partial compression *before*
+ the executor abandons a non-preemptible worker, so it is worthless unless
+ it sits below the deadline that does the abandoning. A bare default could
+ not promise that: an operator who lowers
+ ``HEADROOM_COMPRESSION_TIMEOUT_SECONDS`` below 15s got a guard that could
+ never fire first -- exactly the failure this exists to prevent. The budget
+ is therefore clamped to :data:`_ML_STAGE_BUDGET_FRACTION` of the effective
+ outer timeout, leaving the remainder for the non-ML work in the same
+ request (lossless transforms, assembly) plus the overrun of the one ML call
+ already in flight when the ceiling is reached.
+
+ The outer budget read here is ``HEADROOM_COMPRESSION_TIMEOUT_SECONDS``, the
+ executor timeout that quarantines the stage -- NOT
+ ``HEADROOM_COMPRESSION_DEADLINE_MS``, which bounds a different, softer
+ in-router deadline (see ``_compression_deadline_seconds``).
+
+ ``0`` disables the ceiling (previous behavior), and an explicit request
+ below the clamp is honoured -- the clamp only ever lowers.
+ """
+ try:
+ configured = max(
+ 0.0,
+ float(os.environ.get("HEADROOM_ML_STAGE_DEADLINE_SECONDS", "15")),
+ )
+ except ValueError:
+ configured = 15.0
+ if configured <= 0.0:
+ return 0.0
+ try:
+ outer = float(os.environ.get("HEADROOM_COMPRESSION_TIMEOUT_SECONDS", "30"))
+ except ValueError:
+ outer = 30.0
+ if outer <= 0.0:
+ return configured
+ return min(configured, outer * _ML_STAGE_BUDGET_FRACTION)
+
+
def _compression_deadline_seconds() -> float:
try:
return max(
@@ -2047,6 +2109,22 @@ class _PerRequestRuntimeState:
compression_policy: Any = None
target_ratio: float | None = None
+ # Whether an `apply()` call owns this state and the ML budget therefore
+ # applies. False on the INSTANCE-SCOPED DEFAULT that direct `compress()`
+ # callers read (see `ContentRouter.__init__`): that object is shared by
+ # every such call for the life of the router, so charging it would
+ # accumulate across unrelated calls until ML switched itself off forever.
+ # Direct callers keep the old unbounded behaviour.
+ ml_budget_active: bool = False
+ # Seconds this request has spent INSIDE the ML compressor (#3711).
+ # Cumulative rather than a wall-clock deadline stamped at `apply()` entry:
+ # the budget is a statement about ML work, so time spent on lifecycle,
+ # classification and lossless transforms must not draw it down.
+ ml_elapsed: float = 0.0
+ # Longest single ML call observed in THIS request, seconds per estimated
+ # token. Used to project whether the next block can finish inside what is
+ # left; see `_try_ml_compressor`.
+ ml_seconds_per_token: float = 0.0
force_kompress: bool = False
skip_kompress: bool = False
kompress_model: str | None = None
@@ -4378,6 +4456,28 @@ class ContentRouter(Transform):
return None
return elided, _estimate_tokens(elided)
+ def _charge_ml_time(self, elapsed: float, text: str) -> None:
+ """Bill *elapsed* seconds of ML work to this request's budget.
+
+ Also records the worst seconds-per-token seen so far, which is what the
+ admission check in :meth:`_try_ml_compressor` projects the NEXT block
+ with. Worst case rather than an average: the ceiling exists to stop a
+ single long call from overrunning the outer timeout, and an average
+ would let one slow block hide behind several fast ones.
+
+ Best-effort -- accounting must never break a compression that worked.
+ """
+ try:
+ state = self._runtime_state_var.get()
+ if not state.ml_budget_active:
+ return
+ state.ml_elapsed += max(0.0, elapsed)
+ tokens = _estimate_tokens(text)
+ if tokens > 0 and elapsed > 0.0:
+ state.ml_seconds_per_token = max(state.ml_seconds_per_token, elapsed / tokens)
+ except Exception: # pragma: no cover - defensive
+ logger.debug("ML budget accounting failed", exc_info=True)
+
def _embedded_json_route(
self, content: str, context: str, question: str | None, bias: float
) -> str | None:
@@ -4468,6 +4568,60 @@ class ContentRouter(Transform):
# (under the 50,000 cap, gate silent) but 55,557 estimator tokens, 11%
# over. That is exactly the >30s non-preemptible ONNX inference this gate
# exists to prevent (#1171).
+ # Request-scoped ML ceiling (#3711). The size check below bounds ONE
+ # block; this bounds the request. Without it a payload split into many
+ # individually-small sections calls this function once per section, each
+ # passing the size gate, until the sum blows the compression budget and
+ # quarantines the stage for every later request. Checked before the size
+ # gate so an over-budget request stops paying the estimator too.
+ # An admission check that only asks "is the budget gone yet?" lets a
+ # call start at 14.9s of a 15s budget and run for another minute: the
+ # ONNX worker is not preemptible, so nothing stops it once it begins,
+ # and it can overrun the outer executor timeout the ceiling exists to
+ # protect. So project this block's cost and refuse it unless it can
+ # plausibly FINISH inside what is left.
+ #
+ # The projection uses the worst seconds-per-token this request has
+ # actually measured. The first block of a request has no measurement,
+ # so it cannot be refused on one -- instead the call itself is capped:
+ # what is left of the budget goes to Kompress as its per-call time
+ # budget (below), and Kompress stops at the next chunk boundary when
+ # it runs out, returning the block unchanged. That holds for every
+ # call, so a projection that guessed low cannot overrun either.
+ _ml_state = self._runtime_state_var.get()
+ _ml_budget = _ml_stage_deadline_seconds() if _ml_state.ml_budget_active else 0.0
+ _remaining = (_ml_budget - _ml_state.ml_elapsed) if _ml_budget > 0 else None
+ # Exhausted is decided before any estimator work, so an over-budget
+ # request stops paying for tokenization too.
+ _over_budget = _remaining is not None and _remaining <= 0.0
+ _block_tokens = -1
+ _projected = 0.0
+ if _remaining is not None and not _over_budget:
+ _rate = _ml_state.ml_seconds_per_token
+ if _rate > 0.0:
+ _block_tokens = _estimate_tokens(text_to_compress)
+ _projected = _block_tokens * _rate
+ _over_budget = _projected > _remaining
+ if _over_budget:
+ self._observe_kompress_size_gate("deadline")
+ if _block_tokens < 0:
+ _block_tokens = _estimate_tokens(text_to_compress)
+ logger.info(
+ "ML stage budget spent (%.1fs of %.1fs used, ~%.1fs projected for a "
+ "~%d tok block, %.1fs left); passing remaining blocks through "
+ "unchanged. Partial compression beats a blown budget.",
+ _ml_state.ml_elapsed,
+ _ml_budget,
+ _projected,
+ _block_tokens,
+ max(0.0, _remaining or 0.0),
+ )
+ # Unchanged, not the size gate's lossy fallback below: this
+ # verdict depends on the request, not the content, and the router
+ # caches it, so the bytes must be ones it can keep sending on
+ # later turns without busting the prompt cache.
+ return content, _estimate_tokens(content)
+
if (
self._kompress_max_tokens > 0
and _estimate_tokens(text_to_compress) > self._kompress_max_tokens
@@ -4564,7 +4718,28 @@ class ContentRouter(Transform):
compressor, "shares_request_deadline", False
):
compress_kwargs["_deadline_started_at"] = deadline_origin
- result = compressor.compress(text_to_compress, **compress_kwargs)
+ # The ML budget's remainder caps THIS call, so the
+ # first one of a request -- admitted without a measured
+ # cost -- cannot run past the budget either. Remote
+ # Kompress takes the cap too (it bounds the whole HTTP
+ # request, and declines the call when too little is
+ # left) but not the shared deadline origin, hence the
+ # separate flag.
+ if _remaining is not None and (
+ getattr(compressor, "shares_request_deadline", False)
+ or getattr(compressor, "accepts_time_budget_cap", False)
+ ):
+ compress_kwargs["_time_budget_cap_seconds"] = _remaining
+ _ml_started = time.monotonic()
+ try:
+ result = compressor.compress(text_to_compress, **compress_kwargs)
+ finally:
+ # In a `finally` so a call that RAISES still draws
+ # down the budget: a failing compressor burns the
+ # same non-preemptible wall clock as a passing one,
+ # and not charging for it let a request retry its
+ # way past the ceiling.
+ self._charge_ml_time(time.monotonic() - _ml_started, text_to_compress)
compressed = result.compressed
compressed_tokens = result.compressed_tokens
except Exception as e:
@@ -5485,8 +5660,18 @@ class ContentRouter(Transform):
# The kompress deadline origin is stamped HERE, with the rest of the
# per-request state, so every block this call compresses draws down one
# shared budget instead of restarting it.
+ #
+ # The ML ceiling (#3711) is deliberately NOT armed here. It used to be,
+ # which meant lifecycle work, message classification, lossless
+ # transforms and task preparation all ran down a budget whose log line
+ # claimed to measure time "in ML". `_try_ml_compressor` accumulates the
+ # time it actually spends inside the compressor instead, so the number
+ # means what it says.
self._runtime_state_var.set(
- _PerRequestRuntimeState(kompress_deadline_started_at=time.perf_counter())
+ _PerRequestRuntimeState(
+ kompress_deadline_started_at=time.perf_counter(),
+ ml_budget_active=True,
+ )
)
# Pre-process: Read lifecycle management (stale/superseded detection)
diff --git a/headroom/transforms/kompress_compressor.py b/headroom/transforms/kompress_compressor.py
index 0b9a8a8e6..e5d48c60a 100644
--- a/headroom/transforms/kompress_compressor.py
+++ b/headroom/transforms/kompress_compressor.py
@@ -441,6 +441,21 @@ def _time_budget_seconds() -> float | None:
return raw if raw > 0 else None
+def _capped_time_budget(cap: float | None) -> float | None:
+ """The per-call budget, lowered to *cap* when the caller has less to give.
+
+ The router passes what is left of its request-scoped ML budget, so a call
+ -- including the first one of a request, which has no measured cost to be
+ refused on -- stops at a chunk boundary when that budget runs out instead
+ of running to its own, larger, per-call budget.
+ """
+ budget = _time_budget_seconds()
+ if cap is None:
+ return budget
+ cap = max(0.0, cap)
+ return cap if budget is None else min(budget, cap)
+
+
def _canary_threshold_seconds() -> float | None:
"""Startup canary threshold; inference slower than this disables Kompress.
@@ -1616,11 +1631,15 @@ class KompressCompressor(Transform):
# ``compress()`` accepts ``_deadline_started_at``, so a caller that
# compresses many blocks for ONE request can hand every call the same
- # origin and have the deadline bound the request. Duck-typed rather than
+ # origin and have the deadline bound the request. It also accepts
+ # ``_time_budget_cap_seconds``, which lowers this call's time budget to
+ # what the caller's request-scoped ML budget has left. Duck-typed rather than
# isinstance-checked at the call site because ``RemoteKompressCompressor``
- # is the other compressor the router may get back and its ``compress()``
- # does not take the argument.
+ # is the other compressor the router may get back: its ``compress()`` takes
+ # the cap (flagged by ``accepts_time_budget_cap``) but not the deadline
+ # origin.
shares_request_deadline: bool = True
+ accepts_time_budget_cap: bool = True
def __init__(self, config: KompressConfig | None = None):
self.config = config or KompressConfig()
@@ -1762,6 +1781,7 @@ class KompressCompressor(Transform):
allow_download: bool = True,
ccr_original: str | None = None,
_deadline_started_at: float | None = None,
+ _time_budget_cap_seconds: float | None = None,
) -> KompressResult:
"""Compress content using Kompress model.
@@ -1824,6 +1844,7 @@ class KompressCompressor(Transform):
batch_size=_batch_size(),
ccr_originals=[ccr_original],
_deadline_started_at=t_deadline,
+ _time_budget_cap_seconds=_time_budget_cap_seconds,
)
if batch_result:
return batch_result[0]
@@ -1834,7 +1855,7 @@ class KompressCompressor(Transform):
chunk_count = 0
acquire_timeout = _acquire_timeout_seconds()
- budget = _time_budget_seconds()
+ budget = _capped_time_budget(_time_budget_cap_seconds)
deadline = time.monotonic() + budget if budget is not None else None
for chunk_start in range(0, n_words, max_chunk_words):
@@ -2104,6 +2125,7 @@ class KompressCompressor(Transform):
*,
ccr_originals: list[str | None] | None = None,
_deadline_started_at: float | None = None,
+ _time_budget_cap_seconds: float | None = None,
) -> list[KompressResult]:
"""Compress multiple texts. Uses batched inference on GPU, sequential on CPU.
@@ -2198,6 +2220,12 @@ class KompressCompressor(Transform):
# per-item slowdown measured on ONNX CPU (~0.7-0.9x vs sequential).
# GPU users still benefit from the batched forward pass below.
if self._should_use_sequential_fallback():
+ # The cap is for the whole batch, so each item gets what is left.
+ cap_ends_at = (
+ None
+ if _time_budget_cap_seconds is None
+ else time.monotonic() + _time_budget_cap_seconds
+ )
return [
self.compress(
content,
@@ -2207,6 +2235,9 @@ class KompressCompressor(Transform):
target_ratio=r,
ccr_original=ccr_source,
_deadline_started_at=t_deadline,
+ _time_budget_cap_seconds=(
+ None if cap_ends_at is None else cap_ends_at - time.monotonic()
+ ),
)
for content, r, ccr_source in zip(contents, ratios, ccr_sources, strict=True)
]
@@ -2257,7 +2288,7 @@ class KompressCompressor(Transform):
self._deadline_s = deadline_s
acquire_timeout = _acquire_timeout_seconds()
- budget = _time_budget_seconds()
+ budget = _capped_time_budget(_time_budget_cap_seconds)
deadline = time.monotonic() + budget if budget is not None else None
def _bail_remaining(reason: str, batch_start: int) -> None:
diff --git a/headroom/transforms/kompress_remote.py b/headroom/transforms/kompress_remote.py
index 4ebabd88d..1af00349a 100644
--- a/headroom/transforms/kompress_remote.py
+++ b/headroom/transforms/kompress_remote.py
@@ -53,7 +53,12 @@ never correctness.
from __future__ import annotations
+import concurrent.futures
+import json
import logging
+import threading
+import time
+from typing import Any
import httpx
@@ -102,6 +107,21 @@ _MIN_WORDS = 10
# Accept-any-shrink CCR gate, identical to KompressCompressor.compress: only
# store + mark when the shrink is worth the retrieval marker's own cost.
+# A remote call is declined outright when the caller's request-scoped ML budget
+# has this little left: no HTTP round trip (connect + write + server inference +
+# read) fits in it, so starting one only burns the budget and leaves a worker
+# behind. The block passes through unchanged instead.
+_MIN_REMOTE_BUDGET_SECONDS = 0.05
+
+# Workers for budget-capped calls, and the number of admission permits. A call
+# whose caller times out is abandoned, not killed (a sync httpx request cannot be
+# interrupted from another thread), so the worker keeps its permit until the
+# request itself ends -- and every request ends at an absolute deadline (see
+# ``_capped_request``). With permits == workers nothing ever waits in the
+# executor's (unbounded) queue: a saturated caller waits for a permit, not with
+# its payload enqueued, and fails open when its own deadline passes.
+_CAPPED_CALL_WORKERS = 16
+
class RemoteKompressCompressor:
"""Drop-in for KompressCompressor that POSTs to a hosted ``/compress`` endpoint.
@@ -112,6 +132,12 @@ class RemoteKompressCompressor:
name = "kompress_compressor"
+ # ``compress()`` accepts ``_time_budget_cap_seconds``: what the router's
+ # request-scoped ML budget has left. Unlike local Kompress this class does
+ # not take ``_deadline_started_at``, so it does not set
+ # ``shares_request_deadline``; the router checks this flag separately.
+ accepts_time_budget_cap: bool = True
+
def __init__(
self,
endpoint: str,
@@ -120,6 +146,7 @@ class RemoteKompressCompressor:
timeout: float = 20.0,
path: str | None = DEFAULT_ENDPOINT_PATH,
headers: dict[str, str] | None = None,
+ transport: httpx.BaseTransport | None = None,
) -> None:
self.config = config or KompressConfig()
# Default keeps the pre-existing behaviour exactly: <endpoint>/compress.
@@ -148,7 +175,13 @@ class RemoteKompressCompressor:
# Kompress, rather than discovering it in a packet capture.
guard_egress("remote Kompress inference", self._url)
# httpx.Client is safe to share across the proxy's worker threads.
- self._client = httpx.Client(timeout=timeout)
+ # ``transport`` exists for tests (httpx.MockTransport); None is the
+ # normal network transport.
+ self._timeout = timeout
+ self._client = httpx.Client(timeout=timeout, transport=transport)
+ self._capped_pool: concurrent.futures.ThreadPoolExecutor | None = None
+ self._capped_pool_lock = threading.Lock()
+ self._capped_slots = threading.BoundedSemaphore(_CAPPED_CALL_WORKERS)
@property
def url(self) -> str:
@@ -193,9 +226,18 @@ class RemoteKompressCompressor:
*,
allow_download: bool = True,
ccr_original: str | None = None,
+ _time_budget_cap_seconds: float | None = None,
) -> KompressResult:
"""Compress via the remote endpoint.
+ ``_time_budget_cap_seconds`` is what is left of the caller's
+ request-scoped ML budget (ContentRouter passes it). It bounds the HTTP
+ request AS A WHOLE -- connect, write, server time and read together,
+ not each phase -- and a call that runs out passes the block through
+ unchanged. When the remainder is at or below
+ :data:`_MIN_REMOTE_BUDGET_SECONDS` no request is made at all. ``None``
+ (direct callers) keeps the configured per-phase timeout, 20s by default.
+
``ccr_original`` mirrors :meth:`KompressCompressor.compress`: text to
store in CCR instead of ``content``, used when ``content`` is a
tag-protected placeholder intermediate ({{HEADROOM_TAG_N}}). It is
@@ -230,14 +272,23 @@ class RemoteKompressCompressor:
# the hole this guard closes.
guard_egress("remote Kompress inference", self._url)
- try:
- resp = self._client.post(
- self._url,
- headers=self._headers,
- json={"content": content, "target_ratio": target_ratio},
+ payload = {"content": content, "target_ratio": target_ratio}
+ if _time_budget_cap_seconds is not None and (
+ _time_budget_cap_seconds <= _MIN_REMOTE_BUDGET_SECONDS
+ ):
+ logger.info(
+ "Remote Kompress declined: %.3fs of request ML budget left (floor %.3fs); "
+ "passing through unchanged",
+ max(0.0, _time_budget_cap_seconds),
+ _MIN_REMOTE_BUDGET_SECONDS,
)
- resp.raise_for_status()
- data = resp.json()
+ return self._passthrough(content, n_words)
+
+ try:
+ if _time_budget_cap_seconds is None:
+ data = self._post(payload)
+ else:
+ data = self._post_within(payload, _time_budget_cap_seconds)
compressed = data["compressed"]
if not isinstance(compressed, str):
raise TypeError("remote Kompress response field 'compressed' must be a string")
@@ -299,5 +350,95 @@ class RemoteKompressCompressor:
return result
+ def _post(self, payload: dict[str, Any], timeout: float | None = None) -> Any:
+ # No ``timeout`` kwarg on the default path: the client's own (20s)
+ # applies, and the call is exactly what it was before the cap existed.
+ extra: dict[str, Any] = {} if timeout is None else {"timeout": timeout}
+ resp = self._client.post(self._url, headers=self._headers, json=payload, **extra)
+ resp.raise_for_status()
+ return resp.json()
+
+ def _post_within(self, payload: dict[str, Any], cap: float) -> Any:
+ """POST and read the response within *cap* seconds in total.
+
+ httpx timeouts are per phase (connect, write, pool, and each read
+ between chunks), so ``timeout=cap`` alone neither bounds the call nor
+ stops a slow-drip body. Three things together do:
+
+ * **Admission.** A permit (one per worker) is taken before submitting,
+ waiting at most until the caller's deadline. Saturated means fail
+ open, with the payload never enqueued -- ``future.cancel()`` does not
+ free a queued payload, so an unbounded queue would retain every
+ timed-out prompt.
+ * **Caller deadline.** The caller waits for the worker with one overall
+ deadline and passes the block through unchanged when it runs out.
+ * **Worker deadline.** The worker keeps its permit until it actually
+ finishes (released in its own ``finally``, never on caller timeout),
+ and it finishes by an absolute deadline: per-phase timeouts are capped
+ to what is left, and the body is streamed and cut off at the
+ deadline, so a dripping response cannot hold a permit indefinitely.
+ """
+ deadline = time.monotonic() + cap
+ if not self._capped_slots.acquire(timeout=max(0.0, cap)):
+ raise TimeoutError(
+ f"all {_CAPPED_CALL_WORKERS} remote Kompress workers busy for the "
+ f"request ML budget ({cap:.3f}s)"
+ )
+ try:
+ future = self._get_capped_pool().submit(self._capped_request, payload, deadline)
+ except BaseException:
+ self._capped_slots.release() # never reached the worker that would
+ raise
+ try:
+ return future.result(timeout=max(0.0, deadline - time.monotonic()))
+ except concurrent.futures.TimeoutError:
+ raise TimeoutError(
+ f"exceeded the request ML budget ({cap:.3f}s) before the endpoint answered"
+ ) from None
+
+ def _capped_request(self, payload: dict[str, Any], deadline: float) -> Any:
+ """The worker side of :meth:`_post_within`. Owns one admission permit.
+
+ Lifetime is bounded by *deadline* plus at most one per-phase timeout
+ (itself no longer than what was left at start): every httpx phase is
+ capped to the time remaining, and the body is read chunk by chunk with
+ the absolute deadline checked after each one.
+ """
+ try:
+ left = deadline - time.monotonic()
+ if left <= 0.0:
+ raise TimeoutError("request ML budget spent before the request started")
+ with self._client.stream(
+ "POST",
+ self._url,
+ headers=self._headers,
+ json=payload,
+ timeout=min(left, self._timeout),
+ ) as resp:
+ resp.raise_for_status()
+ body = bytearray()
+ for chunk in resp.iter_bytes():
+ body += chunk
+ if time.monotonic() >= deadline:
+ raise TimeoutError(
+ "remote Kompress response outlived the request ML budget"
+ )
+ return json.loads(bytes(body))
+ finally:
+ self._capped_slots.release()
+
+ def _get_capped_pool(self) -> concurrent.futures.ThreadPoolExecutor:
+ with self._capped_pool_lock:
+ if self._capped_pool is None:
+ self._capped_pool = concurrent.futures.ThreadPoolExecutor(
+ max_workers=_CAPPED_CALL_WORKERS,
+ thread_name_prefix="remote-kompress",
+ )
+ return self._capped_pool
+
def close(self) -> None:
+ with self._capped_pool_lock:
+ pool, self._capped_pool = self._capped_pool, None
+ if pool is not None:
+ pool.shutdown(wait=False, cancel_futures=True)
self._client.close()
diff --git a/tests/test_content_router_ml_stage_deadline.py b/tests/test_content_router_ml_stage_deadline.py
new file mode 100644
index 000000000..079ff5a06
--- /dev/null
+++ b/tests/test_content_router_ml_stage_deadline.py
@@ -0,0 +1,678 @@
+"""Request-scoped ML ceiling for the mixed-content path (#3711).
+
+``HEADROOM_KOMPRESS_MAX_TOKENS`` bounds a single block. It cannot bound a
+request: ``_compress_mixed`` splits a payload into sections and calls
+``_try_ml_compressor`` once per section, so every section can sit under the
+per-block ceiling while their sum runs for minutes. The reporter measured ~70s
+on a 1.4MB ``tool_result`` of prose blocks separated by small JSON objects,
+which blew the 30s compression budget and then quarantined compression for
+every following request -- while
+``headroom_kompress_size_gate_total`` recorded only ``within``.
+
+A stub stands in for Kompress so these assert the *budget*, not ONNX latency:
+the real model is not available in CI, and the bug is about how many times a
+slow stage is entered, not how slow it is.
+"""
+
+from __future__ import annotations
+
+import concurrent.futures
+import json
+import random
+import threading
+import time
+from dataclasses import dataclass
+
+import httpx
+import pytest
+
+from headroom.transforms import content_router as cr
+from headroom.transforms.kompress_remote import RemoteKompressCompressor
+
+
+class _Tokenizer:
+ def count_text(self, content: str) -> int:
+ return len(content.split())
+
+
+@dataclass
+class _StubResult:
+ compressed: str
+ compressed_tokens: int
+
+
+class _SlowKompress:
+ """Stands in for Kompress: ready, and costs `per_call_s` every call."""
+
+ def __init__(self, per_call_s: float) -> None:
+ self.per_call_s = per_call_s
+ self.calls = 0
+
+ def is_ready(self) -> bool:
+ return True
+
+ def ensure_background_load(self) -> None: # pragma: no cover - never reached
+ raise AssertionError("stub is always ready")
+
+ def compress(self, text: str, **_kwargs: object) -> _StubResult:
+ self.calls += 1
+ time.sleep(self.per_call_s)
+ out = text[: max(1, len(text) // 2)]
+ return _StubResult(compressed=out, compressed_tokens=cr._estimate_tokens(out))
+
+
+def _prose(n_chars: int, seed: int = 7) -> str:
+ rng = random.Random(seed)
+ words = (
+ "analysis deployment configuration throughput latency pipeline compression "
+ "artifact identifier resolution boundary telemetry inference workspace"
+ ).split()
+ out: list[str] = []
+ total = 0
+ while total < n_chars:
+ line = " ".join(rng.choice(words) for _ in range(14))
+ out.append(line)
+ total += len(line) + 1
+ return "\n".join(out)
+
+
+def _reporter_payload(block_chars: int = 40_000, blocks: int = 8) -> str:
+ """Prose blocks separated by small JSON objects -- the #3711 shape.
+
+ Each block stays under the 50k-token per-block gate, so the size gate
+ cannot fire; only a request-scoped ceiling can stop this.
+ """
+ parts: list[str] = []
+ for i in range(blocks):
+ parts.append(_prose(block_chars, seed=i))
+ parts.append(json.dumps({"id": i, "status": "ok", "note": f"marker {i}"}))
+ return "\n\n".join(parts)
+
+
+def _tool_result_messages() -> list[dict]:
+ """The reported shape: the payload arrives as a tool_result, not user text.
+
+ Plain user text is protected from compression
+ (``router:protected:user_message``) and never reaches the ML stage at all.
+ """
+ return [
+ {"role": "user", "content": "go"},
+ {
+ "role": "assistant",
+ "content": [{"type": "tool_use", "id": "t1", "name": "Bash", "input": {}}],
+ },
+ {
+ "role": "user",
+ "content": [
+ {"type": "tool_result", "tool_use_id": "t1", "content": _reporter_payload()}
+ ],
+ },
+ ]
+
+
+def gate_outcomes(monkeypatch) -> dict[str, int]:
+ seen: dict[str, int] = {}
+ monkeypatch.setattr(
+ cr.ContentRouter,
+ "_observe_kompress_size_gate",
+ lambda self, outcome: seen.__setitem__(outcome, seen.get(outcome, 0) + 1),
+ )
+ return seen
+
+
+def slow_kompress(monkeypatch) -> _SlowKompress:
+ stub = _SlowKompress(per_call_s=0.05)
+ monkeypatch.setattr(cr.ContentRouter, "_get_kompress", lambda self: stub)
+ return stub
+
+
+def _apply(router: cr.ContentRouter) -> None:
+ router.apply(
+ _tool_result_messages(),
+ _Tokenizer(),
+ frozen_message_count=1,
+ min_tokens_to_compress=1,
+ )
+
+
+def test_size_gate_alone_cannot_bound_a_request(gate_outcomes, slow_kompress, monkeypatch) -> None:
+ """Every section passes the per-block gate -- this is the #3711 precondition.
+
+ Reproduces the reporter's metric exactly: many ``within`` decisions and no
+ ``exceeded``. If this ever records ``exceeded`` the payload has stopped
+ reproducing the report, and the deadline test below would pass for the
+ wrong reason.
+ """
+ monkeypatch.setenv("HEADROOM_ML_STAGE_DEADLINE_SECONDS", "0") # isolate the size gate
+
+ _apply(cr.ContentRouter(cr.ContentRouterConfig()))
+
+ assert gate_outcomes.get("exceeded", 0) == 0, (
+ f"payload no longer reproduces #3711 (a section exceeded the gate): {gate_outcomes}"
+ )
+ assert gate_outcomes.get("within", 0) > 1, (
+ "expected repeated per-section ML entry, the shape that blows the budget; "
+ f"got {gate_outcomes}"
+ )
+ assert slow_kompress.calls > 1, "the ML stage was entered once or not at all"
+
+
+def test_ml_deadline_stops_further_ml_work_once_the_budget_is_spent(
+ gate_outcomes, slow_kompress, monkeypatch
+) -> None:
+ """Past the ceiling, remaining sections route off ML instead of compounding."""
+ monkeypatch.setenv("HEADROOM_ML_STAGE_DEADLINE_SECONDS", "0.06")
+
+ _apply(cr.ContentRouter(cr.ContentRouterConfig()))
+
+ assert gate_outcomes.get("deadline", 0) > 0, (
+ f"request-scoped ceiling never fired: {gate_outcomes}"
+ )
+
+ # The saving has to be real, so measure it: the same payload with the
+ # ceiling disabled must enter the model strictly more often.
+ bounded_calls = slow_kompress.calls
+ monkeypatch.setenv("HEADROOM_ML_STAGE_DEADLINE_SECONDS", "0")
+ unbounded = _SlowKompress(per_call_s=0.05)
+ monkeypatch.setattr(cr.ContentRouter, "_get_kompress", lambda self: unbounded)
+
+ _apply(cr.ContentRouter(cr.ContentRouterConfig()))
+
+ assert bounded_calls < unbounded.calls, (
+ f"ceiling skipped no model calls: {bounded_calls} with it, {unbounded.calls} without"
+ )
+
+
+def test_deadline_zero_restores_previous_unbounded_behaviour(
+ gate_outcomes, slow_kompress, monkeypatch
+) -> None:
+ monkeypatch.setenv("HEADROOM_ML_STAGE_DEADLINE_SECONDS", "0")
+
+ _apply(cr.ContentRouter(cr.ContentRouterConfig()))
+
+ assert gate_outcomes.get("deadline", 0) == 0, (
+ f"deadline fired although disabled: {gate_outcomes}"
+ )
+
+
+def test_direct_compress_callers_stay_unarmed() -> None:
+ """``compress()`` without ``apply()`` keeps the old unbounded behaviour.
+
+ Tests and the ``/v1/compress`` path call ``compress()`` directly; they must
+ not inherit a request budget nobody set.
+ """
+ router = cr.ContentRouter(cr.ContentRouterConfig())
+ state = router._runtime_state_var.get()
+ assert state.ml_budget_active is False
+
+ # That default is instance-scoped and shared by every direct call for the
+ # life of the router, so charging it would accumulate across unrelated
+ # calls until ML switched itself off permanently.
+ router._charge_ml_time(5.0, "some text to compress")
+ assert state.ml_elapsed == 0.0
+
+
+# --------------------------------------------------------------------------- #
+# Review follow-ups: the budget must mean ML time, must refuse a call it cannot
+# afford, and must sit under the timeout that quarantines the stage.
+#
+# The original tests asserted that LATER model calls are skipped. That is not
+# the same claim as the request-wide time bound the module documents, and all
+# three gaps below passed those tests.
+# --------------------------------------------------------------------------- #
+
+
+class _SlowTokenizer(_Tokenizer):
+ """Expensive NON-ML prework: token counting during classification."""
+
+ def __init__(self, per_call_s: float) -> None:
+ self.per_call_s = per_call_s
+ self.calls = 0
+
+ def count_text(self, content: str) -> int:
+ self.calls += 1
+ time.sleep(self.per_call_s)
+ return super().count_text(content)
+
+
+def charged(monkeypatch) -> list[float]:
+ """Every increment billed to the ML budget, in order."""
+ seen: list[float] = []
+ real = cr.ContentRouter._charge_ml_time
+
+ def _spy(self, elapsed: float, text: str) -> None:
+ seen.append(elapsed)
+ real(self, elapsed, text)
+
+ monkeypatch.setattr(cr.ContentRouter, "_charge_ml_time", _spy)
+ return seen
+
+
+def test_non_ml_prework_does_not_consume_the_ml_budget(
+ gate_outcomes, slow_kompress, charged, monkeypatch
+) -> None:
+ """The budget was armed at ``apply()`` entry, so lifecycle work, message
+ classification and lossless transforms all ran it down before ML began --
+ while the log line claimed to measure time "in ML"."""
+ monkeypatch.setenv("HEADROOM_ML_STAGE_DEADLINE_SECONDS", "0.4")
+ router = cr.ContentRouter(cr.ContentRouterConfig(enable_kompress=True))
+
+ tokenizer = _SlowTokenizer(per_call_s=0.01)
+ started = time.monotonic()
+ router.apply(
+ _tool_result_messages(),
+ tokenizer,
+ frozen_message_count=1,
+ min_tokens_to_compress=1,
+ )
+ wall = time.monotonic() - started
+
+ assert tokenizer.calls > 0, "prework must actually have run"
+ assert slow_kompress.calls > 0, "prework time must not have spent the ML budget"
+ # The bound is on ML time, not on the call.
+ assert sum(charged) <= wall
+ assert sum(charged) == pytest.approx(slow_kompress.calls * slow_kompress.per_call_s, rel=0.5)
+
+
+def test_a_slow_call_cannot_be_admitted_against_a_budget_it_will_overrun(
+ gate_outcomes, charged, monkeypatch
+) -> None:
+ """Admission used to be ``now >= deadline`` only, so a call starting at
+ 14.9s of a 15s budget ran to completion well past it -- the ONNX worker is
+ not preemptible, so nothing stops it once it has begun."""
+ monkeypatch.setenv("HEADROOM_ML_STAGE_DEADLINE_SECONDS", "0.5")
+ stub = _SlowKompress(per_call_s=0.35)
+ monkeypatch.setattr(cr.ContentRouter, "_get_kompress", lambda self: stub)
+ router = cr.ContentRouter(cr.ContentRouterConfig(enable_kompress=True))
+
+ _apply(router)
+
+ assert stub.calls >= 1, "the first block has no measurement yet and must run"
+ # The point of the change: total ML time stays inside the budget, rather
+ # than the budget merely being the moment after which no call STARTS.
+ assert sum(charged) <= 0.5, (
+ f"ML overran its 0.5s budget: {sum(charged):.2f}s across {stub.calls} calls"
+ )
+ assert gate_outcomes.get("deadline", 0) >= 1, "remaining blocks must route off ML"
+
+
+def test_budget_is_clamped_below_the_timeout_that_quarantines_the_stage(
+ monkeypatch,
+) -> None:
+ """A 15s guard under a 10s executor timeout can never fire first, which is
+ exactly the failure the ceiling exists to prevent."""
+ monkeypatch.delenv("HEADROOM_ML_STAGE_DEADLINE_SECONDS", raising=False)
+
+ monkeypatch.setenv("HEADROOM_COMPRESSION_TIMEOUT_SECONDS", "30")
+ assert cr._ml_stage_deadline_seconds() == 15.0, "the documented default is unchanged"
+
+ monkeypatch.setenv("HEADROOM_COMPRESSION_TIMEOUT_SECONDS", "10")
+ clamped = cr._ml_stage_deadline_seconds()
+ assert clamped == 5.0
+ assert clamped < 10.0, "must leave room to degrade before the executor gives up"
+
+ # The clamp only ever lowers: an explicit request below it is honoured.
+ monkeypatch.setenv("HEADROOM_ML_STAGE_DEADLINE_SECONDS", "2")
+ assert cr._ml_stage_deadline_seconds() == 2.0
+
+ # And 0 still disables the ceiling entirely.
+ monkeypatch.setenv("HEADROOM_ML_STAGE_DEADLINE_SECONDS", "0")
+ assert cr._ml_stage_deadline_seconds() == 0.0
+
+
+def test_a_failing_ml_call_still_draws_down_the_budget(charged, monkeypatch) -> None:
+ """Otherwise a request could retry its way past the ceiling: a compressor
+ that raises burns the same non-preemptible wall clock as one that returns."""
+ monkeypatch.setenv("HEADROOM_ML_STAGE_DEADLINE_SECONDS", "5")
+
+ class _Exploding(_SlowKompress):
+ def compress(self, text: str, **_kwargs: object):
+ self.calls += 1
+ time.sleep(self.per_call_s)
+ raise RuntimeError("onnx said no")
+
+ stub = _Exploding(per_call_s=0.05)
+ monkeypatch.setattr(cr.ContentRouter, "_get_kompress", lambda self: stub)
+ router = cr.ContentRouter(cr.ContentRouterConfig(enable_kompress=True))
+
+ _apply(router)
+
+ assert stub.calls > 0
+ assert charged, "a raising call must still be billed"
+ assert sum(charged) > 0.0
+
+
+# --------------------------------------------------------------------------- #
+# Second review: the FIRST ML call of a request was always admitted (the seed
+# rate was 0, and there is no measurement yet), and it is non-preemptible, so
+# one huge first block could still run past the ML budget and the executor
+# timeout. The call is now capped rather than estimated.
+# --------------------------------------------------------------------------- #
+
+
+class _CooperativeKompress(_SlowKompress):
+ """Behaves like Kompress under a time cap: stops at a chunk boundary once
+ ``_time_budget_cap_seconds`` is spent and returns the block unchanged."""
+
+ shares_request_deadline = True
+
+ def __init__(self, worst_case_s: float, chunk_s: float = 0.02) -> None:
+ super().__init__(per_call_s=worst_case_s)
+ self.chunk_s = chunk_s
+ self.caps: list[float | None] = []
+
+ def compress(self, text: str, **kwargs: object) -> _StubResult:
+ self.calls += 1
+ cap = kwargs.get("_time_budget_cap_seconds")
+ self.caps.append(cap) # type: ignore[arg-type]
+ ends_at = None if cap is None else time.monotonic() + float(cap) # type: ignore[arg-type]
+ spent = 0.0
+ while spent < self.per_call_s:
+ if ends_at is not None and time.monotonic() >= ends_at:
+ return _StubResult(compressed=text, compressed_tokens=cr._estimate_tokens(text))
+ time.sleep(self.chunk_s)
+ spent += self.chunk_s
+ out = text[: max(1, len(text) // 2)]
+ return _StubResult(compressed=out, compressed_tokens=cr._estimate_tokens(out))
+
+
+def test_first_ml_call_is_capped_by_the_budget_it_was_admitted_against(
+ charged, monkeypatch
+) -> None:
+ """A first block whose worst case (3s) exceeds both the outer timeout (1s)
+ and the ML budget derived from it (0.5s) must still stop inside the budget."""
+ monkeypatch.delenv("HEADROOM_ML_STAGE_DEADLINE_SECONDS", raising=False)
+ monkeypatch.setenv("HEADROOM_COMPRESSION_TIMEOUT_SECONDS", "1")
+ budget = cr._ml_stage_deadline_seconds()
+ assert budget == 0.5
+
+ stub = _CooperativeKompress(worst_case_s=3.0)
+ monkeypatch.setattr(cr.ContentRouter, "_get_kompress", lambda self: stub)
+
+ _apply(cr.ContentRouter(cr.ContentRouterConfig(enable_kompress=True)))
+
+ assert stub.caps, "the first block must still reach the model"
+ first_cap = stub.caps[0]
+ assert first_cap is not None and 0.0 < first_cap <= budget, (
+ f"first call was not capped by the ML budget: {stub.caps}"
+ )
+ # One chunk of slack: the cap is checked at chunk boundaries, as in Kompress.
+ assert sum(charged) <= budget + 0.2, (
+ f"ML ran {sum(charged):.2f}s against a {budget}s budget under a 1s executor timeout"
+ )
+
+
+def test_blocks_refused_by_the_budget_pass_through_unchanged(monkeypatch) -> None:
+ """Not the size gate's lossy fallback: the refusal depends on the request,
+ the router caches the result, and only raw bytes stay the same on later
+ turns, so the prompt cache holds."""
+ router = cr.ContentRouter(cr.ContentRouterConfig(enable_kompress=True))
+ router._runtime_state_var.set(
+ cr._PerRequestRuntimeState(ml_budget_active=True, ml_elapsed=999.0)
+ )
+
+ class _Crusher:
+ def compress(self, text: str, context: str = "") -> _StubResult:
+ return _StubResult(compressed="crushed", compressed_tokens=1)
+
+ monkeypatch.setattr(router, "_get_text_crusher", lambda: _Crusher())
+ stub = _SlowKompress(per_call_s=0.0)
+ monkeypatch.setattr(router, "_get_kompress", lambda: stub)
+
+ text = _prose(2_000)
+ out, _ = router._try_ml_compressor(text, "")
+
+ assert out == text
+ assert stub.calls == 0
+
+
+# --------------------------------------------------------------------------- #
+# Third review: the cap reached only LOCAL Kompress. RemoteKompressCompressor
+# got no cap and kept its independent 20s per-phase HTTP timeouts, so a cold
+# remote call -- no measured rate, so nothing to refuse it on -- could run past
+# both the ML budget and the outer timeout. These drive the real remote class
+# through the public apply() with an httpx.MockTransport (no network).
+# --------------------------------------------------------------------------- #
+
+
+def _tool_result_text(messages: list[dict]) -> str:
+ return messages[-1]["content"][0]["content"]
+
+
+def _assert_prose_blocks_unchanged(messages: list[dict]) -> None:
+ """Every ML-eligible prose block reaches the provider byte-identical.
+
+ (The small JSON separators go through the lossless JSON path, which
+ minifies them; that is not ML and not what is under test.)
+ """
+ out = _tool_result_text(messages)
+ for i in range(8):
+ assert _prose(40_000, seed=i) in out, f"prose block {i} was altered"
+
+
+def _remote_kompress(handler) -> RemoteKompressCompressor:
+ return RemoteKompressCompressor(
+ endpoint="http://kompress.invalid",
+ transport=httpx.MockTransport(handler),
+ )
+
+
+def _compressed_reply(request: httpx.Request) -> httpx.Response:
+ content = json.loads(request.content)["content"]
+ return httpx.Response(200, json={"compressed": " ".join(content.split()[::2])})
+
+
+def test_cold_remote_call_cannot_overrun_a_tiny_ml_budget(charged, monkeypatch) -> None:
+ """The reviewer's probe: 10 ms ML budget, 40 ms outer timeout, an endpoint
+ that takes 100 ms. No HTTP round trip fits in what is left, so the remote
+ call is declined and every block passes through unchanged."""
+ monkeypatch.setenv("HEADROOM_ML_STAGE_DEADLINE_SECONDS", "0.01")
+ monkeypatch.setenv("HEADROOM_COMPRESSION_TIMEOUT_SECONDS", "0.04")
+ assert cr._ml_stage_deadline_seconds() == 0.01
+
+ requests: list[httpx.Request] = []
+
+ def _slow_endpoint(request: httpx.Request) -> httpx.Response:
+ requests.append(request)
+ time.sleep(0.1)
+ return _compressed_reply(request)
+
+ remote = _remote_kompress(_slow_endpoint)
+ monkeypatch.setattr(cr.ContentRouter, "_get_kompress", lambda self: remote)
+ router = cr.ContentRouter(cr.ContentRouterConfig(enable_kompress=True))
+
+ messages = _tool_result_messages()
+ result = router.apply(messages, _Tokenizer(), frozen_message_count=1, min_tokens_to_compress=1)
+
+ assert charged, "the remote compressor must still be reached and billed"
+ assert sum(charged) < 0.04, f"remote ML ran {sum(charged):.3f}s under a 40 ms outer timeout"
+ assert requests == [], "a call that cannot fit the budget must not be sent"
+ _assert_prose_blocks_unchanged(result.messages)
+
+
+def test_cold_remote_call_is_bounded_as_a_whole_by_the_ml_budget(charged, monkeypatch) -> None:
+ """With enough budget to try, the first (unmeasured) remote call is capped
+ by the remainder -- over the whole request, not per httpx phase -- and an
+ endpoint slower than that leaves the block unchanged."""
+ budget = 0.2
+ monkeypatch.setenv("HEADROOM_ML_STAGE_DEADLINE_SECONDS", str(budget))
+ monkeypatch.setenv("HEADROOM_COMPRESSION_TIMEOUT_SECONDS", "30")
+
+ release = threading.Event()
+ timeouts: list[dict] = []
+
+ def _hung_endpoint(request: httpx.Request) -> httpx.Response:
+ timeouts.append(dict(request.extensions["timeout"]))
+ release.wait(5.0) # far past the budget; MockTransport ignores httpx timeouts
+ return _compressed_reply(request)
+
+ remote = _remote_kompress(_hung_endpoint)
+ monkeypatch.setattr(cr.ContentRouter, "_get_kompress", lambda self: remote)
+ router = cr.ContentRouter(cr.ContentRouterConfig(enable_kompress=True))
+
+ messages = _tool_result_messages()
+ try:
+ result = router.apply(
+ messages, _Tokenizer(), frozen_message_count=1, min_tokens_to_compress=1
+ )
+ finally:
+ release.set()
+ remote.close()
+
+ assert timeouts, "the first block must reach the endpoint"
+ assert all(0.0 < t <= budget for t in timeouts[0].values()), (
+ f"per-phase timeouts were not lowered to the budget: {timeouts[0]}"
+ )
+ assert sum(charged) <= budget + 0.1, (
+ f"remote ML ran {sum(charged):.3f}s against a {budget}s budget"
+ )
+ _assert_prose_blocks_unchanged(result.messages)
+
+
+def test_remote_kompress_keeps_its_default_timeout_without_a_budget() -> None:
+ """Direct callers pass no cap: same single request, 20s per-phase timeouts."""
+ timeouts: list[dict] = []
+
+ def _endpoint(request: httpx.Request) -> httpx.Response:
+ timeouts.append(dict(request.extensions["timeout"]))
+ return _compressed_reply(request)
+
+ remote = _remote_kompress(_endpoint)
+ text = _prose(2_000)
+ out = remote.compress(text)
+
+ assert out.compressed != text
+ assert timeouts == [{"connect": 20.0, "read": 20.0, "write": 20.0, "pool": 20.0}]
+ assert remote._capped_pool is None, "the default path must not hop threads"
+
+
+# --------------------------------------------------------------------------- #
+# Fourth review: the capped worker pool had no admission bound. Its queue is
+# unbounded and ``future.cancel()`` does not drop a queued payload, so once
+# every worker was held by an abandoned request, each further capped call
+# queued its full prompt, timed out, and left it there. A slow-drip body could
+# hold a worker forever, since httpx read timeouts reset on every chunk.
+# --------------------------------------------------------------------------- #
+
+
+def _free_permits(remote: RemoteKompressCompressor) -> int:
+ return remote._capped_slots._value # type: ignore[attr-defined]
+
+
+def _wait_until(predicate, timeout_s: float = 3.0) -> bool:
+ ends = time.monotonic() + timeout_s
+ while time.monotonic() < ends:
+ if predicate():
+ return True
+ time.sleep(0.01)
+ return predicate()
+
+
+def test_saturated_remote_calls_fail_open_without_queueing_their_payloads() -> None:
+ """The reviewer's repro: hold all 16 workers with requests whose 200 ms
+ callers have already timed out, then make five more calls with 60 ms caps.
+ Each must fail open within its cap with its payload never enqueued, and
+ capacity must come back once the held requests end."""
+ from headroom.transforms.kompress_remote import _CAPPED_CALL_WORKERS
+
+ release = threading.Event()
+ reached: list[str] = []
+
+ def _stalled_endpoint(request: httpx.Request) -> httpx.Response:
+ reached.append(json.loads(request.content)["content"])
+ release.wait(5.0)
+ return _compressed_reply(request)
+
+ remote = _remote_kompress(_stalled_endpoint)
+ text = _prose(2_000)
+ try:
+ with concurrent.futures.ThreadPoolExecutor(_CAPPED_CALL_WORKERS) as callers:
+ held = list(
+ callers.map(
+ lambda i: remote.compress(f"held{i} {text}", _time_budget_cap_seconds=0.2),
+ range(_CAPPED_CALL_WORKERS),
+ )
+ )
+ assert all(out.compressed == f"held{i} {text}" for i, out in enumerate(held))
+ assert len(reached) == _CAPPED_CALL_WORKERS
+ assert _free_permits(remote) == 0, "abandoned requests must keep their permits"
+
+ for i in range(5):
+ late = f"late{i} {text}"
+ started = time.monotonic()
+ out = remote.compress(late, _time_budget_cap_seconds=0.06)
+ took = time.monotonic() - started
+ assert out.compressed == late, "a saturated call must fail open unchanged"
+ assert took < 0.06 + 0.1, f"saturated call took {took:.3f}s against a 60 ms cap"
+ assert remote._capped_pool._work_queue.qsize() == 0, "a payload was enqueued"
+
+ assert len(reached) == _CAPPED_CALL_WORKERS, "a saturated call reached the endpoint"
+ finally:
+ release.set()
+
+ assert _wait_until(lambda: _free_permits(remote) == _CAPPED_CALL_WORKERS)
+ recovered = remote.compress(text, _time_budget_cap_seconds=2.0)
+ assert recovered.compressed != text, "capacity must return once the held requests end"
+ remote.close()
+
+
+def test_slow_drip_response_is_cut_off_at_an_absolute_deadline() -> None:
+ """A body that keeps dripping (a chunk every 50 ms for 2 s) resets httpx's
+ read timeout on every chunk. The worker must stop at the cap's absolute
+ deadline and give its permit back, not ride the response to the end."""
+ from headroom.transforms.kompress_remote import _CAPPED_CALL_WORKERS
+
+ chunks_sent: list[int] = []
+
+ def _dripping_endpoint(request: httpx.Request) -> httpx.Response:
+ reply = json.dumps({"compressed": "short"}).encode()
+
+ def _drip():
+ for i in range(40):
+ chunks_sent.append(i)
+ time.sleep(0.05)
+ yield b" " # JSON-legal leading whitespace
+ yield reply
+
+ return httpx.Response(200, content=_drip())
+
+ remote = _remote_kompress(_dripping_endpoint)
+ text = _prose(2_000)
+ cap = 0.2
+ started = time.monotonic()
+ out = remote.compress(text, _time_budget_cap_seconds=cap)
+ assert out.compressed == text
+ assert time.monotonic() - started < cap + 0.1
+
+ assert _wait_until(lambda: _free_permits(remote) == _CAPPED_CALL_WORKERS, timeout_s=1.0), (
+ "the dripping request still holds its permit"
+ )
+ assert time.monotonic() - started < 1.0, "the worker rode the drip instead of the deadline"
+ assert len(chunks_sent) < 10, f"read {len(chunks_sent)} chunks past a {cap}s deadline"
+ remote.close()
+
+
+def test_permit_is_held_until_the_worker_finishes_not_the_caller() -> None:
+ """Releasing on caller timeout would let abandoned requests pile up again;
+ the permit belongs to the running request."""
+ from headroom.transforms.kompress_remote import _CAPPED_CALL_WORKERS
+
+ release = threading.Event()
+
+ def _stalled_endpoint(request: httpx.Request) -> httpx.Response:
+ release.wait(5.0)
+ return _compressed_reply(request)
+
+ remote = _remote_kompress(_stalled_endpoint)
+ text = _prose(2_000)
+ try:
+ out = remote.compress(text, _time_budget_cap_seconds=0.1)
+ assert out.compressed == text
+ assert _free_permits(remote) == _CAPPED_CALL_WORKERS - 1
+ finally:
+ release.set()
+ assert _wait_until(lambda: _free_permits(remote) == _CAPPED_CALL_WORKERS)
+ remote.close()
diff --git a/tests/test_transforms/test_kompress_deadline.py b/tests/test_transforms/test_kompress_deadline.py
index 2af170040..612b58bcb 100644
--- a/tests/test_transforms/test_kompress_deadline.py
+++ b/tests/test_transforms/test_kompress_deadline.py
@@ -218,3 +218,66 @@ def test_a_compressor_that_cannot_take_the_deadline_is_not_handed_one(monkeypatc
assert fake.deadlines, "the stub should still have been called"
assert all(d is None for d in fake.deadlines), fake.deadlines
+
+
+def _chunked_kompress(monkeypatch, chunk_s: float):
+ """Kompress over a fake model; each chunk advances a fake monotonic clock."""
+ clock = {"now": 0.0, "chunks": 0}
+
+ class _Enc(dict):
+ def word_ids(self, batch_index=0):
+ return self["_word_ids"]
+
+ class _Tok:
+ def __call__(self, chunk_words, **kw):
+ n = len(chunk_words)
+ return _Enc(input_ids=[[0] * n], attention_mask=[[1] * n], _word_ids=list(range(n)))
+
+ class _Model:
+ def get_keep_mask(self, input_ids, attention_mask):
+ clock["now"] += chunk_s
+ clock["chunks"] += 1
+ n = len(input_ids[0])
+ return [[i < n // 2 for i in range(n)]]
+
+ def get_scores(self, input_ids, attention_mask): # the batched path
+ return [
+ [0.9 if keep else 0.1 for keep in row]
+ for row in self.get_keep_mask(input_ids, attention_mask)
+ ]
+
+ monkeypatch.setattr(kc.time, "monotonic", lambda: clock["now"])
+ monkeypatch.setattr(kc, "_load_kompress", lambda *a, **k: (_Model(), _Tok(), "onnx"))
+ monkeypatch.setattr(kc, "_model_device_type", lambda *a, **k: "cpu")
+ monkeypatch.setenv("HEADROOM_COMPRESSION_DEADLINE_MS", "0")
+ monkeypatch.delenv(kc.KOMPRESS_TIME_BUDGET_ENV, raising=False)
+
+ comp = kc.KompressCompressor(kc.KompressConfig(min_input_words=10))
+ comp.config.chunk_words = 100
+ monkeypatch.setattr(comp, "_should_batch_single_content", lambda *a, **k: False)
+ monkeypatch.setattr(comp, "_should_use_sequential_fallback", lambda: False)
+ return comp, clock
+
+
+def test_time_budget_cap_stops_a_call_and_returns_it_unchanged(monkeypatch):
+ """The router hands Kompress what is left of its ML budget, so the first
+ call of a request -- which has no measured cost to be refused on -- stops
+ at a chunk boundary instead of running to the 20s per-call budget. The
+ block comes back byte-identical, so it is stable on later turns."""
+ comp, clock = _chunked_kompress(monkeypatch, chunk_s=0.6)
+ content = " ".join(f"w{i}" for i in range(500)) # five chunks, 3.0s uncapped
+
+ result = comp.compress(content, _time_budget_cap_seconds=1.0)
+
+ assert clock["chunks"] == 2, "must stop at the first chunk boundary past the cap"
+ assert result.compressed == content
+
+
+def test_time_budget_cap_bounds_the_batched_path_too(monkeypatch):
+ comp, clock = _chunked_kompress(monkeypatch, chunk_s=0.6)
+ content = " ".join(f"w{i}" for i in range(500))
+
+ [result] = comp.compress_batch([content], batch_size=1, _time_budget_cap_seconds=1.0)
+
+ assert clock["chunks"] == 2
+ assert result.compressed == content