Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(router): bound ML compression per request, not just per block (#3721)
Tejas Chopraf37980f7f761
diff --git a/headroom/transforms/content_router.py b/headroom/transforms/content_router.pyindex 848cb81fe..c9fac792a 100644--- a/headroom/transforms/content_router.py+++ b/headroom/transforms/content_router.py@@ -166,6 +166,68 @@ def _estimate_tokens(text: str) -> int:return max(1, _TOKEN_ESTIMATOR.count_text(text))+#: Share of the outer compression timeout the ML stage may consume. The+#: remainder has to cover this request's non-ML work AND the overrun of the one+#: ML call that can still be running when the ceiling is reached -- the worker+#: is not preemptible, so the ceiling can only stop the NEXT call.+_ML_STAGE_BUDGET_FRACTION = 0.5+++def _ml_stage_deadline_seconds() -> float:+ """Wall-clock ceiling for ALL ML (Kompress) work inside one request.++ ``_kompress_max_tokens`` bounds a single block. It cannot bound a request:+ the mixed-content path splits a payload into sections and calls+ ``_try_ml_compressor`` once per section (see ``_compress_mixed``), so every+ section can sit under the per-block ceiling while their sum runs for+ minutes. #3711 reported ~70s on a 1.4MB tool_result of prose blocks+ separated by small JSON objects -- with the size gate recording only+ ``within`` -- which blew the 30s compression budget and then quarantined+ compression for every following request.++ So the guard has to match the budget's granularity: once this much time+ has gone into ML for this request, remaining blocks pass through unchanged.+ Raw rather than a cheaper lossy fallback: the router caches that verdict,+ so the block keeps the same bytes on later turns and the prompt cache+ holds.++ The point of the ceiling is to degrade to a partial compression *before*+ the executor abandons a non-preemptible worker, so it is worthless unless+ it sits below the deadline that does the abandoning. A bare default could+ not promise that: an operator who lowers+ ``HEADROOM_COMPRESSION_TIMEOUT_SECONDS`` below 15s got a guard that could+ never fire first -- exactly the failure this exists to prevent. The budget+ is therefore clamped to :data:`_ML_STAGE_BUDGET_FRACTION` of the effective+ outer timeout, leaving the remainder for the non-ML work in the same+ request (lossless transforms, assembly) plus the overrun of the one ML call+ already in flight when the ceiling is reached.++ The outer budget read here is ``HEADROOM_COMPRESSION_TIMEOUT_SECONDS``, the+ executor timeout that quarantines the stage -- NOT+ ``HEADROOM_COMPRESSION_DEADLINE_MS``, which bounds a different, softer+ in-router deadline (see ``_compression_deadline_seconds``).++ ``0`` disables the ceiling (previous behavior), and an explicit request+ below the clamp is honoured -- the clamp only ever lowers.+ """+ try:+ configured = max(+ 0.0,+ float(os.environ.get("HEADROOM_ML_STAGE_DEADLINE_SECONDS", "15")),+ )+ except ValueError:+ configured = 15.0+ if configured <= 0.0:+ return 0.0+ try:+ outer = float(os.environ.get("HEADROOM_COMPRESSION_TIMEOUT_SECONDS", "30"))+ except ValueError:+ outer = 30.0+ if outer <= 0.0:+ return configured+ return min(configured, outer * _ML_STAGE_BUDGET_FRACTION)++def _compression_deadline_seconds() -> float:try:return max(@@ -2047,6 +2109,22 @@ class _PerRequestRuntimeState:compression_policy: Any = Nonetarget_ratio: float | None = None+ # Whether an `apply()` call owns this state and the ML budget therefore+ # applies. False on the INSTANCE-SCOPED DEFAULT that direct `compress()`+ # callers read (see `ContentRouter.__init__`): that object is shared by+ # every such call for the life of the router, so charging it would+ # accumulate across unrelated calls until ML switched itself off forever.+ # Direct callers keep the old unbounded behaviour.+ ml_budget_active: bool = False+ # Seconds this request has spent INSIDE the ML compressor (#3711).+ # Cumulative rather than a wall-clock deadline stamped at `apply()` entry:+ # the budget is a statement about ML work, so time spent on lifecycle,+ # classification and lossless transforms must not draw it down.+ ml_elapsed: float = 0.0+ # Longest single ML call observed in THIS request, seconds per estimated+ # token. Used to project whether the next block can finish inside what is+ # left; see `_try_ml_compressor`.+ ml_seconds_per_token: float = 0.0force_kompress: bool = Falseskip_kompress: bool = Falsekompress_model: str | None = None@@ -4378,6 +4456,28 @@ class ContentRouter(Transform):return Nonereturn elided, _estimate_tokens(elided)+ def _charge_ml_time(self, elapsed: float, text: str) -> None:+ """Bill *elapsed* seconds of ML work to this request's budget.++ Also records the worst seconds-per-token seen so far, which is what the+ admission check in :meth:`_try_ml_compressor` projects the NEXT block+ with. Worst case rather than an average: the ceiling exists to stop a+ single long call from overrunning the outer timeout, and an average+ would let one slow block hide behind several fast ones.++ Best-effort -- accounting must never break a compression that worked.+ """+ try:+ state = self._runtime_state_var.get()+ if not state.ml_budget_active:+ return+ state.ml_elapsed += max(0.0, elapsed)+ tokens = _estimate_tokens(text)+ if tokens > 0 and elapsed > 0.0:+ state.ml_seconds_per_token = max(state.ml_seconds_per_token, elapsed / tokens)+ except Exception: # pragma: no cover - defensive+ logger.debug("ML budget accounting failed", exc_info=True)+def _embedded_json_route(self, content: str, context: str, question: str | None, bias: float) -> str | None:@@ -4468,6 +4568,60 @@ class ContentRouter(Transform):# (under the 50,000 cap, gate silent) but 55,557 estimator tokens, 11%# over. That is exactly the >30s non-preemptible ONNX inference this gate# exists to prevent (#1171).+ # Request-scoped ML ceiling (#3711). The size check below bounds ONE+ # block; this bounds the request. Without it a payload split into many+ # individually-small sections calls this function once per section, each+ # passing the size gate, until the sum blows the compression budget and+ # quarantines the stage for every later request. Checked before the size+ # gate so an over-budget request stops paying the estimator too.+ # An admission check that only asks "is the budget gone yet?" lets a+ # call start at 14.9s of a 15s budget and run for another minute: the+ # ONNX worker is not preemptible, so nothing stops it once it begins,+ # and it can overrun the outer executor timeout the ceiling exists to+ # protect. So project this block's cost and refuse it unless it can+ # plausibly FINISH inside what is left.+ #+ # The projection uses the worst seconds-per-token this request has+ # actually measured. The first block of a request has no measurement,+ # so it cannot be refused on one -- instead the call itself is capped:+ # what is left of the budget goes to Kompress as its per-call time+ # budget (below), and Kompress stops at the next chunk boundary when+ # it runs out, returning the block unchanged. That holds for every+ # call, so a projection that guessed low cannot overrun either.+ _ml_state = self._runtime_state_var.get()+ _ml_budget = _ml_stage_deadline_seconds() if _ml_state.ml_budget_active else 0.0+ _remaining = (_ml_budget - _ml_state.ml_elapsed) if _ml_budget > 0 else None+ # Exhausted is decided before any estimator work, so an over-budget+ # request stops paying for tokenization too.+ _over_budget = _remaining is not None and _remaining <= 0.0+ _block_tokens = -1+ _projected = 0.0+ if _remaining is not None and not _over_budget:+ _rate = _ml_state.ml_seconds_per_token+ if _rate > 0.0:+ _block_tokens = _estimate_tokens(text_to_compress)+ _projected = _block_tokens * _rate+ _over_budget = _projected > _remaining+ if _over_budget:+ self._observe_kompress_size_gate("deadline")+ if _block_tokens < 0:+ _block_tokens = _estimate_tokens(text_to_compress)+ logger.info(+ "ML stage budget spent (%.1fs of %.1fs used, ~%.1fs projected for a "+ "~%d tok block, %.1fs left); passing remaining blocks through "+ "unchanged. Partial compression beats a blown budget.",+ _ml_state.ml_elapsed,+ _ml_budget,+ _projected,+ _block_tokens,+ max(0.0, _remaining or 0.0),+ )+ # Unchanged, not the size gate's lossy fallback below: this+ # verdict depends on the request, not the content, and the router+ # caches it, so the bytes must be ones it can keep sending on+ # later turns without busting the prompt cache.+ return content, _estimate_tokens(content)+if (self._kompress_max_tokens > 0and _estimate_tokens(text_to_compress) > self._kompress_max_tokens@@ -4564,7 +4718,28 @@ class ContentRouter(Transform):compressor, "shares_request_deadline", False):compress_kwargs["_deadline_started_at"] = deadline_origin- result = compressor.compress(text_to_compress, **compress_kwargs)+ # The ML budget's remainder caps THIS call, so the+ # first one of a request -- admitted without a measured+ # cost -- cannot run past the budget either. Remote+ # Kompress takes the cap too (it bounds the whole HTTP+ # request, and declines the call when too little is+ # left) but not the shared deadline origin, hence the+ # separate flag.+ if _remaining is not None and (+ getattr(compressor, "shares_request_deadline", False)+ or getattr(compressor, "accepts_time_budget_cap", False)+ ):+ compress_kwargs["_time_budget_cap_seconds"] = _remaining+ _ml_started = time.monotonic()+ try:+ result = compressor.compress(text_to_compress, **compress_kwargs)+ finally:+ # In a `finally` so a call that RAISES still draws+ # down the budget: a failing compressor burns the+ # same non-preemptible wall clock as a passing one,+ # and not charging for it let a request retry its+ # way past the ceiling.+ self._charge_ml_time(time.monotonic() - _ml_started, text_to_compress)compressed = result.compressedcompressed_tokens = result.compressed_tokensexcept Exception as e:@@ -5485,8 +5660,18 @@ class ContentRouter(Transform):# The kompress deadline origin is stamped HERE, with the rest of the# per-request state, so every block this call compresses draws down one# shared budget instead of restarting it.+ #+ # The ML ceiling (#3711) is deliberately NOT armed here. It used to be,+ # which meant lifecycle work, message classification, lossless+ # transforms and task preparation all ran down a budget whose log line+ # claimed to measure time "in ML". `_try_ml_compressor` accumulates the+ # time it actually spends inside the compressor instead, so the number+ # means what it says.self._runtime_state_var.set(- _PerRequestRuntimeState(kompress_deadline_started_at=time.perf_counter())+ _PerRequestRuntimeState(+ kompress_deadline_started_at=time.perf_counter(),+ ml_budget_active=True,+ ))# Pre-process: Read lifecycle management (stale/superseded detection)diff --git a/headroom/transforms/kompress_compressor.py b/headroom/transforms/kompress_compressor.pyindex 0b9a8a8e6..e5d48c60a 100644--- a/headroom/transforms/kompress_compressor.py+++ b/headroom/transforms/kompress_compressor.py@@ -441,6 +441,21 @@ def _time_budget_seconds() -> float | None:return raw if raw > 0 else None+def _capped_time_budget(cap: float | None) -> float | None:+ """The per-call budget, lowered to *cap* when the caller has less to give.++ The router passes what is left of its request-scoped ML budget, so a call+ -- including the first one of a request, which has no measured cost to be+ refused on -- stops at a chunk boundary when that budget runs out instead+ of running to its own, larger, per-call budget.+ """+ budget = _time_budget_seconds()+ if cap is None:+ return budget+ cap = max(0.0, cap)+ return cap if budget is None else min(budget, cap)++def _canary_threshold_seconds() -> float | None:"""Startup canary threshold; inference slower than this disables Kompress.@@ -1616,11 +1631,15 @@ class KompressCompressor(Transform):# ``compress()`` accepts ``_deadline_started_at``, so a caller that# compresses many blocks for ONE request can hand every call the same- # origin and have the deadline bound the request. Duck-typed rather than+ # origin and have the deadline bound the request. It also accepts+ # ``_time_budget_cap_seconds``, which lowers this call's time budget to+ # what the caller's request-scoped ML budget has left. Duck-typed rather than# isinstance-checked at the call site because ``RemoteKompressCompressor``- # is the other compressor the router may get back and its ``compress()``- # does not take the argument.+ # is the other compressor the router may get back: its ``compress()`` takes+ # the cap (flagged by ``accepts_time_budget_cap``) but not the deadline+ # origin.shares_request_deadline: bool = True+ accepts_time_budget_cap: bool = Truedef __init__(self, config: KompressConfig | None = None):self.config = config or KompressConfig()@@ -1762,6 +1781,7 @@ class KompressCompressor(Transform):allow_download: bool = True,ccr_original: str | None = None,_deadline_started_at: float | None = None,+ _time_budget_cap_seconds: float | None = None,) -> KompressResult:"""Compress content using Kompress model.@@ -1824,6 +1844,7 @@ class KompressCompressor(Transform):batch_size=_batch_size(),ccr_originals=[ccr_original],_deadline_started_at=t_deadline,+ _time_budget_cap_seconds=_time_budget_cap_seconds,)if batch_result:return batch_result[0]@@ -1834,7 +1855,7 @@ class KompressCompressor(Transform):chunk_count = 0acquire_timeout = _acquire_timeout_seconds()- budget = _time_budget_seconds()+ budget = _capped_time_budget(_time_budget_cap_seconds)deadline = time.monotonic() + budget if budget is not None else Nonefor chunk_start in range(0, n_words, max_chunk_words):@@ -2104,6 +2125,7 @@ class KompressCompressor(Transform):*,ccr_originals: list[str | None] | None = None,_deadline_started_at: float | None = None,+ _time_budget_cap_seconds: float | None = None,) -> list[KompressResult]:"""Compress multiple texts. Uses batched inference on GPU, sequential on CPU.@@ -2198,6 +2220,12 @@ class KompressCompressor(Transform):# per-item slowdown measured on ONNX CPU (~0.7-0.9x vs sequential).# GPU users still benefit from the batched forward pass below.if self._should_use_sequential_fallback():+ # The cap is for the whole batch, so each item gets what is left.+ cap_ends_at = (+ None+ if _time_budget_cap_seconds is None+ else time.monotonic() + _time_budget_cap_seconds+ )return [self.compress(content,@@ -2207,6 +2235,9 @@ class KompressCompressor(Transform):target_ratio=r,ccr_original=ccr_source,_deadline_started_at=t_deadline,+ _time_budget_cap_seconds=(+ None if cap_ends_at is None else cap_ends_at - time.monotonic()+ ),)for content, r, ccr_source in zip(contents, ratios, ccr_sources, strict=True)]@@ -2257,7 +2288,7 @@ class KompressCompressor(Transform):self._deadline_s = deadline_sacquire_timeout = _acquire_timeout_seconds()- budget = _time_budget_seconds()+ budget = _capped_time_budget(_time_budget_cap_seconds)deadline = time.monotonic() + budget if budget is not None else Nonedef _bail_remaining(reason: str, batch_start: int) -> None:diff --git a/headroom/transforms/kompress_remote.py b/headroom/transforms/kompress_remote.pyindex 4ebabd88d..1af00349a 100644--- a/headroom/transforms/kompress_remote.py+++ b/headroom/transforms/kompress_remote.py@@ -53,7 +53,12 @@ never correctness.from __future__ import annotations+import concurrent.futures+import jsonimport logging+import threading+import time+from typing import Anyimport httpx@@ -102,6 +107,21 @@ _MIN_WORDS = 10# Accept-any-shrink CCR gate, identical to KompressCompressor.compress: only# store + mark when the shrink is worth the retrieval marker's own cost.+# A remote call is declined outright when the caller's request-scoped ML budget+# has this little left: no HTTP round trip (connect + write + server inference ++# read) fits in it, so starting one only burns the budget and leaves a worker+# behind. The block passes through unchanged instead.+_MIN_REMOTE_BUDGET_SECONDS = 0.05++# Workers for budget-capped calls, and the number of admission permits. A call+# whose caller times out is abandoned, not killed (a sync httpx request cannot be+# interrupted from another thread), so the worker keeps its permit until the+# request itself ends -- and every request ends at an absolute deadline (see+# ``_capped_request``). With permits == workers nothing ever waits in the+# executor's (unbounded) queue: a saturated caller waits for a permit, not with+# its payload enqueued, and fails open when its own deadline passes.+_CAPPED_CALL_WORKERS = 16+class RemoteKompressCompressor:"""Drop-in for KompressCompressor that POSTs to a hosted ``/compress`` endpoint.@@ -112,6 +132,12 @@ class RemoteKompressCompressor:name = "kompress_compressor"+ # ``compress()`` accepts ``_time_budget_cap_seconds``: what the router's+ # request-scoped ML budget has left. Unlike local Kompress this class does+ # not take ``_deadline_started_at``, so it does not set+ # ``shares_request_deadline``; the router checks this flag separately.+ accepts_time_budget_cap: bool = True+def __init__(self,endpoint: str,@@ -120,6 +146,7 @@ class RemoteKompressCompressor:timeout: float = 20.0,path: str | None = DEFAULT_ENDPOINT_PATH,headers: dict[str, str] | None = None,+ transport: httpx.BaseTransport | None = None,) -> None:self.config = config or KompressConfig()# Default keeps the pre-existing behaviour exactly: <endpoint>/compress.@@ -148,7 +175,13 @@ class RemoteKompressCompressor:# Kompress, rather than discovering it in a packet capture.guard_egress("remote Kompress inference", self._url)# httpx.Client is safe to share across the proxy's worker threads.- self._client = httpx.Client(timeout=timeout)+ # ``transport`` exists for tests (httpx.MockTransport); None is the+ # normal network transport.+ self._timeout = timeout+ self._client = httpx.Client(timeout=timeout, transport=transport)+ self._capped_pool: concurrent.futures.ThreadPoolExecutor | None = None+ self._capped_pool_lock = threading.Lock()+ self._capped_slots = threading.BoundedSemaphore(_CAPPED_CALL_WORKERS)@propertydef url(self) -> str:@@ -193,9 +226,18 @@ class RemoteKompressCompressor:*,allow_download: bool = True,ccr_original: str | None = None,+ _time_budget_cap_seconds: float | None = None,) -> KompressResult:"""Compress via the remote endpoint.+ ``_time_budget_cap_seconds`` is what is left of the caller's+ request-scoped ML budget (ContentRouter passes it). It bounds the HTTP+ request AS A WHOLE -- connect, write, server time and read together,+ not each phase -- and a call that runs out passes the block through+ unchanged. When the remainder is at or below+ :data:`_MIN_REMOTE_BUDGET_SECONDS` no request is made at all. ``None``+ (direct callers) keeps the configured per-phase timeout, 20s by default.+``ccr_original`` mirrors :meth:`KompressCompressor.compress`: text tostore in CCR instead of ``content``, used when ``content`` is atag-protected placeholder intermediate ({{HEADROOM_TAG_N}}). It is@@ -230,14 +272,23 @@ class RemoteKompressCompressor:# the hole this guard closes.guard_egress("remote Kompress inference", self._url)- try:- resp = self._client.post(- self._url,- headers=self._headers,- json={"content": content, "target_ratio": target_ratio},+ payload = {"content": content, "target_ratio": target_ratio}+ if _time_budget_cap_seconds is not None and (+ _time_budget_cap_seconds <= _MIN_REMOTE_BUDGET_SECONDS+ ):+ logger.info(+ "Remote Kompress declined: %.3fs of request ML budget left (floor %.3fs); "+ "passing through unchanged",+ max(0.0, _time_budget_cap_seconds),+ _MIN_REMOTE_BUDGET_SECONDS,)- resp.raise_for_status()- data = resp.json()+ return self._passthrough(content, n_words)++ try:+ if _time_budget_cap_seconds is None:+ data = self._post(payload)+ else:+ data = self._post_within(payload, _time_budget_cap_seconds)compressed = data["compressed"]if not isinstance(compressed, str):raise TypeError("remote Kompress response field 'compressed' must be a string")@@ -299,5 +350,95 @@ class RemoteKompressCompressor:return result+ def _post(self, payload: dict[str, Any], timeout: float | None = None) -> Any:+ # No ``timeout`` kwarg on the default path: the client's own (20s)+ # applies, and the call is exactly what it was before the cap existed.+ extra: dict[str, Any] = {} if timeout is None else {"timeout": timeout}+ resp = self._client.post(self._url, headers=self._headers, json=payload, **extra)+ resp.raise_for_status()+ return resp.json()++ def _post_within(self, payload: dict[str, Any], cap: float) -> Any:+ """POST and read the response within *cap* seconds in total.++ httpx timeouts are per phase (connect, write, pool, and each read+ between chunks), so ``timeout=cap`` alone neither bounds the call nor+ stops a slow-drip body. Three things together do:++ * **Admission.** A permit (one per worker) is taken before submitting,+ waiting at most until the caller's deadline. Saturated means fail+ open, with the payload never enqueued -- ``future.cancel()`` does not+ free a queued payload, so an unbounded queue would retain every+ timed-out prompt.+ * **Caller deadline.** The caller waits for the worker with one overall+ deadline and passes the block through unchanged when it runs out.+ * **Worker deadline.** The worker keeps its permit until it actually+ finishes (released in its own ``finally``, never on caller timeout),+ and it finishes by an absolute deadline: per-phase timeouts are capped+ to what is left, and the body is streamed and cut off at the+ deadline, so a dripping response cannot hold a permit indefinitely.+ """+ deadline = time.monotonic() + cap+ if not self._capped_slots.acquire(timeout=max(0.0, cap)):+ raise TimeoutError(+ f"all {_CAPPED_CALL_WORKERS} remote Kompress workers busy for the "+ f"request ML budget ({cap:.3f}s)"+ )+ try:+ future = self._get_capped_pool().submit(self._capped_request, payload, deadline)+ except BaseException:+ self._capped_slots.release() # never reached the worker that would+ raise+ try:+ return future.result(timeout=max(0.0, deadline - time.monotonic()))+ except concurrent.futures.TimeoutError:+ raise TimeoutError(+ f"exceeded the request ML budget ({cap:.3f}s) before the endpoint answered"+ ) from None++ def _capped_request(self, payload: dict[str, Any], deadline: float) -> Any:+ """The worker side of :meth:`_post_within`. Owns one admission permit.++ Lifetime is bounded by *deadline* plus at most one per-phase timeout+ (itself no longer than what was left at start): every httpx phase is+ capped to the time remaining, and the body is read chunk by chunk with+ the absolute deadline checked after each one.+ """+ try:+ left = deadline - time.monotonic()+ if left <= 0.0:+ raise TimeoutError("request ML budget spent before the request started")+ with self._client.stream(+ "POST",+ self._url,+ headers=self._headers,+ json=payload,+ timeout=min(left, self._timeout),+ ) as resp:+ resp.raise_for_status()+ body = bytearray()+ for chunk in resp.iter_bytes():+ body += chunk+ if time.monotonic() >= deadline:+ raise TimeoutError(+ "remote Kompress response outlived the request ML budget"+ )+ return json.loads(bytes(body))+ finally:+ self._capped_slots.release()++ def _get_capped_pool(self) -> concurrent.futures.ThreadPoolExecutor:+ with self._capped_pool_lock:+ if self._capped_pool is None:+ self._capped_pool = concurrent.futures.ThreadPoolExecutor(+ max_workers=_CAPPED_CALL_WORKERS,+ thread_name_prefix="remote-kompress",+ )+ return self._capped_pool+def close(self) -> None:+ with self._capped_pool_lock:+ pool, self._capped_pool = self._capped_pool, None+ if pool is not None:+ pool.shutdown(wait=False, cancel_futures=True)self._client.close()diff --git a/tests/test_content_router_ml_stage_deadline.py b/tests/test_content_router_ml_stage_deadline.pynew file mode 100644index 000000000..079ff5a06--- /dev/null+++ b/tests/test_content_router_ml_stage_deadline.py@@ -0,0 +1,678 @@+"""Request-scoped ML ceiling for the mixed-content path (#3711).++``HEADROOM_KOMPRESS_MAX_TOKENS`` bounds a single block. It cannot bound a+request: ``_compress_mixed`` splits a payload into sections and calls+``_try_ml_compressor`` once per section, so every section can sit under the+per-block ceiling while their sum runs for minutes. The reporter measured ~70s+on a 1.4MB ``tool_result`` of prose blocks separated by small JSON objects,+which blew the 30s compression budget and then quarantined compression for+every following request -- while+``headroom_kompress_size_gate_total`` recorded only ``within``.++A stub stands in for Kompress so these assert the *budget*, not ONNX latency:+the real model is not available in CI, and the bug is about how many times a+slow stage is entered, not how slow it is.+"""++from __future__ import annotations++import concurrent.futures+import json+import random+import threading+import time+from dataclasses import dataclass++import httpx+import pytest++from headroom.transforms import content_router as cr+from headroom.transforms.kompress_remote import RemoteKompressCompressor+++class _Tokenizer:+ def count_text(self, content: str) -> int:+ return len(content.split())+++@dataclass+class _StubResult:+ compressed: str+ compressed_tokens: int+++class _SlowKompress:+ """Stands in for Kompress: ready, and costs `per_call_s` every call."""++ def __init__(self, per_call_s: float) -> None:+ self.per_call_s = per_call_s+ self.calls = 0++ def is_ready(self) -> bool:+ return True++ def ensure_background_load(self) -> None: # pragma: no cover - never reached+ raise AssertionError("stub is always ready")++ def compress(self, text: str, **_kwargs: object) -> _StubResult:+ self.calls += 1+ time.sleep(self.per_call_s)+ out = text[: max(1, len(text) // 2)]+ return _StubResult(compressed=out, compressed_tokens=cr._estimate_tokens(out))+++def _prose(n_chars: int, seed: int = 7) -> str:+ rng = random.Random(seed)+ words = (+ "analysis deployment configuration throughput latency pipeline compression "+ "artifact identifier resolution boundary telemetry inference workspace"+ ).split()+ out: list[str] = []+ total = 0+ while total < n_chars:+ line = " ".join(rng.choice(words) for _ in range(14))+ out.append(line)+ total += len(line) + 1+ return "\n".join(out)+++def _reporter_payload(block_chars: int = 40_000, blocks: int = 8) -> str:+ """Prose blocks separated by small JSON objects -- the #3711 shape.++ Each block stays under the 50k-token per-block gate, so the size gate+ cannot fire; only a request-scoped ceiling can stop this.+ """+ parts: list[str] = []+ for i in range(blocks):+ parts.append(_prose(block_chars, seed=i))+ parts.append(json.dumps({"id": i, "status": "ok", "note": f"marker {i}"}))+ return "\n\n".join(parts)+++def _tool_result_messages() -> list[dict]:+ """The reported shape: the payload arrives as a tool_result, not user text.++ Plain user text is protected from compression+ (``router:protected:user_message``) and never reaches the ML stage at all.+ """+ return [+ {"role": "user", "content": "go"},+ {+ "role": "assistant",+ "content": [{"type": "tool_use", "id": "t1", "name": "Bash", "input": {}}],+ },+ {+ "role": "user",+ "content": [+ {"type": "tool_result", "tool_use_id": "t1", "content": _reporter_payload()}+ ],+ },+ ]+++def gate_outcomes(monkeypatch) -> dict[str, int]:+ seen: dict[str, int] = {}+ monkeypatch.setattr(+ cr.ContentRouter,+ "_observe_kompress_size_gate",+ lambda self, outcome: seen.__setitem__(outcome, seen.get(outcome, 0) + 1),+ )+ return seen+++def slow_kompress(monkeypatch) -> _SlowKompress:+ stub = _SlowKompress(per_call_s=0.05)+ monkeypatch.setattr(cr.ContentRouter, "_get_kompress", lambda self: stub)+ return stub+++def _apply(router: cr.ContentRouter) -> None:+ router.apply(+ _tool_result_messages(),+ _Tokenizer(),+ frozen_message_count=1,+ min_tokens_to_compress=1,+ )+++def test_size_gate_alone_cannot_bound_a_request(gate_outcomes, slow_kompress, monkeypatch) -> None:+ """Every section passes the per-block gate -- this is the #3711 precondition.++ Reproduces the reporter's metric exactly: many ``within`` decisions and no+ ``exceeded``. If this ever records ``exceeded`` the payload has stopped+ reproducing the report, and the deadline test below would pass for the+ wrong reason.+ """+ monkeypatch.setenv("HEADROOM_ML_STAGE_DEADLINE_SECONDS", "0") # isolate the size gate++ _apply(cr.ContentRouter(cr.ContentRouterConfig()))++ assert gate_outcomes.get("exceeded", 0) == 0, (+ f"payload no longer reproduces #3711 (a section exceeded the gate): {gate_outcomes}"+ )+ assert gate_outcomes.get("within", 0) > 1, (+ "expected repeated per-section ML entry, the shape that blows the budget; "+ f"got {gate_outcomes}"+ )+ assert slow_kompress.calls > 1, "the ML stage was entered once or not at all"+++def test_ml_deadline_stops_further_ml_work_once_the_budget_is_spent(+ gate_outcomes, slow_kompress, monkeypatch+) -> None:+ """Past the ceiling, remaining sections route off ML instead of compounding."""+ monkeypatch.setenv("HEADROOM_ML_STAGE_DEADLINE_SECONDS", "0.06")++ _apply(cr.ContentRouter(cr.ContentRouterConfig()))++ assert gate_outcomes.get("deadline", 0) > 0, (+ f"request-scoped ceiling never fired: {gate_outcomes}"+ )++ # The saving has to be real, so measure it: the same payload with the+ # ceiling disabled must enter the model strictly more often.+ bounded_calls = slow_kompress.calls+ monkeypatch.setenv("HEADROOM_ML_STAGE_DEADLINE_SECONDS", "0")+ unbounded = _SlowKompress(per_call_s=0.05)+ monkeypatch.setattr(cr.ContentRouter, "_get_kompress", lambda self: unbounded)++ _apply(cr.ContentRouter(cr.ContentRouterConfig()))++ assert bounded_calls < unbounded.calls, (+ f"ceiling skipped no model calls: {bounded_calls} with it, {unbounded.calls} without"+ )+++def test_deadline_zero_restores_previous_unbounded_behaviour(+ gate_outcomes, slow_kompress, monkeypatch+) -> None:+ monkeypatch.setenv("HEADROOM_ML_STAGE_DEADLINE_SECONDS", "0")++ _apply(cr.ContentRouter(cr.ContentRouterConfig()))++ assert gate_outcomes.get("deadline", 0) == 0, (+ f"deadline fired although disabled: {gate_outcomes}"+ )+++def test_direct_compress_callers_stay_unarmed() -> None:+ """``compress()`` without ``apply()`` keeps the old unbounded behaviour.++ Tests and the ``/v1/compress`` path call ``compress()`` directly; they must+ not inherit a request budget nobody set.+ """+ router = cr.ContentRouter(cr.ContentRouterConfig())+ state = router._runtime_state_var.get()+ assert state.ml_budget_active is False++ # That default is instance-scoped and shared by every direct call for the+ # life of the router, so charging it would accumulate across unrelated+ # calls until ML switched itself off permanently.+ router._charge_ml_time(5.0, "some text to compress")+ assert state.ml_elapsed == 0.0+++# --------------------------------------------------------------------------- #+# Review follow-ups: the budget must mean ML time, must refuse a call it cannot+# afford, and must sit under the timeout that quarantines the stage.+#+# The original tests asserted that LATER model calls are skipped. That is not+# the same claim as the request-wide time bound the module documents, and all+# three gaps below passed those tests.+# --------------------------------------------------------------------------- #+++class _SlowTokenizer(_Tokenizer):+ """Expensive NON-ML prework: token counting during classification."""++ def __init__(self, per_call_s: float) -> None:+ self.per_call_s = per_call_s+ self.calls = 0++ def count_text(self, content: str) -> int:+ self.calls += 1+ time.sleep(self.per_call_s)+ return super().count_text(content)+++def charged(monkeypatch) -> list[float]:+ """Every increment billed to the ML budget, in order."""+ seen: list[float] = []+ real = cr.ContentRouter._charge_ml_time++ def _spy(self, elapsed: float, text: str) -> None:+ seen.append(elapsed)+ real(self, elapsed, text)++ monkeypatch.setattr(cr.ContentRouter, "_charge_ml_time", _spy)+ return seen+++def test_non_ml_prework_does_not_consume_the_ml_budget(+ gate_outcomes, slow_kompress, charged, monkeypatch+) -> None:+ """The budget was armed at ``apply()`` entry, so lifecycle work, message+ classification and lossless transforms all ran it down before ML began --+ while the log line claimed to measure time "in ML"."""+ monkeypatch.setenv("HEADROOM_ML_STAGE_DEADLINE_SECONDS", "0.4")+ router = cr.ContentRouter(cr.ContentRouterConfig(enable_kompress=True))++ tokenizer = _SlowTokenizer(per_call_s=0.01)+ started = time.monotonic()+ router.apply(+ _tool_result_messages(),+ tokenizer,+ frozen_message_count=1,+ min_tokens_to_compress=1,+ )+ wall = time.monotonic() - started++ assert tokenizer.calls > 0, "prework must actually have run"+ assert slow_kompress.calls > 0, "prework time must not have spent the ML budget"+ # The bound is on ML time, not on the call.+ assert sum(charged) <= wall+ assert sum(charged) == pytest.approx(slow_kompress.calls * slow_kompress.per_call_s, rel=0.5)+++def test_a_slow_call_cannot_be_admitted_against_a_budget_it_will_overrun(+ gate_outcomes, charged, monkeypatch+) -> None:+ """Admission used to be ``now >= deadline`` only, so a call starting at+ 14.9s of a 15s budget ran to completion well past it -- the ONNX worker is+ not preemptible, so nothing stops it once it has begun."""+ monkeypatch.setenv("HEADROOM_ML_STAGE_DEADLINE_SECONDS", "0.5")+ stub = _SlowKompress(per_call_s=0.35)+ monkeypatch.setattr(cr.ContentRouter, "_get_kompress", lambda self: stub)+ router = cr.ContentRouter(cr.ContentRouterConfig(enable_kompress=True))++ _apply(router)++ assert stub.calls >= 1, "the first block has no measurement yet and must run"+ # The point of the change: total ML time stays inside the budget, rather+ # than the budget merely being the moment after which no call STARTS.+ assert sum(charged) <= 0.5, (+ f"ML overran its 0.5s budget: {sum(charged):.2f}s across {stub.calls} calls"+ )+ assert gate_outcomes.get("deadline", 0) >= 1, "remaining blocks must route off ML"+++def test_budget_is_clamped_below_the_timeout_that_quarantines_the_stage(+ monkeypatch,+) -> None:+ """A 15s guard under a 10s executor timeout can never fire first, which is+ exactly the failure the ceiling exists to prevent."""+ monkeypatch.delenv("HEADROOM_ML_STAGE_DEADLINE_SECONDS", raising=False)++ monkeypatch.setenv("HEADROOM_COMPRESSION_TIMEOUT_SECONDS", "30")+ assert cr._ml_stage_deadline_seconds() == 15.0, "the documented default is unchanged"++ monkeypatch.setenv("HEADROOM_COMPRESSION_TIMEOUT_SECONDS", "10")+ clamped = cr._ml_stage_deadline_seconds()+ assert clamped == 5.0+ assert clamped < 10.0, "must leave room to degrade before the executor gives up"++ # The clamp only ever lowers: an explicit request below it is honoured.+ monkeypatch.setenv("HEADROOM_ML_STAGE_DEADLINE_SECONDS", "2")+ assert cr._ml_stage_deadline_seconds() == 2.0++ # And 0 still disables the ceiling entirely.+ monkeypatch.setenv("HEADROOM_ML_STAGE_DEADLINE_SECONDS", "0")+ assert cr._ml_stage_deadline_seconds() == 0.0+++def test_a_failing_ml_call_still_draws_down_the_budget(charged, monkeypatch) -> None:+ """Otherwise a request could retry its way past the ceiling: a compressor+ that raises burns the same non-preemptible wall clock as one that returns."""+ monkeypatch.setenv("HEADROOM_ML_STAGE_DEADLINE_SECONDS", "5")++ class _Exploding(_SlowKompress):+ def compress(self, text: str, **_kwargs: object):+ self.calls += 1+ time.sleep(self.per_call_s)+ raise RuntimeError("onnx said no")++ stub = _Exploding(per_call_s=0.05)+ monkeypatch.setattr(cr.ContentRouter, "_get_kompress", lambda self: stub)+ router = cr.ContentRouter(cr.ContentRouterConfig(enable_kompress=True))++ _apply(router)++ assert stub.calls > 0+ assert charged, "a raising call must still be billed"+ assert sum(charged) > 0.0+++# --------------------------------------------------------------------------- #+# Second review: the FIRST ML call of a request was always admitted (the seed+# rate was 0, and there is no measurement yet), and it is non-preemptible, so+# one huge first block could still run past the ML budget and the executor+# timeout. The call is now capped rather than estimated.+# --------------------------------------------------------------------------- #+++class _CooperativeKompress(_SlowKompress):+ """Behaves like Kompress under a time cap: stops at a chunk boundary once+ ``_time_budget_cap_seconds`` is spent and returns the block unchanged."""++ shares_request_deadline = True++ def __init__(self, worst_case_s: float, chunk_s: float = 0.02) -> None:+ super().__init__(per_call_s=worst_case_s)+ self.chunk_s = chunk_s+ self.caps: list[float | None] = []++ def compress(self, text: str, **kwargs: object) -> _StubResult:+ self.calls += 1+ cap = kwargs.get("_time_budget_cap_seconds")+ self.caps.append(cap) # type: ignore[arg-type]+ ends_at = None if cap is None else time.monotonic() + float(cap) # type: ignore[arg-type]+ spent = 0.0+ while spent < self.per_call_s:+ if ends_at is not None and time.monotonic() >= ends_at:+ return _StubResult(compressed=text, compressed_tokens=cr._estimate_tokens(text))+ time.sleep(self.chunk_s)+ spent += self.chunk_s+ out = text[: max(1, len(text) // 2)]+ return _StubResult(compressed=out, compressed_tokens=cr._estimate_tokens(out))+++def test_first_ml_call_is_capped_by_the_budget_it_was_admitted_against(+ charged, monkeypatch+) -> None:+ """A first block whose worst case (3s) exceeds both the outer timeout (1s)+ and the ML budget derived from it (0.5s) must still stop inside the budget."""+ monkeypatch.delenv("HEADROOM_ML_STAGE_DEADLINE_SECONDS", raising=False)+ monkeypatch.setenv("HEADROOM_COMPRESSION_TIMEOUT_SECONDS", "1")+ budget = cr._ml_stage_deadline_seconds()+ assert budget == 0.5++ stub = _CooperativeKompress(worst_case_s=3.0)+ monkeypatch.setattr(cr.ContentRouter, "_get_kompress", lambda self: stub)++ _apply(cr.ContentRouter(cr.ContentRouterConfig(enable_kompress=True)))++ assert stub.caps, "the first block must still reach the model"+ first_cap = stub.caps[0]+ assert first_cap is not None and 0.0 < first_cap <= budget, (+ f"first call was not capped by the ML budget: {stub.caps}"+ )+ # One chunk of slack: the cap is checked at chunk boundaries, as in Kompress.+ assert sum(charged) <= budget + 0.2, (+ f"ML ran {sum(charged):.2f}s against a {budget}s budget under a 1s executor timeout"+ )+++def test_blocks_refused_by_the_budget_pass_through_unchanged(monkeypatch) -> None:+ """Not the size gate's lossy fallback: the refusal depends on the request,+ the router caches the result, and only raw bytes stay the same on later+ turns, so the prompt cache holds."""+ router = cr.ContentRouter(cr.ContentRouterConfig(enable_kompress=True))+ router._runtime_state_var.set(+ cr._PerRequestRuntimeState(ml_budget_active=True, ml_elapsed=999.0)+ )++ class _Crusher:+ def compress(self, text: str, context: str = "") -> _StubResult:+ return _StubResult(compressed="crushed", compressed_tokens=1)++ monkeypatch.setattr(router, "_get_text_crusher", lambda: _Crusher())+ stub = _SlowKompress(per_call_s=0.0)+ monkeypatch.setattr(router, "_get_kompress", lambda: stub)++ text = _prose(2_000)+ out, _ = router._try_ml_compressor(text, "")++ assert out == text+ assert stub.calls == 0+++# --------------------------------------------------------------------------- #+# Third review: the cap reached only LOCAL Kompress. RemoteKompressCompressor+# got no cap and kept its independent 20s per-phase HTTP timeouts, so a cold+# remote call -- no measured rate, so nothing to refuse it on -- could run past+# both the ML budget and the outer timeout. These drive the real remote class+# through the public apply() with an httpx.MockTransport (no network).+# --------------------------------------------------------------------------- #+++def _tool_result_text(messages: list[dict]) -> str:+ return messages[-1]["content"][0]["content"]+++def _assert_prose_blocks_unchanged(messages: list[dict]) -> None:+ """Every ML-eligible prose block reaches the provider byte-identical.++ (The small JSON separators go through the lossless JSON path, which+ minifies them; that is not ML and not what is under test.)+ """+ out = _tool_result_text(messages)+ for i in range(8):+ assert _prose(40_000, seed=i) in out, f"prose block {i} was altered"+++def _remote_kompress(handler) -> RemoteKompressCompressor:+ return RemoteKompressCompressor(+ endpoint="http://kompress.invalid",+ transport=httpx.MockTransport(handler),+ )+++def _compressed_reply(request: httpx.Request) -> httpx.Response:+ content = json.loads(request.content)["content"]+ return httpx.Response(200, json={"compressed": " ".join(content.split()[::2])})+++def test_cold_remote_call_cannot_overrun_a_tiny_ml_budget(charged, monkeypatch) -> None:+ """The reviewer's probe: 10 ms ML budget, 40 ms outer timeout, an endpoint+ that takes 100 ms. No HTTP round trip fits in what is left, so the remote+ call is declined and every block passes through unchanged."""+ monkeypatch.setenv("HEADROOM_ML_STAGE_DEADLINE_SECONDS", "0.01")+ monkeypatch.setenv("HEADROOM_COMPRESSION_TIMEOUT_SECONDS", "0.04")+ assert cr._ml_stage_deadline_seconds() == 0.01++ requests: list[httpx.Request] = []++ def _slow_endpoint(request: httpx.Request) -> httpx.Response:+ requests.append(request)+ time.sleep(0.1)+ return _compressed_reply(request)++ remote = _remote_kompress(_slow_endpoint)+ monkeypatch.setattr(cr.ContentRouter, "_get_kompress", lambda self: remote)+ router = cr.ContentRouter(cr.ContentRouterConfig(enable_kompress=True))++ messages = _tool_result_messages()+ result = router.apply(messages, _Tokenizer(), frozen_message_count=1, min_tokens_to_compress=1)++ assert charged, "the remote compressor must still be reached and billed"+ assert sum(charged) < 0.04, f"remote ML ran {sum(charged):.3f}s under a 40 ms outer timeout"+ assert requests == [], "a call that cannot fit the budget must not be sent"+ _assert_prose_blocks_unchanged(result.messages)+++def test_cold_remote_call_is_bounded_as_a_whole_by_the_ml_budget(charged, monkeypatch) -> None:+ """With enough budget to try, the first (unmeasured) remote call is capped+ by the remainder -- over the whole request, not per httpx phase -- and an+ endpoint slower than that leaves the block unchanged."""+ budget = 0.2+ monkeypatch.setenv("HEADROOM_ML_STAGE_DEADLINE_SECONDS", str(budget))+ monkeypatch.setenv("HEADROOM_COMPRESSION_TIMEOUT_SECONDS", "30")++ release = threading.Event()+ timeouts: list[dict] = []++ def _hung_endpoint(request: httpx.Request) -> httpx.Response:+ timeouts.append(dict(request.extensions["timeout"]))+ release.wait(5.0) # far past the budget; MockTransport ignores httpx timeouts+ return _compressed_reply(request)++ remote = _remote_kompress(_hung_endpoint)+ monkeypatch.setattr(cr.ContentRouter, "_get_kompress", lambda self: remote)+ router = cr.ContentRouter(cr.ContentRouterConfig(enable_kompress=True))++ messages = _tool_result_messages()+ try:+ result = router.apply(+ messages, _Tokenizer(), frozen_message_count=1, min_tokens_to_compress=1+ )+ finally:+ release.set()+ remote.close()++ assert timeouts, "the first block must reach the endpoint"+ assert all(0.0 < t <= budget for t in timeouts[0].values()), (+ f"per-phase timeouts were not lowered to the budget: {timeouts[0]}"+ )+ assert sum(charged) <= budget + 0.1, (+ f"remote ML ran {sum(charged):.3f}s against a {budget}s budget"+ )+ _assert_prose_blocks_unchanged(result.messages)+++def test_remote_kompress_keeps_its_default_timeout_without_a_budget() -> None:+ """Direct callers pass no cap: same single request, 20s per-phase timeouts."""+ timeouts: list[dict] = []++ def _endpoint(request: httpx.Request) -> httpx.Response:+ timeouts.append(dict(request.extensions["timeout"]))+ return _compressed_reply(request)++ remote = _remote_kompress(_endpoint)+ text = _prose(2_000)+ out = remote.compress(text)++ assert out.compressed != text+ assert timeouts == [{"connect": 20.0, "read": 20.0, "write": 20.0, "pool": 20.0}]+ assert remote._capped_pool is None, "the default path must not hop threads"+++# --------------------------------------------------------------------------- #+# Fourth review: the capped worker pool had no admission bound. Its queue is+# unbounded and ``future.cancel()`` does not drop a queued payload, so once+# every worker was held by an abandoned request, each further capped call+# queued its full prompt, timed out, and left it there. A slow-drip body could+# hold a worker forever, since httpx read timeouts reset on every chunk.+# --------------------------------------------------------------------------- #+++def _free_permits(remote: RemoteKompressCompressor) -> int:+ return remote._capped_slots._value # type: ignore[attr-defined]+++def _wait_until(predicate, timeout_s: float = 3.0) -> bool:+ ends = time.monotonic() + timeout_s+ while time.monotonic() < ends:+ if predicate():+ return True+ time.sleep(0.01)+ return predicate()+++def test_saturated_remote_calls_fail_open_without_queueing_their_payloads() -> None:+ """The reviewer's repro: hold all 16 workers with requests whose 200 ms+ callers have already timed out, then make five more calls with 60 ms caps.+ Each must fail open within its cap with its payload never enqueued, and+ capacity must come back once the held requests end."""+ from headroom.transforms.kompress_remote import _CAPPED_CALL_WORKERS++ release = threading.Event()+ reached: list[str] = []++ def _stalled_endpoint(request: httpx.Request) -> httpx.Response:+ reached.append(json.loads(request.content)["content"])+ release.wait(5.0)+ return _compressed_reply(request)++ remote = _remote_kompress(_stalled_endpoint)+ text = _prose(2_000)+ try:+ with concurrent.futures.ThreadPoolExecutor(_CAPPED_CALL_WORKERS) as callers:+ held = list(+ callers.map(+ lambda i: remote.compress(f"held{i} {text}", _time_budget_cap_seconds=0.2),+ range(_CAPPED_CALL_WORKERS),+ )+ )+ assert all(out.compressed == f"held{i} {text}" for i, out in enumerate(held))+ assert len(reached) == _CAPPED_CALL_WORKERS+ assert _free_permits(remote) == 0, "abandoned requests must keep their permits"++ for i in range(5):+ late = f"late{i} {text}"+ started = time.monotonic()+ out = remote.compress(late, _time_budget_cap_seconds=0.06)+ took = time.monotonic() - started+ assert out.compressed == late, "a saturated call must fail open unchanged"+ assert took < 0.06 + 0.1, f"saturated call took {took:.3f}s against a 60 ms cap"+ assert remote._capped_pool._work_queue.qsize() == 0, "a payload was enqueued"++ assert len(reached) == _CAPPED_CALL_WORKERS, "a saturated call reached the endpoint"+ finally:+ release.set()++ assert _wait_until(lambda: _free_permits(remote) == _CAPPED_CALL_WORKERS)+ recovered = remote.compress(text, _time_budget_cap_seconds=2.0)+ assert recovered.compressed != text, "capacity must return once the held requests end"+ remote.close()+++def test_slow_drip_response_is_cut_off_at_an_absolute_deadline() -> None:+ """A body that keeps dripping (a chunk every 50 ms for 2 s) resets httpx's+ read timeout on every chunk. The worker must stop at the cap's absolute+ deadline and give its permit back, not ride the response to the end."""+ from headroom.transforms.kompress_remote import _CAPPED_CALL_WORKERS++ chunks_sent: list[int] = []++ def _dripping_endpoint(request: httpx.Request) -> httpx.Response:+ reply = json.dumps({"compressed": "short"}).encode()++ def _drip():+ for i in range(40):+ chunks_sent.append(i)+ time.sleep(0.05)+ yield b" " # JSON-legal leading whitespace+ yield reply++ return httpx.Response(200, content=_drip())++ remote = _remote_kompress(_dripping_endpoint)+ text = _prose(2_000)+ cap = 0.2+ started = time.monotonic()+ out = remote.compress(text, _time_budget_cap_seconds=cap)+ assert out.compressed == text+ assert time.monotonic() - started < cap + 0.1++ assert _wait_until(lambda: _free_permits(remote) == _CAPPED_CALL_WORKERS, timeout_s=1.0), (+ "the dripping request still holds its permit"+ )+ assert time.monotonic() - started < 1.0, "the worker rode the drip instead of the deadline"+ assert len(chunks_sent) < 10, f"read {len(chunks_sent)} chunks past a {cap}s deadline"+ remote.close()+++def test_permit_is_held_until_the_worker_finishes_not_the_caller() -> None:+ """Releasing on caller timeout would let abandoned requests pile up again;+ the permit belongs to the running request."""+ from headroom.transforms.kompress_remote import _CAPPED_CALL_WORKERS++ release = threading.Event()++ def _stalled_endpoint(request: httpx.Request) -> httpx.Response:+ release.wait(5.0)+ return _compressed_reply(request)++ remote = _remote_kompress(_stalled_endpoint)+ text = _prose(2_000)+ try:+ out = remote.compress(text, _time_budget_cap_seconds=0.1)+ assert out.compressed == text+ assert _free_permits(remote) == _CAPPED_CALL_WORKERS - 1+ finally:+ release.set()+ assert _wait_until(lambda: _free_permits(remote) == _CAPPED_CALL_WORKERS)+ remote.close()diff --git a/tests/test_transforms/test_kompress_deadline.py b/tests/test_transforms/test_kompress_deadline.pyindex 2af170040..612b58bcb 100644--- a/tests/test_transforms/test_kompress_deadline.py+++ b/tests/test_transforms/test_kompress_deadline.py@@ -218,3 +218,66 @@ def test_a_compressor_that_cannot_take_the_deadline_is_not_handed_one(monkeypatcassert fake.deadlines, "the stub should still have been called"assert all(d is None for d in fake.deadlines), fake.deadlines+++def _chunked_kompress(monkeypatch, chunk_s: float):+ """Kompress over a fake model; each chunk advances a fake monotonic clock."""+ clock = {"now": 0.0, "chunks": 0}++ class _Enc(dict):+ def word_ids(self, batch_index=0):+ return self["_word_ids"]++ class _Tok:+ def __call__(self, chunk_words, **kw):+ n = len(chunk_words)+ return _Enc(input_ids=[[0] * n], attention_mask=[[1] * n], _word_ids=list(range(n)))++ class _Model:+ def get_keep_mask(self, input_ids, attention_mask):+ clock["now"] += chunk_s+ clock["chunks"] += 1+ n = len(input_ids[0])+ return [[i < n // 2 for i in range(n)]]++ def get_scores(self, input_ids, attention_mask): # the batched path+ return [+ [0.9 if keep else 0.1 for keep in row]+ for row in self.get_keep_mask(input_ids, attention_mask)+ ]++ monkeypatch.setattr(kc.time, "monotonic", lambda: clock["now"])+ monkeypatch.setattr(kc, "_load_kompress", lambda *a, **k: (_Model(), _Tok(), "onnx"))+ monkeypatch.setattr(kc, "_model_device_type", lambda *a, **k: "cpu")+ monkeypatch.setenv("HEADROOM_COMPRESSION_DEADLINE_MS", "0")+ monkeypatch.delenv(kc.KOMPRESS_TIME_BUDGET_ENV, raising=False)++ comp = kc.KompressCompressor(kc.KompressConfig(min_input_words=10))+ comp.config.chunk_words = 100+ monkeypatch.setattr(comp, "_should_batch_single_content", lambda *a, **k: False)+ monkeypatch.setattr(comp, "_should_use_sequential_fallback", lambda: False)+ return comp, clock+++def test_time_budget_cap_stops_a_call_and_returns_it_unchanged(monkeypatch):+ """The router hands Kompress what is left of its ML budget, so the first+ call of a request -- which has no measured cost to be refused on -- stops+ at a chunk boundary instead of running to the 20s per-call budget. The+ block comes back byte-identical, so it is stable on later turns."""+ comp, clock = _chunked_kompress(monkeypatch, chunk_s=0.6)+ content = " ".join(f"w{i}" for i in range(500)) # five chunks, 3.0s uncapped++ result = comp.compress(content, _time_budget_cap_seconds=1.0)++ assert clock["chunks"] == 2, "must stop at the first chunk boundary past the cap"+ assert result.compressed == content+++def test_time_budget_cap_bounds_the_batched_path_too(monkeypatch):+ comp, clock = _chunked_kompress(monkeypatch, chunk_s=0.6)+ content = " ".join(f"w{i}" for i in range(500))++ [result] = comp.compress_batch([content], batch_size=1, _time_budget_cap_seconds=1.0)++ assert clock["chunks"] == 2+ assert result.compressed == content