Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(windows): unwedge compression on degraded ONNX runtimes (every request timing out at 30s, 0% savings) (#822)
Tejas Chopra36202f4d0bd4
diff --git a/headroom/onnx_runtime.py b/headroom/onnx_runtime.pyindex 09306fe12..0ee6353a4 100644--- a/headroom/onnx_runtime.py+++ b/headroom/onnx_runtime.py@@ -3,10 +3,49 @@from __future__ import annotationsimport ctypes+import loggingimport osimport sysfrom typing import Any+logger = logging.getLogger(__name__)++# Override for the CPU memory-arena default below: "1"/"true" forces the+# arena ON, "0"/"false" forces it OFF, unset/"auto" uses the platform default.+ONNX_CPU_ARENA_ENV = "HEADROOM_ONNX_CPU_ARENA"++_TRUTHY = frozenset({"1", "true", "yes", "on"})+_FALSY = frozenset({"0", "false", "no", "off"})+++def _env_flag(name: str) -> bool | None:+ raw = os.environ.get(name, "").strip().lower()+ if raw in _TRUTHY:+ return True+ if raw in _FALSY:+ return False+ if raw and raw != "auto":+ logger.warning("%s must be a boolean or 'auto', got %r; using auto", name, raw)+ return None+++def cpu_arena_enabled() -> bool:+ """Whether ONNX Runtime's CPU memory arena should stay enabled.++ Disabling the arena trades peak throughput for lower retained RSS, which+ is the right call on the small Linux VMs Headroom commonly runs on. On+ Windows the same setting is catastrophic: without the arena every+ ``Run()`` falls back to per-node VirtualAlloc/free, slowing transformer+ inference by 2-3 orders of magnitude (onnxruntime#11627). That turned+ every compression into a >30s timeout for Windows proxy users, so the+ arena stays at ORT's default (enabled) there.+ """+ override = _env_flag(ONNX_CPU_ARENA_ENV)+ if override is not None:+ return override+ return sys.platform == "win32"++# Pin model artifacts to immutable commit SHAs so a changed or compromised# upstream HuggingFace repo cannot be pulled silently (supply-chain integrity).# Repos not listed here fall back to the floating default ref. Set@@ -90,6 +129,9 @@ def create_cpu_session_options(memory usage over peak ONNX throughput. Disabling ORT's CPU arena and memorypattern caches reduces retained anonymous RSS after variable-size inferenceworkloads, which is especially important on small VMs.++ The arena is left at ORT's default on Windows (see ``cpu_arena_enabled``),+ where disabling it degrades inference latency by orders of magnitude."""sess_options = ort.SessionOptions()@@ -98,10 +140,11 @@ def create_cpu_session_options(if inter_op_num_threads is not None:sess_options.inter_op_num_threads = inter_op_num_threads- if hasattr(sess_options, "enable_cpu_mem_arena"):- sess_options.enable_cpu_mem_arena = False- if hasattr(sess_options, "enable_mem_pattern"):- sess_options.enable_mem_pattern = False+ if not cpu_arena_enabled():+ if hasattr(sess_options, "enable_cpu_mem_arena"):+ sess_options.enable_cpu_mem_arena = False+ if hasattr(sess_options, "enable_mem_pattern"):+ sess_options.enable_mem_pattern = Falsereturn sess_optionsdiff --git a/headroom/proxy/handlers/openai.py b/headroom/proxy/handlers/openai.pyindex 184ecb7ec..1cc8a7bb5 100644--- a/headroom/proxy/handlers/openai.py+++ b/headroom/proxy/handlers/openai.py@@ -2846,8 +2846,11 @@ class OpenAIHandlerMixin:if result.waste_signals:waste_signals_dict = result.waste_signals.to_dict()except Exception as e:+ # Include type so TimeoutError vs other failures is distinguishable+ # in bug reports — str(asyncio.TimeoutError()) is empty otherwise.logger.warning(- f"Optimization failed: {type(e).__name__}: {e}",+ f"[{request_id}] Optimization failed: {type(e).__name__}: {e} "+ f"(request forwarded unoptimized)",exc_info=True,)# Flag compression failure for observabilitydiff --git a/headroom/proxy/helpers.py b/headroom/proxy/helpers.pyindex 13fddbbcc..26680d574 100644--- a/headroom/proxy/helpers.py+++ b/headroom/proxy/helpers.py@@ -1128,6 +1128,10 @@ def _read_rtk_lifetime_stats() -> dict[str, Any] | None:_rtk_gain_command(rtk_path, scope),capture_output=True,text=True,+ # rtk output is UTF-8 (emoji etc.); without this, Windows decodes+ # with cp1252 and the reader thread dies with UnicodeDecodeError.+ encoding="utf-8",+ errors="replace",timeout=5,)if result.returncode == 0 and result.stdout.strip():@@ -1177,6 +1181,9 @@ def _read_lean_ctx_lifetime_stats() -> dict[str, Any] | None:[str(lean_ctx_path), "gain", "--json"],capture_output=True,text=True,+ # UTF-8 regardless of the Windows console code page (cp1252).+ encoding="utf-8",+ errors="replace",timeout=5,)# Failed reads return None ("no data") — mirrors the rtk reader sodiff --git a/headroom/proxy/interceptors/astgrep.py b/headroom/proxy/interceptors/astgrep.pyindex 2b896df36..460192892 100644--- a/headroom/proxy/interceptors/astgrep.py+++ b/headroom/proxy/interceptors/astgrep.py@@ -214,6 +214,10 @@ def _run_ast_grep(],capture_output=True,text=True,+ # ast-grep emits UTF-8 (it matches arbitrary source text);+ # without this, Windows decodes with cp1252 and raises.+ encoding="utf-8",+ errors="replace",timeout=5,check=False,)diff --git a/headroom/transforms/kompress_compressor.py b/headroom/transforms/kompress_compressor.pyindex 560261914..c8e8bb057 100644--- a/headroom/transforms/kompress_compressor.py+++ b/headroom/transforms/kompress_compressor.py@@ -27,6 +27,7 @@ from typing import Any, Literalfrom ..config import TransformResultfrom ..onnx_runtime import (+ ONNX_CPU_ARENA_ENV,create_cpu_session_options,hf_hub_download_local_first,trim_process_heap,@@ -97,6 +98,18 @@ KOMPRESS_MAX_CONCURRENT_ENV = "HEADROOM_KOMPRESS_MAX_CONCURRENT"KOMPRESS_EXECUTION_SEMAPHORE_WAIT_MS_ENV = "HEADROOM_KOMPRESS_EXECUTION_TIMEOUT_MS"KOMPRESS_EXECUTION_SEMAPHORE_WAIT_MS_DEFAULT = 25KOMPRESS_BATCH_SIZE_ENV = "HEADROOM_KOMPRESS_BATCH_SIZE"+KOMPRESS_ACQUIRE_TIMEOUT_ENV = "HEADROOM_KOMPRESS_ACQUIRE_TIMEOUT_SECONDS"+KOMPRESS_TIME_BUDGET_ENV = "HEADROOM_KOMPRESS_TIME_BUDGET_SECONDS"+KOMPRESS_CANARY_THRESHOLD_ENV = "HEADROOM_KOMPRESS_CANARY_SECONDS"++# Both defaults sit well under the proxy's 30s compression-stage timeout so a+# slow model gives up (passthrough) before the request is abandoned. A thread+# abandoned by asyncio.wait_for cannot be killed; before these bounds existed,+# one pathologically slow inference would hold the execution semaphore forever+# and wedge every subsequent compression in the process.+_DEFAULT_ACQUIRE_TIMEOUT_SECONDS = 5.0+_DEFAULT_TIME_BUDGET_SECONDS = 20.0+_DEFAULT_CANARY_THRESHOLD_SECONDS = 5.0KompressBackend = Literal["auto", "onnx", "onnx_cpu", "onnx_coreml", "pytorch", "pytorch_mps"]@@ -244,6 +257,80 @@ def _env_int(name: str) -> int | None:return value+def _env_float(name: str) -> float | None:+ raw = os.environ.get(name)+ if raw is None or raw.strip() == "":+ return None+ try:+ return float(raw)+ except ValueError:+ logger.warning("%s must be a number, got %r; ignoring", name, raw)+ return None+++def _acquire_timeout_seconds() -> float | None:+ """Max seconds to wait for the execution semaphore. <=0 disables the bound."""+ raw = _env_float(KOMPRESS_ACQUIRE_TIMEOUT_ENV)+ if raw is None:+ return _DEFAULT_ACQUIRE_TIMEOUT_SECONDS+ return raw if raw > 0 else None+++def _time_budget_seconds() -> float | None:+ """Wall-clock budget for one compress/compress_batch call. <=0 disables."""+ raw = _env_float(KOMPRESS_TIME_BUDGET_ENV)+ if raw is None:+ return _DEFAULT_TIME_BUDGET_SECONDS+ return raw if raw > 0 else None+++def _canary_threshold_seconds() -> float | None:+ """Startup canary threshold; inference slower than this disables Kompress.++ <=0 disables the canary entirely.+ """+ raw = _env_float(KOMPRESS_CANARY_THRESHOLD_ENV)+ if raw is None:+ return _DEFAULT_CANARY_THRESHOLD_SECONDS+ return raw if raw > 0 else None+++_CANARY_SENTENCE = (+ "Headroom probes model latency at startup so a degraded runtime is "+ "detected before live traffic depends on it."+)+++def _canary_words(target_words: int = 120) -> list[str]:+ """Deterministic probe input sized like a small real compression chunk."""+ words = _CANARY_SENTENCE.split()+ reps = (target_words // len(words)) + 1+ return (words * reps)[:target_words]+++# First give-up is a WARNING with remediation hints; repeats drop to DEBUG so a+# persistently slow machine doesn't emit one warning per compressible message.+_giveup_warned = False+++def _log_giveup(reason: str, *, backend: str, device_type: str, n_words: int) -> None:+ global _giveup_warned+ level = logging.DEBUG if _giveup_warned else logging.WARNING+ _giveup_warned = True+ logger.log(+ level,+ "Kompress giving up (%s) backend=%s device=%s words=%d — content passes "+ "through uncompressed. If this persists, ML inference on this machine is "+ "too slow for inline compression; tune %s / %s or disable Kompress.",+ reason,+ backend,+ device_type,+ n_words,+ KOMPRESS_TIME_BUDGET_ENV,+ KOMPRESS_ACQUIRE_TIMEOUT_ENV,+ )++def _onnx_session_options(ort: Any) -> Any:return create_cpu_session_options(ort,@@ -290,6 +377,25 @@ def _execution_semaphore(backend: str, device_type: str) -> threading.BoundedSemreturn semaphore+def _acquire_bounded(+ semaphore: threading.BoundedSemaphore,+ acquire_timeout: float | None,+ remaining_budget: float | None,+) -> bool:+ """Acquire ``semaphore``, waiting at most the tighter of the two bounds.++ Returns False on timeout. An unbounded wait here is how one stuck+ inference (e.g. degraded ONNX on Windows) used to wedge every other+ compression in the process: abandoned executor threads queued forever+ on the semaphore. With both bounds disabled (<=0) this degrades to the+ legacy blocking acquire.+ """+ bounds = [b for b in (acquire_timeout, remaining_budget) if b is not None]+ if not bounds:+ return semaphore.acquire()+ return semaphore.acquire(timeout=max(0.0, min(bounds)))++def _batch_size() -> int:return _env_int(KOMPRESS_BATCH_SIZE_ENV) or 32@@ -942,6 +1048,10 @@ class KompressCompressor(Transform):def __init__(self, config: KompressConfig | None = None):self.config = config or KompressConfig()+ # Set by the preload canary when inference is too slow to be useful;+ # compress()/compress_batch() then pass content through untouched.+ self._degraded_reason: str | None = None+ self._canary_thread: threading.Thread | None = Nonedef preload(self, *, allow_download: bool = True) -> str:"""Load the backing model/tokenizer and return the selected backend.@@ -951,13 +1061,87 @@ class KompressCompressor(Transform):raised so the caller can defer the download to first use. Startup eagerpreload uses this so a cold cache cannot block the proxy from bindingits port.++ On a successful load this also kicks off a timed canary inference on a+ background thread: a machine where one small forward pass takes+ multiple seconds (seen with degraded ONNX runtimes on Windows) can+ never finish real compression inside the proxy's stage timeout, so+ Kompress fail-safes to passthrough once the probe verdict lands. The+ probe must NOT run inline — preload blocks proxy startup (the HTTP+ server binds after it), and on slow hardware the probe itself takes+ tens of seconds. Until the verdict, the per-call acquire/budget bounds+ keep any slow inference from wedging requests."""- _model, _tokenizer, backend = _load_kompress(+ model, tokenizer, backend = _load_kompress(self.config.model_id, self.config.device, allow_download=allow_download)++ threshold = _canary_threshold_seconds()+ if threshold is not None:+ self._canary_thread = threading.Thread(+ target=self._run_canary,+ args=(model, tokenizer, backend, threshold),+ name="kompress-canary",+ daemon=True,+ )+ self._canary_thread.start()return backend+ def _run_canary(self, model: Any, tokenizer: Any, backend: str, threshold: float) -> None:+ try:+ elapsed = self._timed_canary(model, tokenizer, backend)+ if elapsed > threshold:+ # One retry: the first Run pays one-off kernel/allocator+ # warmup that shouldn't condemn a healthy machine.+ elapsed = self._timed_canary(model, tokenizer, backend)+ except Exception as e:+ # The canary must never poison the compressor — compress() has+ # its own error handling for whatever is wrong with the model.+ logger.debug("Kompress canary probe skipped: %s", e)+ return++ if elapsed > threshold:+ self._degraded_reason = (+ f"canary inference took {elapsed:.1f}s (threshold {threshold:.1f}s)"+ )+ logger.warning(+ "Kompress canary inference took %.1fs (threshold %.1fs, %d-word probe) — "+ "ML compression DISABLED for this run; content passes through "+ "uncompressed instead of timing out every request. On Windows ensure "+ "the ONNX CPU arena is enabled (%s=1); tune or disable this check "+ "via %s.",+ elapsed,+ threshold,+ len(_canary_words()),+ ONNX_CPU_ARENA_ENV,+ KOMPRESS_CANARY_THRESHOLD_ENV,+ )+ else:+ logger.debug("Kompress canary inference: %.0fms", elapsed * 1000)++ def _timed_canary(self, model: Any, tokenizer: Any, backend: str) -> float:+ """Run one small inference and return its wall-clock seconds."""+ words = _canary_words()+ is_onnx = backend == "onnx"+ encoding = tokenizer(+ words,+ is_split_into_words=True,+ truncation=True,+ max_length=512,+ padding=True,+ return_tensors="np" if is_onnx else "pt",+ )+ input_ids = encoding["input_ids"]+ attention_mask = encoding["attention_mask"]+ if not is_onnx:+ device = next(model.parameters()).device+ input_ids = input_ids.to(device)+ attention_mask = attention_mask.to(device)+ started = time.perf_counter()+ model.get_keep_mask(input_ids, attention_mask)+ return time.perf_counter() - started+def is_ready(self) -> bool:"""True if the model is loaded so :meth:`compress` won't touch the network.@@ -1010,7 +1194,7 @@ class KompressCompressor(Transform):words = content.split()n_words = len(words)- if n_words < 10:+ if n_words < 10 or self._degraded_reason is not None:return self._passthrough(content, n_words)# Cooperative wall-clock budget (#1171): kompress ONNX inference is@@ -1057,7 +1241,23 @@ class KompressCompressor(Transform):chunk_count = 0t_deadline = time.perf_counter()+ acquire_timeout = _acquire_timeout_seconds()+ budget = _time_budget_seconds()+ deadline = time.monotonic() + budget if budget is not None else None+for chunk_start in range(0, n_words, max_chunk_words):+ remaining: float | None = None+ if deadline is not None:+ remaining = deadline - time.monotonic()+ if remaining <= 0:+ _log_giveup(+ "time budget exhausted",+ backend=backend,+ device_type=device_type,+ n_words=n_words,+ )+ return self._passthrough(content, n_words)+if deadline_s and (time.perf_counter() - t_deadline) > deadline_s:# Keep everything from here on verbatim and stop: a partial# compression that returns NOW beats a full one that leaks a@@ -1095,10 +1295,20 @@ class KompressCompressor(Transform):input_ids = input_ids.to(device)attention_mask = attention_mask.to(device)+ acquire_bounds = [+ bound+ for bound in (+ _execution_wait_budget_seconds(),+ acquire_timeout,+ remaining,+ )+ if bound is not None+ ]+ execution_timeout = max(0.0, min(acquire_bounds)) if acquire_bounds else Nonesemaphore, _wait_ms = _acquire_execution_slot(backend,device_type,- timeout_seconds=_execution_wait_budget_seconds(),+ timeout_seconds=execution_timeout,)if semaphore is None:logger.warning(@@ -1290,6 +1500,9 @@ class KompressCompressor(Transform):if n == 0:return []+ if self._degraded_reason is not None:+ return [self._passthrough(c, len(c.split())) for c in contents]+# Normalize target_ratio to a per-text listif isinstance(target_ratio, list):if len(target_ratio) != n:@@ -1364,7 +1577,35 @@ class KompressCompressor(Transform):kept_ids_per_text: dict[int, set[int]] = {i: set() for i in range(n) if results[i] is None}inference_ms = 0.0+ acquire_timeout = _acquire_timeout_seconds()+ budget = _time_budget_seconds()+ deadline = time.monotonic() + budget if budget is not None else None++ def _bail_remaining(reason: str, batch_start: int) -> None:+ # A text with ANY unprocessed chunk must pass through whole —+ # compressing from partial chunk coverage would silently drop the+ # words of the chunks that never ran.+ _log_giveup(+ reason,+ backend=backend,+ device_type=device_type,+ n_words=sum(len(c[2]) for c in chunk_queue[batch_start:]),+ )+ for text_idx, _, _, _ in chunk_queue[batch_start:]:+ if results[text_idx] is None:+ results[text_idx] = self._passthrough(+ contents[text_idx], len(word_lists[text_idx])+ )+ kept_ids_per_text.pop(text_idx, None)+for batch_start in range(0, len(chunk_queue), batch_size):+ remaining: float | None = None+ if deadline is not None:+ remaining = deadline - time.monotonic()+ if remaining <= 0:+ _bail_remaining("time budget exhausted", batch_start)+ break+batch = chunk_queue[batch_start : batch_start + batch_size]batch_word_lists = [c[2] for c in batch]@@ -1388,10 +1629,20 @@ class KompressCompressor(Transform):attention_mask = attention_mask.to(device)# Single forward pass for all chunks in this batch.+ acquire_bounds = [+ bound+ for bound in (+ _execution_wait_budget_seconds(),+ acquire_timeout,+ remaining,+ )+ if bound is not None+ ]+ execution_timeout = max(0.0, min(acquire_bounds)) if acquire_bounds else Nonesemaphore, wait_ms = _acquire_execution_slot(backend,device_type,- timeout_seconds=_execution_wait_budget_seconds(),+ timeout_seconds=execution_timeout,)if semaphore is None:logger.warning(@@ -1399,13 +1650,8 @@ class KompressCompressor(Transform):"passing through remaining batch inputs",wait_ms,)- for text_idx, _, _, _ in batch:- if results[text_idx] is None:- results[text_idx] = self._passthrough(- contents[text_idx], len(word_lists[text_idx])- )- kept_ids_per_text.pop(text_idx, None)- continue+ _bail_remaining("model busy (semaphore acquire timed out)", batch_start)+ breakwith contextlib.ExitStack() as stack:stack.callback(semaphore.release)diff --git a/tests/test_kompress_failsafe.py b/tests/test_kompress_failsafe.pynew file mode 100644index 000000000..9c44a841a--- /dev/null+++ b/tests/test_kompress_failsafe.py@@ -0,0 +1,350 @@+"""Fail-safe behavior for Kompress on degraded machines.++Reproduces the Windows incident where one pathologically slow ONNX inference+held the execution semaphore forever: every later compression blocked on an+unbounded acquire, every request hit the proxy's 30s stage timeout, and the+proxy delivered 0% savings plus +30s latency until restart. These tests pin+the three layers of defense (bounded acquire, wall-clock budget, preload+canary) and that the normal fast path is untouched.++No ML dependencies — the model/tokenizer are fakes injected via+``_load_kompress``.+"""++import time++import pytest++import headroom.transforms.kompress_compressor as kc+from headroom.transforms.kompress_compressor import (+ KOMPRESS_ACQUIRE_TIMEOUT_ENV,+ KOMPRESS_CANARY_THRESHOLD_ENV,+ KOMPRESS_TIME_BUDGET_ENV,+ KompressCompressor,+ KompressConfig,+)+++class FakeEncoding:+ """Mimics a transformers BatchEncoding for is_split_into_words inputs.++ One token per word, no special tokens — word_ids(i) is identity.+ """++ def __init__(self, rows: list[list[str]]):+ self._rows = rows++ def __getitem__(self, key: str):+ if key == "input_ids":+ return [[0] * len(r) for r in self._rows]+ if key == "attention_mask":+ return [[1] * len(r) for r in self._rows]+ raise KeyError(key)++ def word_ids(self, batch_index: int = 0):+ return list(range(len(self._rows[batch_index])))+++class FakeTokenizer:+ def __call__(self, words, **kwargs):+ # is_split_into_words inputs: either one word list or a batch of them.+ rows = words if words and isinstance(words[0], list) else [words]+ return FakeEncoding(rows)+++class FakeModel:+ """Keeps every other word; optional per-call delay to simulate slowness."""++ def __init__(self, delay: float = 0.0):+ self.delay = delay+ self.calls = 0++ def _tick(self):+ self.calls += 1+ if self.delay:+ time.sleep(self.delay)++ def get_keep_mask(self, input_ids, attention_mask):+ self._tick()+ return [[i % 2 == 0 for i in range(len(row))] for row in input_ids]++ def get_scores(self, input_ids, attention_mask):+ self._tick()+ return [[1.0 if i % 2 == 0 else 0.0 for i in range(len(row))] for row in input_ids]++[email protected](autouse=True)+def _reset_module_state(monkeypatch):+ kc._execution_semaphores.clear()+ monkeypatch.setattr(kc, "_giveup_warned", False)+ for env in (+ KOMPRESS_ACQUIRE_TIMEOUT_ENV,+ KOMPRESS_TIME_BUDGET_ENV,+ KOMPRESS_CANARY_THRESHOLD_ENV,+ ):+ monkeypatch.delenv(env, raising=False)+ yield+ kc._execution_semaphores.clear()+++def _make_compressor(monkeypatch, model: FakeModel, **config_kwargs) -> KompressCompressor:+ config_kwargs.setdefault("enable_ccr", False)+ compressor = KompressCompressor(config=KompressConfig(**config_kwargs))+ monkeypatch.setattr(+ kc,+ "_load_kompress",+ lambda model_id, device="auto", **kwargs: (model, FakeTokenizer(), "onnx"),+ )+ return compressor+++CONTENT_40_WORDS = " ".join(f"word{i}" for i in range(40))+++# ── Normal path: behavior and performance must be unchanged ───────────+++def test_fast_model_compresses_normally(monkeypatch):+ model = FakeModel()+ compressor = _make_compressor(monkeypatch, model)++ result = compressor.compress(CONTENT_40_WORDS)++ assert result.compressed != CONTENT_40_WORDS+ assert result.compressed_tokens == 20 # every other word kept+ assert result.compression_ratio == 0.5+ assert model.calls == 1+++def test_fast_model_releases_semaphore(monkeypatch):+ compressor = _make_compressor(monkeypatch, FakeModel())+ compressor.compress(CONTENT_40_WORDS)++ semaphore = kc._execution_semaphore("onnx", "onnx")+ assert semaphore.acquire(timeout=0)+ semaphore.release()+++def test_semaphore_released_when_inference_raises(monkeypatch):+ class ExplodingModel(FakeModel):+ def get_keep_mask(self, input_ids, attention_mask):+ raise RuntimeError("boom")++ compressor = _make_compressor(monkeypatch, ExplodingModel())+ result = compressor.compress(CONTENT_40_WORDS)++ assert result.compressed == CONTENT_40_WORDS # passthrough, not an exception+ semaphore = kc._execution_semaphore("onnx", "onnx")+ assert semaphore.acquire(timeout=0)+ semaphore.release()+++# ── Bounded acquire: a stuck inference must not wedge other requests ──+++def test_stuck_semaphore_passes_through_instead_of_blocking(monkeypatch):+ monkeypatch.setenv(KOMPRESS_ACQUIRE_TIMEOUT_ENV, "0.1")+ model = FakeModel()+ compressor = _make_compressor(monkeypatch, model)++ # Simulate the Windows incident: another thread holds the semaphore+ # indefinitely (abandoned by its asyncio timeout but still running).+ stuck = kc._execution_semaphore("onnx", "onnx")+ assert stuck.acquire(timeout=0)+ try:+ started = time.monotonic()+ result = compressor.compress(CONTENT_40_WORDS)+ elapsed = time.monotonic() - started+ finally:+ stuck.release()++ assert result.compressed == CONTENT_40_WORDS+ assert model.calls == 0+ assert elapsed < 3.0 # used to block forever+++def test_stuck_semaphore_batch_passes_through(monkeypatch):+ monkeypatch.setenv(KOMPRESS_ACQUIRE_TIMEOUT_ENV, "0.1")+ compressor = _make_compressor(monkeypatch, FakeModel())+ monkeypatch.setattr(KompressCompressor, "_should_use_sequential_fallback", lambda self: False)++ stuck = kc._execution_semaphore("onnx", "onnx")+ assert stuck.acquire(timeout=0)+ try:+ contents = [CONTENT_40_WORDS, " ".join(f"x{i}" for i in range(30))]+ results = compressor.compress_batch(contents)+ finally:+ stuck.release()++ assert [r.compressed for r in results] == contents # all passthrough, no data loss+++def test_acquire_bounded_unbounded_when_both_disabled():+ semaphore = kc._execution_semaphore("onnx", "onnx")+ assert kc._acquire_bounded(semaphore, None, None) is True+ semaphore.release()+++def test_acquire_bounded_negative_remaining_does_not_raise():+ semaphore = kc._execution_semaphore("onnx", "onnx")+ assert semaphore.acquire(timeout=0)+ try:+ assert kc._acquire_bounded(semaphore, 5.0, -1.0) is False+ finally:+ semaphore.release()+++# ── Wall-clock budget: give up before the proxy's stage timeout ───────+++def test_time_budget_bails_to_passthrough(monkeypatch):+ monkeypatch.setenv(KOMPRESS_TIME_BUDGET_ENV, "0.2")+ model = FakeModel(delay=0.15)+ compressor = _make_compressor(monkeypatch, model, chunk_words=10)++ result = compressor.compress(CONTENT_40_WORDS) # 4 chunks at ~0.15s each++ assert result.compressed == CONTENT_40_WORDS+ assert model.calls < 4 # bailed before processing every chunk+++def test_time_budget_disabled_processes_all_chunks(monkeypatch):+ monkeypatch.setenv(KOMPRESS_TIME_BUDGET_ENV, "0")+ model = FakeModel(delay=0.01)+ compressor = _make_compressor(monkeypatch, model, chunk_words=10)++ result = compressor.compress(CONTENT_40_WORDS)++ assert model.calls == 4+ assert result.compression_ratio == 0.5+++def test_time_budget_batch_keeps_completed_texts(monkeypatch):+ """Mid-queue bail: fully processed texts stay compressed; any text with+ an unprocessed chunk passes through whole (never partially dropped)."""+ monkeypatch.setenv(KOMPRESS_TIME_BUDGET_ENV, "0.2")+ model = FakeModel(delay=0.25) # one batch alone exhausts the budget+ compressor = _make_compressor(monkeypatch, model)+ monkeypatch.setattr(KompressCompressor, "_should_use_sequential_fallback", lambda self: False)++ contents = [+ " ".join(f"a{i}" for i in range(20)),+ " ".join(f"b{i}" for i in range(20)),+ " ".join(f"c{i}" for i in range(20)),+ ]+ results = compressor.compress_batch(contents, batch_size=1)++ assert len(results) == 3+ # First batch ran; later ones bailed to passthrough.+ assert results[0].compression_ratio == 0.5+ assert results[1].compressed == contents[1]+ assert results[2].compressed == contents[2]+ # Every result preserves all information (compressed or original).+ for r in results:+ assert r.compressed+++# ── Preload canary: detect degraded runtimes before live traffic ──────+++def _join_canary(compressor: KompressCompressor) -> None:+ assert compressor._canary_thread is not None+ compressor._canary_thread.join(timeout=10)+ assert not compressor._canary_thread.is_alive()+++def test_canary_disables_kompress_on_slow_inference(monkeypatch, caplog):+ monkeypatch.setenv(KOMPRESS_CANARY_THRESHOLD_ENV, "0.05")+ model = FakeModel(delay=0.15)+ compressor = _make_compressor(monkeypatch, model)++ with caplog.at_level("WARNING"):+ backend = compressor.preload()+ _join_canary(compressor)++ assert backend == "onnx"+ assert compressor._degraded_reason is not None+ assert model.calls == 2 # probe + one retry+ assert "DISABLED" in caplog.text++ result = compressor.compress(CONTENT_40_WORDS)+ assert result.compressed == CONTENT_40_WORDS+ assert model.calls == 2 # model never touched again++ batch = compressor.compress_batch([CONTENT_40_WORDS])+ assert batch[0].compressed == CONTENT_40_WORDS+ assert model.calls == 2+++def test_canary_fast_inference_stays_enabled(monkeypatch):+ monkeypatch.setenv(KOMPRESS_CANARY_THRESHOLD_ENV, "5")+ model = FakeModel()+ compressor = _make_compressor(monkeypatch, model)++ compressor.preload()+ _join_canary(compressor)++ assert compressor._degraded_reason is None+ result = compressor.compress(CONTENT_40_WORDS)+ assert result.compression_ratio == 0.5+++def test_canary_retry_forgives_oneoff_warmup_slowness(monkeypatch):+ """First inference pays one-off warmup costs; only a slow retry condemns."""+ monkeypatch.setenv(KOMPRESS_CANARY_THRESHOLD_ENV, "0.1")++ class WarmupModel(FakeModel):+ def get_keep_mask(self, input_ids, attention_mask):+ self.calls += 1+ if self.calls == 1:+ time.sleep(0.2) # cold first run+ return [[i % 2 == 0 for i in range(len(row))] for row in input_ids]++ model = WarmupModel()+ compressor = _make_compressor(monkeypatch, model)+ compressor.preload()+ _join_canary(compressor)++ assert compressor._degraded_reason is None+ assert model.calls == 2+++def test_canary_disabled_via_env(monkeypatch):+ monkeypatch.setenv(KOMPRESS_CANARY_THRESHOLD_ENV, "0")+ model = FakeModel(delay=0.2)+ compressor = _make_compressor(monkeypatch, model)++ compressor.preload()++ assert compressor._canary_thread is None # probe never scheduled+ assert model.calls == 0+ assert compressor._degraded_reason is None+++def test_preload_does_not_block_on_slow_canary(monkeypatch):+ """The probe runs off the startup path: preload blocks proxy boot (the+ HTTP server binds after it), and a slow probe once pushed the wrap-e2e+ container past its 30s health-check timeout."""+ monkeypatch.setenv(KOMPRESS_CANARY_THRESHOLD_ENV, "0.05")+ model = FakeModel(delay=1.0)+ compressor = _make_compressor(monkeypatch, model)++ started = time.monotonic()+ compressor.preload()+ preload_elapsed = time.monotonic() - started++ assert preload_elapsed < 0.5 # returns before the ~2s of probe inference+ _join_canary(compressor)+ assert compressor._degraded_reason is not None+++def test_canary_probe_error_never_breaks_preload(monkeypatch):+ class ExplodingModel(FakeModel):+ def get_keep_mask(self, input_ids, attention_mask):+ raise RuntimeError("probe boom")++ compressor = _make_compressor(monkeypatch, ExplodingModel())++ assert compressor.preload() == "onnx"+ _join_canary(compressor)+ assert compressor._degraded_reason is Nonediff --git a/tests/test_onnx_runtime.py b/tests/test_onnx_runtime.pyindex 0f0c192cf..076afd2eb 100644--- a/tests/test_onnx_runtime.py+++ b/tests/test_onnx_runtime.py@@ -1,4 +1,10 @@-from headroom.onnx_runtime import create_cpu_session_options+import sys++from headroom.onnx_runtime import (+ ONNX_CPU_ARENA_ENV,+ cpu_arena_enabled,+ create_cpu_session_options,+)class _FakeSessionOptions:@@ -23,7 +29,11 @@ class _FakeOrtWithoutToggles:SessionOptions = _FakeSessionOptionsWithoutToggles-def test_create_cpu_session_options_disables_retention_features():+def test_create_cpu_session_options_disables_retention_features(monkeypatch):+ """Non-Windows keeps the legacy low-RSS behavior: arena + mem pattern off."""+ monkeypatch.delenv(ONNX_CPU_ARENA_ENV, raising=False)+ monkeypatch.setattr(sys, "platform", "linux")+options = create_cpu_session_options(_FakeOrt,intra_op_num_threads=1,@@ -36,7 +46,60 @@ def test_create_cpu_session_options_disables_retention_features():assert options.enable_mem_pattern is False-def test_create_cpu_session_options_handles_older_session_options():+def test_create_cpu_session_options_darwin_unchanged(monkeypatch):+ monkeypatch.delenv(ONNX_CPU_ARENA_ENV, raising=False)+ monkeypatch.setattr(sys, "platform", "darwin")++ options = create_cpu_session_options(_FakeOrt)++ assert options.enable_cpu_mem_arena is False+ assert options.enable_mem_pattern is False+++def test_create_cpu_session_options_keeps_arena_on_windows(monkeypatch):+ """Disabling the arena on Windows degrades inference by orders of+ magnitude (onnxruntime#11627) — ORT defaults must stay untouched there."""+ monkeypatch.delenv(ONNX_CPU_ARENA_ENV, raising=False)+ monkeypatch.setattr(sys, "platform", "win32")++ options = create_cpu_session_options(_FakeOrt, intra_op_num_threads=3)++ assert options.enable_cpu_mem_arena is True+ assert options.enable_mem_pattern is True+ assert options.intra_op_num_threads == 3+++def test_arena_env_override_forces_on(monkeypatch):+ monkeypatch.setattr(sys, "platform", "linux")+ monkeypatch.setenv(ONNX_CPU_ARENA_ENV, "1")++ assert cpu_arena_enabled() is True+ options = create_cpu_session_options(_FakeOrt)+ assert options.enable_cpu_mem_arena is True+++def test_arena_env_override_forces_off(monkeypatch):+ monkeypatch.setattr(sys, "platform", "win32")+ monkeypatch.setenv(ONNX_CPU_ARENA_ENV, "0")++ assert cpu_arena_enabled() is False+ options = create_cpu_session_options(_FakeOrt)+ assert options.enable_cpu_mem_arena is False+++def test_arena_env_invalid_falls_back_to_platform_default(monkeypatch):+ monkeypatch.setenv(ONNX_CPU_ARENA_ENV, "bananas")++ monkeypatch.setattr(sys, "platform", "win32")+ assert cpu_arena_enabled() is True+ monkeypatch.setattr(sys, "platform", "linux")+ assert cpu_arena_enabled() is False+++def test_create_cpu_session_options_handles_older_session_options(monkeypatch):+ monkeypatch.delenv(ONNX_CPU_ARENA_ENV, raising=False)+ monkeypatch.setattr(sys, "platform", "linux")+options = create_cpu_session_options(_FakeOrtWithoutToggles)assert options.intra_op_num_threads is Nonediff --git a/tests/test_subprocess_encoding.py b/tests/test_subprocess_encoding.pynew file mode 100644index 000000000..0e84df980--- /dev/null+++ b/tests/test_subprocess_encoding.py@@ -0,0 +1,55 @@+"""Proxy-runtime subprocess calls must decode UTF-8 explicitly.++On Windows, ``subprocess.run(text=True)`` without an ``encoding`` decodes+child output with the console code page (cp1252). rtk's emoji-laden output+then kills the reader thread with ``UnicodeDecodeError: 'charmap' codec+can't decode byte ...`` (seen in user proxy logs). These tests pin the+``encoding="utf-8"`` kwarg on every proxy-runtime subprocess call.+"""++import subprocess++import headroom.lean_ctx+import headroom.proxy.helpers as helpers+import headroom.rtk+from headroom.proxy.interceptors import astgrep+++def _capture_run(captured, returncode=0, stdout='{"summary": {}}'):+ def fake_run(cmd, **kwargs):+ captured.update(kwargs)+ return subprocess.CompletedProcess(cmd, returncode, stdout=stdout, stderr="")++ return fake_run+++def test_rtk_stats_subprocess_uses_utf8(monkeypatch):+ captured: dict = {}+ monkeypatch.setattr(helpers, "run", _capture_run(captured))+ monkeypatch.setattr(headroom.rtk, "get_rtk_path", lambda: "/fake/rtk")++ helpers._read_rtk_lifetime_stats()++ assert captured["encoding"] == "utf-8"+ assert captured["errors"] == "replace"+++def test_lean_ctx_stats_subprocess_uses_utf8(monkeypatch):+ captured: dict = {}+ monkeypatch.setattr(helpers, "run", _capture_run(captured))+ monkeypatch.setattr(headroom.lean_ctx, "get_lean_ctx_path", lambda: "/fake/lean-ctx")++ helpers._read_lean_ctx_lifetime_stats()++ assert captured["encoding"] == "utf-8"+ assert captured["errors"] == "replace"+++def test_ast_grep_subprocess_uses_utf8(monkeypatch):+ captured: dict = {}+ monkeypatch.setattr(astgrep.subprocess, "run", _capture_run(captured, returncode=1, stdout=""))++ astgrep._run_ast_grep("/fake/sg", "python", "def foo():\n pass\n")++ assert captured["encoding"] == "utf-8"+ assert captured["errors"] == "replace"