Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(windows): unwedge compression on degraded ONNX runtimes (every request timing out at 30s, 0% savings) (#822)

Tejas Chopra36202f4d0bd4
diff --git a/headroom/onnx_runtime.py b/headroom/onnx_runtime.py
index 09306fe12..0ee6353a4 100644
--- a/headroom/onnx_runtime.py
+++ b/headroom/onnx_runtime.py
@@ -3,10 +3,49 @@
from __future__ import annotations
import ctypes
+import logging
import os
import sys
from typing import Any
+logger = logging.getLogger(__name__)
+
+# Override for the CPU memory-arena default below: "1"/"true" forces the
+# arena ON, "0"/"false" forces it OFF, unset/"auto" uses the platform default.
+ONNX_CPU_ARENA_ENV = "HEADROOM_ONNX_CPU_ARENA"
+
+_TRUTHY = frozenset({"1", "true", "yes", "on"})
+_FALSY = frozenset({"0", "false", "no", "off"})
+
+
+def _env_flag(name: str) -> bool | None:
+ raw = os.environ.get(name, "").strip().lower()
+ if raw in _TRUTHY:
+ return True
+ if raw in _FALSY:
+ return False
+ if raw and raw != "auto":
+ logger.warning("%s must be a boolean or 'auto', got %r; using auto", name, raw)
+ return None
+
+
+def cpu_arena_enabled() -> bool:
+ """Whether ONNX Runtime's CPU memory arena should stay enabled.
+
+ Disabling the arena trades peak throughput for lower retained RSS, which
+ is the right call on the small Linux VMs Headroom commonly runs on. On
+ Windows the same setting is catastrophic: without the arena every
+ ``Run()`` falls back to per-node VirtualAlloc/free, slowing transformer
+ inference by 2-3 orders of magnitude (onnxruntime#11627). That turned
+ every compression into a >30s timeout for Windows proxy users, so the
+ arena stays at ORT's default (enabled) there.
+ """
+ override = _env_flag(ONNX_CPU_ARENA_ENV)
+ if override is not None:
+ return override
+ return sys.platform == "win32"
+
+
# Pin model artifacts to immutable commit SHAs so a changed or compromised
# upstream HuggingFace repo cannot be pulled silently (supply-chain integrity).
# Repos not listed here fall back to the floating default ref. Set
@@ -90,6 +129,9 @@ def create_cpu_session_options(
memory usage over peak ONNX throughput. Disabling ORT's CPU arena and memory
pattern caches reduces retained anonymous RSS after variable-size inference
workloads, which is especially important on small VMs.
+
+ The arena is left at ORT's default on Windows (see ``cpu_arena_enabled``),
+ where disabling it degrades inference latency by orders of magnitude.
"""
sess_options = ort.SessionOptions()
@@ -98,10 +140,11 @@ def create_cpu_session_options(
if inter_op_num_threads is not None:
sess_options.inter_op_num_threads = inter_op_num_threads
- if hasattr(sess_options, "enable_cpu_mem_arena"):
- sess_options.enable_cpu_mem_arena = False
- if hasattr(sess_options, "enable_mem_pattern"):
- sess_options.enable_mem_pattern = False
+ if not cpu_arena_enabled():
+ if hasattr(sess_options, "enable_cpu_mem_arena"):
+ sess_options.enable_cpu_mem_arena = False
+ if hasattr(sess_options, "enable_mem_pattern"):
+ sess_options.enable_mem_pattern = False
return sess_options
diff --git a/headroom/proxy/handlers/openai.py b/headroom/proxy/handlers/openai.py
index 184ecb7ec..1cc8a7bb5 100644
--- a/headroom/proxy/handlers/openai.py
+++ b/headroom/proxy/handlers/openai.py
@@ -2846,8 +2846,11 @@ class OpenAIHandlerMixin:
if result.waste_signals:
waste_signals_dict = result.waste_signals.to_dict()
except Exception as e:
+ # Include type so TimeoutError vs other failures is distinguishable
+ # in bug reports — str(asyncio.TimeoutError()) is empty otherwise.
logger.warning(
- f"Optimization failed: {type(e).__name__}: {e}",
+ f"[{request_id}] Optimization failed: {type(e).__name__}: {e} "
+ f"(request forwarded unoptimized)",
exc_info=True,
)
# Flag compression failure for observability
diff --git a/headroom/proxy/helpers.py b/headroom/proxy/helpers.py
index 13fddbbcc..26680d574 100644
--- a/headroom/proxy/helpers.py
+++ b/headroom/proxy/helpers.py
@@ -1128,6 +1128,10 @@ def _read_rtk_lifetime_stats() -> dict[str, Any] | None:
_rtk_gain_command(rtk_path, scope),
capture_output=True,
text=True,
+ # rtk output is UTF-8 (emoji etc.); without this, Windows decodes
+ # with cp1252 and the reader thread dies with UnicodeDecodeError.
+ encoding="utf-8",
+ errors="replace",
timeout=5,
)
if result.returncode == 0 and result.stdout.strip():
@@ -1177,6 +1181,9 @@ def _read_lean_ctx_lifetime_stats() -> dict[str, Any] | None:
[str(lean_ctx_path), "gain", "--json"],
capture_output=True,
text=True,
+ # UTF-8 regardless of the Windows console code page (cp1252).
+ encoding="utf-8",
+ errors="replace",
timeout=5,
)
# Failed reads return None ("no data") — mirrors the rtk reader so
diff --git a/headroom/proxy/interceptors/astgrep.py b/headroom/proxy/interceptors/astgrep.py
index 2b896df36..460192892 100644
--- a/headroom/proxy/interceptors/astgrep.py
+++ b/headroom/proxy/interceptors/astgrep.py
@@ -214,6 +214,10 @@ def _run_ast_grep(
],
capture_output=True,
text=True,
+ # ast-grep emits UTF-8 (it matches arbitrary source text);
+ # without this, Windows decodes with cp1252 and raises.
+ encoding="utf-8",
+ errors="replace",
timeout=5,
check=False,
)
diff --git a/headroom/transforms/kompress_compressor.py b/headroom/transforms/kompress_compressor.py
index 560261914..c8e8bb057 100644
--- a/headroom/transforms/kompress_compressor.py
+++ b/headroom/transforms/kompress_compressor.py
@@ -27,6 +27,7 @@ from typing import Any, Literal
from ..config import TransformResult
from ..onnx_runtime import (
+ ONNX_CPU_ARENA_ENV,
create_cpu_session_options,
hf_hub_download_local_first,
trim_process_heap,
@@ -97,6 +98,18 @@ KOMPRESS_MAX_CONCURRENT_ENV = "HEADROOM_KOMPRESS_MAX_CONCURRENT"
KOMPRESS_EXECUTION_SEMAPHORE_WAIT_MS_ENV = "HEADROOM_KOMPRESS_EXECUTION_TIMEOUT_MS"
KOMPRESS_EXECUTION_SEMAPHORE_WAIT_MS_DEFAULT = 25
KOMPRESS_BATCH_SIZE_ENV = "HEADROOM_KOMPRESS_BATCH_SIZE"
+KOMPRESS_ACQUIRE_TIMEOUT_ENV = "HEADROOM_KOMPRESS_ACQUIRE_TIMEOUT_SECONDS"
+KOMPRESS_TIME_BUDGET_ENV = "HEADROOM_KOMPRESS_TIME_BUDGET_SECONDS"
+KOMPRESS_CANARY_THRESHOLD_ENV = "HEADROOM_KOMPRESS_CANARY_SECONDS"
+
+# Both defaults sit well under the proxy's 30s compression-stage timeout so a
+# slow model gives up (passthrough) before the request is abandoned. A thread
+# abandoned by asyncio.wait_for cannot be killed; before these bounds existed,
+# one pathologically slow inference would hold the execution semaphore forever
+# and wedge every subsequent compression in the process.
+_DEFAULT_ACQUIRE_TIMEOUT_SECONDS = 5.0
+_DEFAULT_TIME_BUDGET_SECONDS = 20.0
+_DEFAULT_CANARY_THRESHOLD_SECONDS = 5.0
KompressBackend = Literal["auto", "onnx", "onnx_cpu", "onnx_coreml", "pytorch", "pytorch_mps"]
@@ -244,6 +257,80 @@ def _env_int(name: str) -> int | None:
return value
+def _env_float(name: str) -> float | None:
+ raw = os.environ.get(name)
+ if raw is None or raw.strip() == "":
+ return None
+ try:
+ return float(raw)
+ except ValueError:
+ logger.warning("%s must be a number, got %r; ignoring", name, raw)
+ return None
+
+
+def _acquire_timeout_seconds() -> float | None:
+ """Max seconds to wait for the execution semaphore. <=0 disables the bound."""
+ raw = _env_float(KOMPRESS_ACQUIRE_TIMEOUT_ENV)
+ if raw is None:
+ return _DEFAULT_ACQUIRE_TIMEOUT_SECONDS
+ return raw if raw > 0 else None
+
+
+def _time_budget_seconds() -> float | None:
+ """Wall-clock budget for one compress/compress_batch call. <=0 disables."""
+ raw = _env_float(KOMPRESS_TIME_BUDGET_ENV)
+ if raw is None:
+ return _DEFAULT_TIME_BUDGET_SECONDS
+ return raw if raw > 0 else None
+
+
+def _canary_threshold_seconds() -> float | None:
+ """Startup canary threshold; inference slower than this disables Kompress.
+
+ <=0 disables the canary entirely.
+ """
+ raw = _env_float(KOMPRESS_CANARY_THRESHOLD_ENV)
+ if raw is None:
+ return _DEFAULT_CANARY_THRESHOLD_SECONDS
+ return raw if raw > 0 else None
+
+
+_CANARY_SENTENCE = (
+ "Headroom probes model latency at startup so a degraded runtime is "
+ "detected before live traffic depends on it."
+)
+
+
+def _canary_words(target_words: int = 120) -> list[str]:
+ """Deterministic probe input sized like a small real compression chunk."""
+ words = _CANARY_SENTENCE.split()
+ reps = (target_words // len(words)) + 1
+ return (words * reps)[:target_words]
+
+
+# First give-up is a WARNING with remediation hints; repeats drop to DEBUG so a
+# persistently slow machine doesn't emit one warning per compressible message.
+_giveup_warned = False
+
+
+def _log_giveup(reason: str, *, backend: str, device_type: str, n_words: int) -> None:
+ global _giveup_warned
+ level = logging.DEBUG if _giveup_warned else logging.WARNING
+ _giveup_warned = True
+ logger.log(
+ level,
+ "Kompress giving up (%s) backend=%s device=%s words=%d — content passes "
+ "through uncompressed. If this persists, ML inference on this machine is "
+ "too slow for inline compression; tune %s / %s or disable Kompress.",
+ reason,
+ backend,
+ device_type,
+ n_words,
+ KOMPRESS_TIME_BUDGET_ENV,
+ KOMPRESS_ACQUIRE_TIMEOUT_ENV,
+ )
+
+
def _onnx_session_options(ort: Any) -> Any:
return create_cpu_session_options(
ort,
@@ -290,6 +377,25 @@ def _execution_semaphore(backend: str, device_type: str) -> threading.BoundedSem
return semaphore
+def _acquire_bounded(
+ semaphore: threading.BoundedSemaphore,
+ acquire_timeout: float | None,
+ remaining_budget: float | None,
+) -> bool:
+ """Acquire ``semaphore``, waiting at most the tighter of the two bounds.
+
+ Returns False on timeout. An unbounded wait here is how one stuck
+ inference (e.g. degraded ONNX on Windows) used to wedge every other
+ compression in the process: abandoned executor threads queued forever
+ on the semaphore. With both bounds disabled (<=0) this degrades to the
+ legacy blocking acquire.
+ """
+ bounds = [b for b in (acquire_timeout, remaining_budget) if b is not None]
+ if not bounds:
+ return semaphore.acquire()
+ return semaphore.acquire(timeout=max(0.0, min(bounds)))
+
+
def _batch_size() -> int:
return _env_int(KOMPRESS_BATCH_SIZE_ENV) or 32
@@ -942,6 +1048,10 @@ class KompressCompressor(Transform):
def __init__(self, config: KompressConfig | None = None):
self.config = config or KompressConfig()
+ # Set by the preload canary when inference is too slow to be useful;
+ # compress()/compress_batch() then pass content through untouched.
+ self._degraded_reason: str | None = None
+ self._canary_thread: threading.Thread | None = None
def preload(self, *, allow_download: bool = True) -> str:
"""Load the backing model/tokenizer and return the selected backend.
@@ -951,13 +1061,87 @@ class KompressCompressor(Transform):
raised so the caller can defer the download to first use. Startup eager
preload uses this so a cold cache cannot block the proxy from binding
its port.
+
+ On a successful load this also kicks off a timed canary inference on a
+ background thread: a machine where one small forward pass takes
+ multiple seconds (seen with degraded ONNX runtimes on Windows) can
+ never finish real compression inside the proxy's stage timeout, so
+ Kompress fail-safes to passthrough once the probe verdict lands. The
+ probe must NOT run inline — preload blocks proxy startup (the HTTP
+ server binds after it), and on slow hardware the probe itself takes
+ tens of seconds. Until the verdict, the per-call acquire/budget bounds
+ keep any slow inference from wedging requests.
"""
- _model, _tokenizer, backend = _load_kompress(
+ model, tokenizer, backend = _load_kompress(
self.config.model_id, self.config.device, allow_download=allow_download
)
+
+ threshold = _canary_threshold_seconds()
+ if threshold is not None:
+ self._canary_thread = threading.Thread(
+ target=self._run_canary,
+ args=(model, tokenizer, backend, threshold),
+ name="kompress-canary",
+ daemon=True,
+ )
+ self._canary_thread.start()
return backend
+ def _run_canary(self, model: Any, tokenizer: Any, backend: str, threshold: float) -> None:
+ try:
+ elapsed = self._timed_canary(model, tokenizer, backend)
+ if elapsed > threshold:
+ # One retry: the first Run pays one-off kernel/allocator
+ # warmup that shouldn't condemn a healthy machine.
+ elapsed = self._timed_canary(model, tokenizer, backend)
+ except Exception as e:
+ # The canary must never poison the compressor — compress() has
+ # its own error handling for whatever is wrong with the model.
+ logger.debug("Kompress canary probe skipped: %s", e)
+ return
+
+ if elapsed > threshold:
+ self._degraded_reason = (
+ f"canary inference took {elapsed:.1f}s (threshold {threshold:.1f}s)"
+ )
+ logger.warning(
+ "Kompress canary inference took %.1fs (threshold %.1fs, %d-word probe) — "
+ "ML compression DISABLED for this run; content passes through "
+ "uncompressed instead of timing out every request. On Windows ensure "
+ "the ONNX CPU arena is enabled (%s=1); tune or disable this check "
+ "via %s.",
+ elapsed,
+ threshold,
+ len(_canary_words()),
+ ONNX_CPU_ARENA_ENV,
+ KOMPRESS_CANARY_THRESHOLD_ENV,
+ )
+ else:
+ logger.debug("Kompress canary inference: %.0fms", elapsed * 1000)
+
+ def _timed_canary(self, model: Any, tokenizer: Any, backend: str) -> float:
+ """Run one small inference and return its wall-clock seconds."""
+ words = _canary_words()
+ is_onnx = backend == "onnx"
+ encoding = tokenizer(
+ words,
+ is_split_into_words=True,
+ truncation=True,
+ max_length=512,
+ padding=True,
+ return_tensors="np" if is_onnx else "pt",
+ )
+ input_ids = encoding["input_ids"]
+ attention_mask = encoding["attention_mask"]
+ if not is_onnx:
+ device = next(model.parameters()).device
+ input_ids = input_ids.to(device)
+ attention_mask = attention_mask.to(device)
+ started = time.perf_counter()
+ model.get_keep_mask(input_ids, attention_mask)
+ return time.perf_counter() - started
+
def is_ready(self) -> bool:
"""True if the model is loaded so :meth:`compress` won't touch the network.
@@ -1010,7 +1194,7 @@ class KompressCompressor(Transform):
words = content.split()
n_words = len(words)
- if n_words < 10:
+ if n_words < 10 or self._degraded_reason is not None:
return self._passthrough(content, n_words)
# Cooperative wall-clock budget (#1171): kompress ONNX inference is
@@ -1057,7 +1241,23 @@ class KompressCompressor(Transform):
chunk_count = 0
t_deadline = time.perf_counter()
+ acquire_timeout = _acquire_timeout_seconds()
+ budget = _time_budget_seconds()
+ deadline = time.monotonic() + budget if budget is not None else None
+
for chunk_start in range(0, n_words, max_chunk_words):
+ remaining: float | None = None
+ if deadline is not None:
+ remaining = deadline - time.monotonic()
+ if remaining <= 0:
+ _log_giveup(
+ "time budget exhausted",
+ backend=backend,
+ device_type=device_type,
+ n_words=n_words,
+ )
+ return self._passthrough(content, n_words)
+
if deadline_s and (time.perf_counter() - t_deadline) > deadline_s:
# Keep everything from here on verbatim and stop: a partial
# compression that returns NOW beats a full one that leaks a
@@ -1095,10 +1295,20 @@ class KompressCompressor(Transform):
input_ids = input_ids.to(device)
attention_mask = attention_mask.to(device)
+ acquire_bounds = [
+ bound
+ for bound in (
+ _execution_wait_budget_seconds(),
+ acquire_timeout,
+ remaining,
+ )
+ if bound is not None
+ ]
+ execution_timeout = max(0.0, min(acquire_bounds)) if acquire_bounds else None
semaphore, _wait_ms = _acquire_execution_slot(
backend,
device_type,
- timeout_seconds=_execution_wait_budget_seconds(),
+ timeout_seconds=execution_timeout,
)
if semaphore is None:
logger.warning(
@@ -1290,6 +1500,9 @@ class KompressCompressor(Transform):
if n == 0:
return []
+ if self._degraded_reason is not None:
+ return [self._passthrough(c, len(c.split())) for c in contents]
+
# Normalize target_ratio to a per-text list
if isinstance(target_ratio, list):
if len(target_ratio) != n:
@@ -1364,7 +1577,35 @@ class KompressCompressor(Transform):
kept_ids_per_text: dict[int, set[int]] = {i: set() for i in range(n) if results[i] is None}
inference_ms = 0.0
+ acquire_timeout = _acquire_timeout_seconds()
+ budget = _time_budget_seconds()
+ deadline = time.monotonic() + budget if budget is not None else None
+
+ def _bail_remaining(reason: str, batch_start: int) -> None:
+ # A text with ANY unprocessed chunk must pass through whole —
+ # compressing from partial chunk coverage would silently drop the
+ # words of the chunks that never ran.
+ _log_giveup(
+ reason,
+ backend=backend,
+ device_type=device_type,
+ n_words=sum(len(c[2]) for c in chunk_queue[batch_start:]),
+ )
+ for text_idx, _, _, _ in chunk_queue[batch_start:]:
+ if results[text_idx] is None:
+ results[text_idx] = self._passthrough(
+ contents[text_idx], len(word_lists[text_idx])
+ )
+ kept_ids_per_text.pop(text_idx, None)
+
for batch_start in range(0, len(chunk_queue), batch_size):
+ remaining: float | None = None
+ if deadline is not None:
+ remaining = deadline - time.monotonic()
+ if remaining <= 0:
+ _bail_remaining("time budget exhausted", batch_start)
+ break
+
batch = chunk_queue[batch_start : batch_start + batch_size]
batch_word_lists = [c[2] for c in batch]
@@ -1388,10 +1629,20 @@ class KompressCompressor(Transform):
attention_mask = attention_mask.to(device)
# Single forward pass for all chunks in this batch.
+ acquire_bounds = [
+ bound
+ for bound in (
+ _execution_wait_budget_seconds(),
+ acquire_timeout,
+ remaining,
+ )
+ if bound is not None
+ ]
+ execution_timeout = max(0.0, min(acquire_bounds)) if acquire_bounds else None
semaphore, wait_ms = _acquire_execution_slot(
backend,
device_type,
- timeout_seconds=_execution_wait_budget_seconds(),
+ timeout_seconds=execution_timeout,
)
if semaphore is None:
logger.warning(
@@ -1399,13 +1650,8 @@ class KompressCompressor(Transform):
"passing through remaining batch inputs",
wait_ms,
)
- for text_idx, _, _, _ in batch:
- if results[text_idx] is None:
- results[text_idx] = self._passthrough(
- contents[text_idx], len(word_lists[text_idx])
- )
- kept_ids_per_text.pop(text_idx, None)
- continue
+ _bail_remaining("model busy (semaphore acquire timed out)", batch_start)
+ break
with contextlib.ExitStack() as stack:
stack.callback(semaphore.release)
diff --git a/tests/test_kompress_failsafe.py b/tests/test_kompress_failsafe.py
new file mode 100644
index 000000000..9c44a841a
--- /dev/null
+++ b/tests/test_kompress_failsafe.py
@@ -0,0 +1,350 @@
+"""Fail-safe behavior for Kompress on degraded machines.
+
+Reproduces the Windows incident where one pathologically slow ONNX inference
+held the execution semaphore forever: every later compression blocked on an
+unbounded acquire, every request hit the proxy's 30s stage timeout, and the
+proxy delivered 0% savings plus +30s latency until restart. These tests pin
+the three layers of defense (bounded acquire, wall-clock budget, preload
+canary) and that the normal fast path is untouched.
+
+No ML dependencies — the model/tokenizer are fakes injected via
+``_load_kompress``.
+"""
+
+import time
+
+import pytest
+
+import headroom.transforms.kompress_compressor as kc
+from headroom.transforms.kompress_compressor import (
+ KOMPRESS_ACQUIRE_TIMEOUT_ENV,
+ KOMPRESS_CANARY_THRESHOLD_ENV,
+ KOMPRESS_TIME_BUDGET_ENV,
+ KompressCompressor,
+ KompressConfig,
+)
+
+
+class FakeEncoding:
+ """Mimics a transformers BatchEncoding for is_split_into_words inputs.
+
+ One token per word, no special tokens — word_ids(i) is identity.
+ """
+
+ def __init__(self, rows: list[list[str]]):
+ self._rows = rows
+
+ def __getitem__(self, key: str):
+ if key == "input_ids":
+ return [[0] * len(r) for r in self._rows]
+ if key == "attention_mask":
+ return [[1] * len(r) for r in self._rows]
+ raise KeyError(key)
+
+ def word_ids(self, batch_index: int = 0):
+ return list(range(len(self._rows[batch_index])))
+
+
+class FakeTokenizer:
+ def __call__(self, words, **kwargs):
+ # is_split_into_words inputs: either one word list or a batch of them.
+ rows = words if words and isinstance(words[0], list) else [words]
+ return FakeEncoding(rows)
+
+
+class FakeModel:
+ """Keeps every other word; optional per-call delay to simulate slowness."""
+
+ def __init__(self, delay: float = 0.0):
+ self.delay = delay
+ self.calls = 0
+
+ def _tick(self):
+ self.calls += 1
+ if self.delay:
+ time.sleep(self.delay)
+
+ def get_keep_mask(self, input_ids, attention_mask):
+ self._tick()
+ return [[i % 2 == 0 for i in range(len(row))] for row in input_ids]
+
+ def get_scores(self, input_ids, attention_mask):
+ self._tick()
+ return [[1.0 if i % 2 == 0 else 0.0 for i in range(len(row))] for row in input_ids]
+
+
[email protected](autouse=True)
+def _reset_module_state(monkeypatch):
+ kc._execution_semaphores.clear()
+ monkeypatch.setattr(kc, "_giveup_warned", False)
+ for env in (
+ KOMPRESS_ACQUIRE_TIMEOUT_ENV,
+ KOMPRESS_TIME_BUDGET_ENV,
+ KOMPRESS_CANARY_THRESHOLD_ENV,
+ ):
+ monkeypatch.delenv(env, raising=False)
+ yield
+ kc._execution_semaphores.clear()
+
+
+def _make_compressor(monkeypatch, model: FakeModel, **config_kwargs) -> KompressCompressor:
+ config_kwargs.setdefault("enable_ccr", False)
+ compressor = KompressCompressor(config=KompressConfig(**config_kwargs))
+ monkeypatch.setattr(
+ kc,
+ "_load_kompress",
+ lambda model_id, device="auto", **kwargs: (model, FakeTokenizer(), "onnx"),
+ )
+ return compressor
+
+
+CONTENT_40_WORDS = " ".join(f"word{i}" for i in range(40))
+
+
+# ── Normal path: behavior and performance must be unchanged ───────────
+
+
+def test_fast_model_compresses_normally(monkeypatch):
+ model = FakeModel()
+ compressor = _make_compressor(monkeypatch, model)
+
+ result = compressor.compress(CONTENT_40_WORDS)
+
+ assert result.compressed != CONTENT_40_WORDS
+ assert result.compressed_tokens == 20 # every other word kept
+ assert result.compression_ratio == 0.5
+ assert model.calls == 1
+
+
+def test_fast_model_releases_semaphore(monkeypatch):
+ compressor = _make_compressor(monkeypatch, FakeModel())
+ compressor.compress(CONTENT_40_WORDS)
+
+ semaphore = kc._execution_semaphore("onnx", "onnx")
+ assert semaphore.acquire(timeout=0)
+ semaphore.release()
+
+
+def test_semaphore_released_when_inference_raises(monkeypatch):
+ class ExplodingModel(FakeModel):
+ def get_keep_mask(self, input_ids, attention_mask):
+ raise RuntimeError("boom")
+
+ compressor = _make_compressor(monkeypatch, ExplodingModel())
+ result = compressor.compress(CONTENT_40_WORDS)
+
+ assert result.compressed == CONTENT_40_WORDS # passthrough, not an exception
+ semaphore = kc._execution_semaphore("onnx", "onnx")
+ assert semaphore.acquire(timeout=0)
+ semaphore.release()
+
+
+# ── Bounded acquire: a stuck inference must not wedge other requests ──
+
+
+def test_stuck_semaphore_passes_through_instead_of_blocking(monkeypatch):
+ monkeypatch.setenv(KOMPRESS_ACQUIRE_TIMEOUT_ENV, "0.1")
+ model = FakeModel()
+ compressor = _make_compressor(monkeypatch, model)
+
+ # Simulate the Windows incident: another thread holds the semaphore
+ # indefinitely (abandoned by its asyncio timeout but still running).
+ stuck = kc._execution_semaphore("onnx", "onnx")
+ assert stuck.acquire(timeout=0)
+ try:
+ started = time.monotonic()
+ result = compressor.compress(CONTENT_40_WORDS)
+ elapsed = time.monotonic() - started
+ finally:
+ stuck.release()
+
+ assert result.compressed == CONTENT_40_WORDS
+ assert model.calls == 0
+ assert elapsed < 3.0 # used to block forever
+
+
+def test_stuck_semaphore_batch_passes_through(monkeypatch):
+ monkeypatch.setenv(KOMPRESS_ACQUIRE_TIMEOUT_ENV, "0.1")
+ compressor = _make_compressor(monkeypatch, FakeModel())
+ monkeypatch.setattr(KompressCompressor, "_should_use_sequential_fallback", lambda self: False)
+
+ stuck = kc._execution_semaphore("onnx", "onnx")
+ assert stuck.acquire(timeout=0)
+ try:
+ contents = [CONTENT_40_WORDS, " ".join(f"x{i}" for i in range(30))]
+ results = compressor.compress_batch(contents)
+ finally:
+ stuck.release()
+
+ assert [r.compressed for r in results] == contents # all passthrough, no data loss
+
+
+def test_acquire_bounded_unbounded_when_both_disabled():
+ semaphore = kc._execution_semaphore("onnx", "onnx")
+ assert kc._acquire_bounded(semaphore, None, None) is True
+ semaphore.release()
+
+
+def test_acquire_bounded_negative_remaining_does_not_raise():
+ semaphore = kc._execution_semaphore("onnx", "onnx")
+ assert semaphore.acquire(timeout=0)
+ try:
+ assert kc._acquire_bounded(semaphore, 5.0, -1.0) is False
+ finally:
+ semaphore.release()
+
+
+# ── Wall-clock budget: give up before the proxy's stage timeout ───────
+
+
+def test_time_budget_bails_to_passthrough(monkeypatch):
+ monkeypatch.setenv(KOMPRESS_TIME_BUDGET_ENV, "0.2")
+ model = FakeModel(delay=0.15)
+ compressor = _make_compressor(monkeypatch, model, chunk_words=10)
+
+ result = compressor.compress(CONTENT_40_WORDS) # 4 chunks at ~0.15s each
+
+ assert result.compressed == CONTENT_40_WORDS
+ assert model.calls < 4 # bailed before processing every chunk
+
+
+def test_time_budget_disabled_processes_all_chunks(monkeypatch):
+ monkeypatch.setenv(KOMPRESS_TIME_BUDGET_ENV, "0")
+ model = FakeModel(delay=0.01)
+ compressor = _make_compressor(monkeypatch, model, chunk_words=10)
+
+ result = compressor.compress(CONTENT_40_WORDS)
+
+ assert model.calls == 4
+ assert result.compression_ratio == 0.5
+
+
+def test_time_budget_batch_keeps_completed_texts(monkeypatch):
+ """Mid-queue bail: fully processed texts stay compressed; any text with
+ an unprocessed chunk passes through whole (never partially dropped)."""
+ monkeypatch.setenv(KOMPRESS_TIME_BUDGET_ENV, "0.2")
+ model = FakeModel(delay=0.25) # one batch alone exhausts the budget
+ compressor = _make_compressor(monkeypatch, model)
+ monkeypatch.setattr(KompressCompressor, "_should_use_sequential_fallback", lambda self: False)
+
+ contents = [
+ " ".join(f"a{i}" for i in range(20)),
+ " ".join(f"b{i}" for i in range(20)),
+ " ".join(f"c{i}" for i in range(20)),
+ ]
+ results = compressor.compress_batch(contents, batch_size=1)
+
+ assert len(results) == 3
+ # First batch ran; later ones bailed to passthrough.
+ assert results[0].compression_ratio == 0.5
+ assert results[1].compressed == contents[1]
+ assert results[2].compressed == contents[2]
+ # Every result preserves all information (compressed or original).
+ for r in results:
+ assert r.compressed
+
+
+# ── Preload canary: detect degraded runtimes before live traffic ──────
+
+
+def _join_canary(compressor: KompressCompressor) -> None:
+ assert compressor._canary_thread is not None
+ compressor._canary_thread.join(timeout=10)
+ assert not compressor._canary_thread.is_alive()
+
+
+def test_canary_disables_kompress_on_slow_inference(monkeypatch, caplog):
+ monkeypatch.setenv(KOMPRESS_CANARY_THRESHOLD_ENV, "0.05")
+ model = FakeModel(delay=0.15)
+ compressor = _make_compressor(monkeypatch, model)
+
+ with caplog.at_level("WARNING"):
+ backend = compressor.preload()
+ _join_canary(compressor)
+
+ assert backend == "onnx"
+ assert compressor._degraded_reason is not None
+ assert model.calls == 2 # probe + one retry
+ assert "DISABLED" in caplog.text
+
+ result = compressor.compress(CONTENT_40_WORDS)
+ assert result.compressed == CONTENT_40_WORDS
+ assert model.calls == 2 # model never touched again
+
+ batch = compressor.compress_batch([CONTENT_40_WORDS])
+ assert batch[0].compressed == CONTENT_40_WORDS
+ assert model.calls == 2
+
+
+def test_canary_fast_inference_stays_enabled(monkeypatch):
+ monkeypatch.setenv(KOMPRESS_CANARY_THRESHOLD_ENV, "5")
+ model = FakeModel()
+ compressor = _make_compressor(monkeypatch, model)
+
+ compressor.preload()
+ _join_canary(compressor)
+
+ assert compressor._degraded_reason is None
+ result = compressor.compress(CONTENT_40_WORDS)
+ assert result.compression_ratio == 0.5
+
+
+def test_canary_retry_forgives_oneoff_warmup_slowness(monkeypatch):
+ """First inference pays one-off warmup costs; only a slow retry condemns."""
+ monkeypatch.setenv(KOMPRESS_CANARY_THRESHOLD_ENV, "0.1")
+
+ class WarmupModel(FakeModel):
+ def get_keep_mask(self, input_ids, attention_mask):
+ self.calls += 1
+ if self.calls == 1:
+ time.sleep(0.2) # cold first run
+ return [[i % 2 == 0 for i in range(len(row))] for row in input_ids]
+
+ model = WarmupModel()
+ compressor = _make_compressor(monkeypatch, model)
+ compressor.preload()
+ _join_canary(compressor)
+
+ assert compressor._degraded_reason is None
+ assert model.calls == 2
+
+
+def test_canary_disabled_via_env(monkeypatch):
+ monkeypatch.setenv(KOMPRESS_CANARY_THRESHOLD_ENV, "0")
+ model = FakeModel(delay=0.2)
+ compressor = _make_compressor(monkeypatch, model)
+
+ compressor.preload()
+
+ assert compressor._canary_thread is None # probe never scheduled
+ assert model.calls == 0
+ assert compressor._degraded_reason is None
+
+
+def test_preload_does_not_block_on_slow_canary(monkeypatch):
+ """The probe runs off the startup path: preload blocks proxy boot (the
+ HTTP server binds after it), and a slow probe once pushed the wrap-e2e
+ container past its 30s health-check timeout."""
+ monkeypatch.setenv(KOMPRESS_CANARY_THRESHOLD_ENV, "0.05")
+ model = FakeModel(delay=1.0)
+ compressor = _make_compressor(monkeypatch, model)
+
+ started = time.monotonic()
+ compressor.preload()
+ preload_elapsed = time.monotonic() - started
+
+ assert preload_elapsed < 0.5 # returns before the ~2s of probe inference
+ _join_canary(compressor)
+ assert compressor._degraded_reason is not None
+
+
+def test_canary_probe_error_never_breaks_preload(monkeypatch):
+ class ExplodingModel(FakeModel):
+ def get_keep_mask(self, input_ids, attention_mask):
+ raise RuntimeError("probe boom")
+
+ compressor = _make_compressor(monkeypatch, ExplodingModel())
+
+ assert compressor.preload() == "onnx"
+ _join_canary(compressor)
+ assert compressor._degraded_reason is None
diff --git a/tests/test_onnx_runtime.py b/tests/test_onnx_runtime.py
index 0f0c192cf..076afd2eb 100644
--- a/tests/test_onnx_runtime.py
+++ b/tests/test_onnx_runtime.py
@@ -1,4 +1,10 @@
-from headroom.onnx_runtime import create_cpu_session_options
+import sys
+
+from headroom.onnx_runtime import (
+ ONNX_CPU_ARENA_ENV,
+ cpu_arena_enabled,
+ create_cpu_session_options,
+)
class _FakeSessionOptions:
@@ -23,7 +29,11 @@ class _FakeOrtWithoutToggles:
SessionOptions = _FakeSessionOptionsWithoutToggles
-def test_create_cpu_session_options_disables_retention_features():
+def test_create_cpu_session_options_disables_retention_features(monkeypatch):
+ """Non-Windows keeps the legacy low-RSS behavior: arena + mem pattern off."""
+ monkeypatch.delenv(ONNX_CPU_ARENA_ENV, raising=False)
+ monkeypatch.setattr(sys, "platform", "linux")
+
options = create_cpu_session_options(
_FakeOrt,
intra_op_num_threads=1,
@@ -36,7 +46,60 @@ def test_create_cpu_session_options_disables_retention_features():
assert options.enable_mem_pattern is False
-def test_create_cpu_session_options_handles_older_session_options():
+def test_create_cpu_session_options_darwin_unchanged(monkeypatch):
+ monkeypatch.delenv(ONNX_CPU_ARENA_ENV, raising=False)
+ monkeypatch.setattr(sys, "platform", "darwin")
+
+ options = create_cpu_session_options(_FakeOrt)
+
+ assert options.enable_cpu_mem_arena is False
+ assert options.enable_mem_pattern is False
+
+
+def test_create_cpu_session_options_keeps_arena_on_windows(monkeypatch):
+ """Disabling the arena on Windows degrades inference by orders of
+ magnitude (onnxruntime#11627) — ORT defaults must stay untouched there."""
+ monkeypatch.delenv(ONNX_CPU_ARENA_ENV, raising=False)
+ monkeypatch.setattr(sys, "platform", "win32")
+
+ options = create_cpu_session_options(_FakeOrt, intra_op_num_threads=3)
+
+ assert options.enable_cpu_mem_arena is True
+ assert options.enable_mem_pattern is True
+ assert options.intra_op_num_threads == 3
+
+
+def test_arena_env_override_forces_on(monkeypatch):
+ monkeypatch.setattr(sys, "platform", "linux")
+ monkeypatch.setenv(ONNX_CPU_ARENA_ENV, "1")
+
+ assert cpu_arena_enabled() is True
+ options = create_cpu_session_options(_FakeOrt)
+ assert options.enable_cpu_mem_arena is True
+
+
+def test_arena_env_override_forces_off(monkeypatch):
+ monkeypatch.setattr(sys, "platform", "win32")
+ monkeypatch.setenv(ONNX_CPU_ARENA_ENV, "0")
+
+ assert cpu_arena_enabled() is False
+ options = create_cpu_session_options(_FakeOrt)
+ assert options.enable_cpu_mem_arena is False
+
+
+def test_arena_env_invalid_falls_back_to_platform_default(monkeypatch):
+ monkeypatch.setenv(ONNX_CPU_ARENA_ENV, "bananas")
+
+ monkeypatch.setattr(sys, "platform", "win32")
+ assert cpu_arena_enabled() is True
+ monkeypatch.setattr(sys, "platform", "linux")
+ assert cpu_arena_enabled() is False
+
+
+def test_create_cpu_session_options_handles_older_session_options(monkeypatch):
+ monkeypatch.delenv(ONNX_CPU_ARENA_ENV, raising=False)
+ monkeypatch.setattr(sys, "platform", "linux")
+
options = create_cpu_session_options(_FakeOrtWithoutToggles)
assert options.intra_op_num_threads is None
diff --git a/tests/test_subprocess_encoding.py b/tests/test_subprocess_encoding.py
new file mode 100644
index 000000000..0e84df980
--- /dev/null
+++ b/tests/test_subprocess_encoding.py
@@ -0,0 +1,55 @@
+"""Proxy-runtime subprocess calls must decode UTF-8 explicitly.
+
+On Windows, ``subprocess.run(text=True)`` without an ``encoding`` decodes
+child output with the console code page (cp1252). rtk's emoji-laden output
+then kills the reader thread with ``UnicodeDecodeError: 'charmap' codec
+can't decode byte ...`` (seen in user proxy logs). These tests pin the
+``encoding="utf-8"`` kwarg on every proxy-runtime subprocess call.
+"""
+
+import subprocess
+
+import headroom.lean_ctx
+import headroom.proxy.helpers as helpers
+import headroom.rtk
+from headroom.proxy.interceptors import astgrep
+
+
+def _capture_run(captured, returncode=0, stdout='{"summary": {}}'):
+ def fake_run(cmd, **kwargs):
+ captured.update(kwargs)
+ return subprocess.CompletedProcess(cmd, returncode, stdout=stdout, stderr="")
+
+ return fake_run
+
+
+def test_rtk_stats_subprocess_uses_utf8(monkeypatch):
+ captured: dict = {}
+ monkeypatch.setattr(helpers, "run", _capture_run(captured))
+ monkeypatch.setattr(headroom.rtk, "get_rtk_path", lambda: "/fake/rtk")
+
+ helpers._read_rtk_lifetime_stats()
+
+ assert captured["encoding"] == "utf-8"
+ assert captured["errors"] == "replace"
+
+
+def test_lean_ctx_stats_subprocess_uses_utf8(monkeypatch):
+ captured: dict = {}
+ monkeypatch.setattr(helpers, "run", _capture_run(captured))
+ monkeypatch.setattr(headroom.lean_ctx, "get_lean_ctx_path", lambda: "/fake/lean-ctx")
+
+ helpers._read_lean_ctx_lifetime_stats()
+
+ assert captured["encoding"] == "utf-8"
+ assert captured["errors"] == "replace"
+
+
+def test_ast_grep_subprocess_uses_utf8(monkeypatch):
+ captured: dict = {}
+ monkeypatch.setattr(astgrep.subprocess, "run", _capture_run(captured, returncode=1, stdout=""))
+
+ astgrep._run_ast_grep("/fake/sg", "python", "def foo():\n pass\n")
+
+ assert captured["encoding"] == "utf-8"
+ assert captured["errors"] == "replace"