Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(kompress): time each model-load stage and stop false not-ready warnings during warm-up (#4195)
Kenneth Wong89b229ce5fea
diff --git a/headroom/proxy/server.py b/headroom/proxy/server.pyindex fa111f9f9..231deefcd 100644--- a/headroom/proxy/server.py+++ b/headroom/proxy/server.py@@ -2038,7 +2038,11 @@ class HeadroomProxy(backend,)- threading.Thread(target=_warm, name="kompress-warmup", daemon=True).start()+ from headroom.transforms.kompress_compressor import start_kompress_warmup_thread++ # Registered as the model's loader, so a request during the warm-up+ # doesn't start a second load thread.+ start_kompress_warmup_thread(compressor.config.model_id, _warm)return Trueasync def startup(self):diff --git a/headroom/transforms/content_router.py b/headroom/transforms/content_router.pyindex f73d269ea..4e0de0fc0 100644--- a/headroom/transforms/content_router.py+++ b/headroom/transforms/content_router.py@@ -2742,6 +2742,21 @@ class ContentRouter(Transform):except Exception:return True+ @staticmethod+ def _kompress_load_in_progress(compressor: Any) -> bool:+ """Whether a background thread is still fetching or loading ``compressor``'s model.++ Compressors without ``is_loading`` (the remote one is always ready,+ test doubles) count as not loading.+ """+ is_loading = getattr(compressor, "is_loading", None)+ if is_loading is None:+ return False+ try:+ return bool(is_loading())+ except Exception:+ return False+def _record_to_toin(self,strategy: CompressionStrategy,@@ -4725,10 +4740,22 @@ class ContentRouter(Transform):if compressor:if not compressor.is_ready():compressor.ensure_background_load()- # Surface: warn once per ContentRouter instance so operators- # know compression is degraded — model not cached, or- # HuggingFace unreachable (corporate firewall, SSL, etc.).- if not getattr(self, "_kompress_warned", False):+ # Surface once per ContentRouter instance that requests pass+ # through uncompressed. While a thread is still fetching or+ # loading the model (the post-startup warm-up, or the+ # download just started above) that is expected, so say it+ # at INFO. With no loader running, e.g. a failed download+ # in its retry backoff, compression is degraded: warn and+ # name the usual causes (_background_download has already+ # logged the actual error).+ if self._kompress_load_in_progress(compressor):+ if not getattr(self, "_kompress_loading_noted", False):+ logger.info(+ "Kompress model still loading in the background; "+ "requests pass through uncompressed until it is ready."+ )+ self._kompress_loading_noted = True+ elif not getattr(self, "_kompress_warned", False) and not compressor.is_ready():logger.warning("Kompress model not ready; requests will not be ""compressed. Check HuggingFace connectivity or "diff --git a/headroom/transforms/kompress_compressor.py b/headroom/transforms/kompress_compressor.pyindex 5897dfded..e5afcd125 100644--- a/headroom/transforms/kompress_compressor.py+++ b/headroom/transforms/kompress_compressor.py@@ -23,6 +23,7 @@ import osimport reimport threadingimport time+from collections.abc import Callable, Iteratorfrom dataclasses import dataclassfrom functools import lru_cachefrom typing import Any, Literal@@ -797,8 +798,46 @@ def _smoke_run(session: Any) -> None:)+#: Per-stage load cost: stage name -> (wall-clock ms, loading-thread CPU ms).+StageTimings = dict[str, tuple[float, float]]+++def _timed_stage(stage_ms: StageTimings | None, stage: str) -> Iterator[None]:+ """Add the block's wall-clock and thread-CPU milliseconds to ``stage_ms[stage]``.++ Accumulates rather than overwrites, so a stage that runs once per ONNX+ candidate reports the cost of every attempt, failed ones included. Records+ on exceptions too. A no-op when ``stage_ms`` is ``None``.++ CPU is this thread's only (``time.thread_time``): work on onnxruntime's own+ worker threads is not counted, and wall far above CPU means the thread was+ waiting (disk, the GIL, a lock) rather than working.+ """+ if stage_ms is None:+ yield+ return+ wall_started = time.perf_counter()+ cpu_started = time.thread_time()+ try:+ yield+ finally:+ wall_ms = (time.perf_counter() - wall_started) * 1000.0+ cpu_ms = (time.thread_time() - cpu_started) * 1000.0+ prev_wall, prev_cpu = stage_ms.get(stage, (0.0, 0.0))+ stage_ms[stage] = (prev_wall + wall_ms, prev_cpu + cpu_ms)+++def _format_stage_ms(stage_ms: StageTimings) -> str:+ return " ".join(f"{stage}={wall:.0f}/{cpu:.0f}" for stage, (wall, cpu) in stage_ms.items())++def _create_onnx_session(- model_id: str, providers: list[Any], *, allow_download: bool = True+ model_id: str,+ providers: list[Any],+ *,+ allow_download: bool = True,+ stage_ms: StageTimings | None = None,) -> Any:"""Resolve and load the model's ONNX artifact, trying candidates in order.@@ -812,6 +851,10 @@ def _create_onnx_session(cache only; if none is cached, :class:`KompressModelNotCached` is raisedinstead of hitting the network. ``onnxruntime`` is imported only after acandidate resolves, so a cache-only miss never requires it.++ ``stage_ms``, when given, accumulates the cost of ``artifact_resolve``,+ ``onnxruntime_import``, ``session_build`` and ``smoke_run`` across every+ candidate tried. See :func:`_timed_stage`."""last_err: Exception | None = Nonecache_miss = False@@ -819,9 +862,10 @@ def _create_onnx_session(ort: Any = Nonefor filename in _onnx_filename_candidates():try:- onnx_path = _hf_artifact(- model_id, filename, allow_network=allow_download and refused is None- )+ with _timed_stage(stage_ms, "artifact_resolve"):+ onnx_path = _hf_artifact(+ model_id, filename, allow_network=allow_download and refused is None+ )except KompressModelNotCached as exc:# Only _hf_artifact's air-gap translation raises this here. It says# THIS candidate is not cached; a later one may be. Keep looking,@@ -834,16 +878,19 @@ def _create_onnx_session(logger.debug("ONNX artifact %r unavailable for %s: %s", filename, model_id, exc)continueif ort is None:- import onnxruntime+ with _timed_stage(stage_ms, "onnxruntime_import"):+ import onnxruntimeort = onnxruntimetry:- session = ort.InferenceSession(- onnx_path,- _onnx_session_options(ort),- providers=providers,- )- _smoke_run(session)+ with _timed_stage(stage_ms, "session_build"):+ session = ort.InferenceSession(+ onnx_path,+ _onnx_session_options(ort),+ providers=providers,+ )+ with _timed_stage(stage_ms, "smoke_run"):+ _smoke_run(session)return sessionexcept Exception as exc:last_err = exc@@ -878,7 +925,12 @@ def _load_kompress_onnx(if model_id in _kompress_cache:return _kompress_cache[model_id]- logger.info("Downloading Kompress ONNX model from %s ...", model_id)+ # Logged before any cache lookup: the artifacts usually come from disk+ # (startup prefetch), and _hf_artifact goes to the Hub only on a miss.+ logger.info("Loading Kompress ONNX model from %s ...", model_id)+ load_started = time.perf_counter()+ load_cpu_started = time.thread_time()+ stage_ms: StageTimings = {}backend = "onnx_coreml" if use_coreml else "onnx"providers: list[Any]@@ -907,15 +959,26 @@ def _load_kompress_onnx(else:providers = ["CPUExecutionProvider"]- session = _create_onnx_session(model_id, providers, allow_download=allow_download)+ session = _create_onnx_session(+ model_id, providers, allow_download=allow_download, stage_ms=stage_ms+ )model = _OnnxModel(session)- from transformers import AutoTokenizer+ with _timed_stage(stage_ms, "transformers_import"):+ from transformers import AutoTokenizer- tokenizer = _load_modernbert_tokenizer(AutoTokenizer, allow_download=allow_download)+ with _timed_stage(stage_ms, "tokenizer_load"):+ tokenizer = _load_modernbert_tokenizer(AutoTokenizer, allow_download=allow_download)_kompress_cache[model_id] = (model, tokenizer, backend)- logger.info("Kompress ONNX loaded: %s backend=%s", model_id, backend)+ logger.info(+ "Kompress ONNX loaded: %s backend=%s total_ms=%.0f/%.0f stages_ms(wall/cpu): %s",+ model_id,+ backend,+ (time.perf_counter() - load_started) * 1000.0,+ (time.thread_time() - load_cpu_started) * 1000.0,+ _format_stage_ms(stage_ms),+ )return model, tokenizer, backend@@ -1341,6 +1404,48 @@ def ensure_background_download(model_id: str = HF_MODEL_ID, device: str = "auto"thread.start()+def is_kompress_load_in_progress(model_id: str = HF_MODEL_ID) -> bool:+ """True while a registered thread (prefetch, download or warm-up) is working on ``model_id``."""+ with _download_threads_lock:+ thread = _download_threads.get(model_id)+ return thread is not None and thread.is_alive()+++def start_kompress_warmup_thread(model_id: str, warm: Callable[[], None]) -> threading.Thread:+ """Run ``warm`` on a daemon ``kompress-warmup`` thread registered as ``model_id``'s loader.++ The proxy's post-startup warm-up loads the model on its own thread. Unless+ that thread is in the per-model registry, a request arriving mid-warm-up+ finds the model not ready, sees no live loader, and has+ :func:`ensure_background_download` start a second thread that only queues+ on ``_kompress_lock`` and then returns the warm-up's cache entry. Registered+ before it starts, so the warm-up's initial delay is covered too.++ If a prefetch or download thread is already registered, the warm-up takes+ its place in the registry and waits for it to finish before running+ ``warm``. Without the wait, a warm-up that failed while a startup prefetch+ was still downloading would leave the registry pointing at a dead thread,+ and the next request would start another download next to the hidden+ prefetch. Waiting also keeps the warm-up from fetching the same files at+ the same time. Once the warm-up finishes, a failed one included, the next+ :func:`ensure_background_download` call replaces the dead entry as usual.+ """+ with _download_threads_lock:+ previous = _download_threads.get(model_id)++ def _run() -> None:+ # Registry entries are always started threads, so join() is safe;+ # it returns at once for one that has already finished.+ if previous is not None:+ previous.join()+ warm()++ thread = threading.Thread(target=_run, name="kompress-warmup", daemon=True)+ _download_threads[model_id] = thread+ thread.start()+ return thread++def prefetch_kompress_artifacts(model_id: str = HF_MODEL_ID) -> bool:"""Download the model's ONNX artifact to the local cache. No native init.@@ -1751,6 +1856,10 @@ class KompressCompressor(Transform):"""ensure_background_download(self.config.model_id, self.config.device)+ def is_loading(self) -> bool:+ """True while a background thread is fetching or loading this model."""+ return is_kompress_load_in_progress(self.config.model_id)+def compress(self,content: str,diff --git a/tests/test_content_router_kompress_not_ready_log.py b/tests/test_content_router_kompress_not_ready_log.pynew file mode 100644index 000000000..34913e728--- /dev/null+++ b/tests/test_content_router_kompress_not_ready_log.py@@ -0,0 +1,126 @@+"""A Kompress model that is still loading is expected, not a connectivity problem.++The router used to log "Kompress model not ready ... Check HuggingFace+connectivity" at WARNING on the first request after every proxy start, because+the post-startup warm-up takes ~20 s and requests arrive meanwhile. It now says+"still loading" at INFO while a loader thread runs, and keeps the WARNING for+when nothing is loading the model (e.g. a failed download in its retry backoff).+"""++from __future__ import annotations++import logging+import threading++import pytest++from headroom.transforms import kompress_compressor as kc+from headroom.transforms.content_router import ContentRouter, ContentRouterConfig++_LOGGER = "headroom.transforms.content_router"+_CONTENT = "some plain text that the ML model would compress. " * 4+++class _NotReadyKompress:+ def __init__(self, *, loading: bool) -> None:+ self.loading = loading+ self.background_loads = 0++ def is_ready(self) -> bool:+ return False++ def ensure_background_load(self) -> None:+ self.background_loads += 1++ def is_loading(self) -> bool:+ return self.loading+++def _router(monkeypatch: pytest.MonkeyPatch, compressor: object) -> ContentRouter:+ router = ContentRouter(+ ContentRouterConfig(+ relevance_split=False,+ lossless_then_lossy=False,+ ccr_inject_marker=False,+ )+ )+ monkeypatch.setattr(router, "_get_kompress", lambda: compressor)+ return router+++def _logged(caplog: pytest.LogCaptureFixture, level: int, text: str) -> int:+ return sum(+ 1+ for r in caplog.records+ if r.name == _LOGGER and r.levelno == level and text in r.getMessage()+ )+++def test_still_loading_is_info_not_a_connectivity_warning(monkeypatch, caplog):+ compressor = _NotReadyKompress(loading=True)+ router = _router(monkeypatch, compressor)++ with caplog.at_level(logging.INFO, logger=_LOGGER):+ for _ in range(3):+ assert router._try_ml_compressor(_CONTENT, "ctx")[0] == _CONTENT++ assert _logged(caplog, logging.INFO, "still loading in the background") == 1+ assert _logged(caplog, logging.WARNING, "Check HuggingFace connectivity") == 0+ assert compressor.background_loads == 3+++def test_no_loader_running_still_warns_once(monkeypatch, caplog):+ router = _router(monkeypatch, _NotReadyKompress(loading=False))++ with caplog.at_level(logging.INFO, logger=_LOGGER):+ for _ in range(3):+ router._try_ml_compressor(_CONTENT, "ctx")++ assert _logged(caplog, logging.WARNING, "Check HuggingFace connectivity") == 1+ assert _logged(caplog, logging.INFO, "still loading in the background") == 0+++def test_warns_once_the_load_ends_without_a_model(monkeypatch, caplog):+ """A warm-up that fails leaves no loader running: then it is degraded."""+ compressor = _NotReadyKompress(loading=True)+ router = _router(monkeypatch, compressor)++ with caplog.at_level(logging.INFO, logger=_LOGGER):+ router._try_ml_compressor(_CONTENT, "ctx")+ compressor.loading = False+ router._try_ml_compressor(_CONTENT, "ctx")++ assert _logged(caplog, logging.INFO, "still loading in the background") == 1+ assert _logged(caplog, logging.WARNING, "Check HuggingFace connectivity") == 1+++def test_compressor_without_is_loading_keeps_the_warning(monkeypatch, caplog):+ class _Legacy:+ def is_ready(self) -> bool:+ return False++ def ensure_background_load(self) -> None:+ pass++ router = _router(monkeypatch, _Legacy())++ with caplog.at_level(logging.INFO, logger=_LOGGER):+ router._try_ml_compressor(_CONTENT, "ctx")++ assert _logged(caplog, logging.WARNING, "Check HuggingFace connectivity") == 1+++def test_registered_warm_up_thread_counts_as_loading():+ """The real signal: the warm-up thread is the model's registered loader."""+ model_id = "test/not-ready-log-model"+ release = threading.Event()+ try:+ thread = kc.start_kompress_warmup_thread(model_id, lambda: release.wait(5))+ assert kc.is_kompress_load_in_progress(model_id)+ release.set()+ thread.join(timeout=5)+ assert not kc.is_kompress_load_in_progress(model_id)+ finally:+ release.set()+ with kc._download_threads_lock:+ kc._download_threads.pop(model_id, None)diff --git a/tests/test_kompress_download_backoff.py b/tests/test_kompress_download_backoff.pyindex ff83a2089..b6ce3bafa 100644--- a/tests/test_kompress_download_backoff.py+++ b/tests/test_kompress_download_backoff.py@@ -154,3 +154,30 @@ def test_a_live_thread_is_never_duplicated(monkeypatch):finally:gate.set()_drain()+++def test_warm_up_waits_for_a_live_loader_it_replaces(monkeypatch):+ """A startup prefetch still downloading stays covered, even if the warm-up fails."""+ started = _spawned(monkeypatch, fails=False)+ prefetch_done = threading.Event()+ prefetch = threading.Thread(target=prefetch_done.wait, args=(10,), daemon=True)+ with kc._download_threads_lock:+ prefetch.start()+ kc._download_threads["some/model"] = prefetch++ # A warm-up that returns without caching the model stands in for a failed one.+ warmed = threading.Event()+ warm_up = kc.start_kompress_warmup_thread("some/model", warmed.set)+ try:+ kc.ensure_background_download("some/model")+ assert not warmed.wait(0.2), "warm-up ran while the prefetch was still live"+ assert started == [], "a request started a download beside the live prefetch"+ finally:+ prefetch_done.set()+ warm_up.join(timeout=10)+ assert warmed.is_set()++ # Both are done and the model is still not cached: a request may load it now.+ kc.ensure_background_download("some/model")+ _drain()+ assert started == ["some/model"]diff --git a/tests/test_kompress_failsafe.py b/tests/test_kompress_failsafe.pyindex 26e4c7d1b..973ab7bfc 100644--- a/tests/test_kompress_failsafe.py+++ b/tests/test_kompress_failsafe.py@@ -609,6 +609,71 @@ def test_all_artifacts_failing_at_run_raises_rather_than_returning_a_dead_sessiokc._create_onnx_session("org/model", ["CPUExecutionProvider"])+# ── Load-stage timings: say where a slow warm-up spent its time ────────────────+++def _install_ticking_clock(monkeypatch):+ """Each clock reading is 1s after the last, so every timed stage costs 1000ms."""++ def ticking_clock():+ now = [0.0]++ def tick() -> float:+ now[0] += 1.0+ return now[0]++ return staticmethod(tick)++ clock = type("_Clock", (), {"perf_counter": ticking_clock(), "thread_time": ticking_clock()})+ monkeypatch.setattr(kc, "time", clock)+++def test_stage_timings_count_every_candidate_attempt(monkeypatch):+ """A candidate rejected at smoke-run still costs time, so it shows in the totals."""+ _install_fake_ort(monkeypatch, run_fails_for={"int8-wo"})+ _install_ticking_clock(monkeypatch)+ stage_ms: kc.StageTimings = {}++ kc._create_onnx_session("org/model", ["CPUExecutionProvider"], stage_ms=stage_ms)++ # int8-wo resolved, built and smoke-run (rejected); fp32 the same (accepted).+ # onnxruntime is imported once, on the first resolved candidate.+ assert stage_ms == {+ "artifact_resolve": (2000.0, 2000.0),+ "onnxruntime_import": (1000.0, 1000.0),+ "session_build": (2000.0, 2000.0),+ "smoke_run": (2000.0, 2000.0),+ }+++def test_onnx_load_logs_every_stage(monkeypatch, caplog):+ import sys+ from types import SimpleNamespace++ _install_fake_ort(monkeypatch, run_fails_for=set())+ monkeypatch.setitem(sys.modules, "transformers", SimpleNamespace(AutoTokenizer=object()))+ monkeypatch.setattr(kc, "_kompress_cache", {})+ monkeypatch.setattr(+ kc, "_load_modernbert_tokenizer", lambda _auto, *, allow_download: "tokenizer"+ )++ with caplog.at_level("INFO", logger=kc.logger.name):+ kc._load_kompress_onnx("org/model")++ loaded = [r.getMessage() for r in caplog.records if "Kompress ONNX loaded" in r.getMessage()]+ assert len(loaded) == 1+ for stage in (+ "artifact_resolve=",+ "onnxruntime_import=",+ "session_build=",+ "smoke_run=",+ "transformers_import=",+ "tokenizer_load=",+ ):+ assert stage in loaded[0]+ assert "Downloading" not in caplog.text++# ── Failure latch: a broken model stops costing us every request ───────────────diff --git a/tests/test_proxy/test_kompress_background_warmup.py b/tests/test_proxy/test_kompress_background_warmup.pyindex 69d68da0b..d03e61372 100644--- a/tests/test_proxy/test_kompress_background_warmup.py+++ b/tests/test_proxy/test_kompress_background_warmup.py@@ -14,10 +14,14 @@ from types import SimpleNamespaceimport pytestfrom headroom.proxy.server import HeadroomProxy+from headroom.transforms import kompress_compressor as kc++_MODEL_ID = "test/warmup-model"class _Kompress:def __init__(self) -> None:+ self.config = SimpleNamespace(model_id=_MODEL_ID)self.preloaded = threading.Event()def preload(self, *, allow_download: bool = True) -> str:@@ -37,6 +41,16 @@ def _no_delay(monkeypatch: pytest.MonkeyPatch) -> None:monkeypatch.delenv("HEADROOM_KOMPRESS_WARMUP", raising=False)[email protected](autouse=True)+def _clean_registry():+ yield+ with kc._download_threads_lock:+ thread = kc._download_threads.pop(_MODEL_ID, None)+ kc._download_failures.pop(_MODEL_ID, None)+ if thread is not None:+ thread.join(timeout=5)++def test_warms_on_a_background_thread(monkeypatch: pytest.MonkeyPatch) -> None:monkeypatch.setattr("platform.libc_ver", lambda: ("glibc", "2.35"))monkeypatch.delenv("PYTEST_CURRENT_TEST", raising=False)@@ -74,3 +88,41 @@ def test_test_processes_do_not_warm_unless_forced(monkeypatch: pytest.MonkeyPatcmonkeypatch.setattr("platform.libc_ver", lambda: ("glibc", "2.35"))monkeypatch.setenv("PYTEST_CURRENT_TEST", "x")assert HeadroomProxy._start_kompress_background_warmup(_proxy_with(_Kompress())) is False+++def test_request_during_warm_up_does_not_start_a_second_loader(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """The warm-up is the model's registered loader until it finishes."""+ monkeypatch.setattr("platform.libc_ver", lambda: ("glibc", "2.35"))+ monkeypatch.delenv("PYTEST_CURRENT_TEST", raising=False)+ monkeypatch.setattr(kc, "_kompress_cache", {})+ loads: list[str] = []+ monkeypatch.setattr(+ kc,+ "_load_kompress",+ lambda model_id, device, allow_download=True: (+ loads.append(model_id) or (None, None, "onnx")+ ),+ )+ release = threading.Event()++ class _SlowKompress(_Kompress):+ def preload(self, *, allow_download: bool = True) -> str:+ self.preloaded.set()+ release.wait(5)+ raise OSError("warm-up failed")++ kompress = _SlowKompress()+ assert HeadroomProxy._start_kompress_background_warmup(_proxy_with(kompress)) is True+ assert kompress.preloaded.wait(5)++ kc.ensure_background_download(_MODEL_ID)+ assert loads == []++ # Once the warm-up is over (here, failed), a request still starts its own load.+ release.set()+ kc._download_threads[_MODEL_ID].join(timeout=5)+ kc.ensure_background_download(_MODEL_ID)+ kc._download_threads[_MODEL_ID].join(timeout=5)+ assert loads == [_MODEL_ID]