Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(kompress): time each model-load stage and stop false not-ready warnings during warm-up (#4195)

Kenneth Wong89b229ce5fea
diff --git a/headroom/proxy/server.py b/headroom/proxy/server.py
index fa111f9f9..231deefcd 100644
--- a/headroom/proxy/server.py
+++ b/headroom/proxy/server.py
@@ -2038,7 +2038,11 @@ class HeadroomProxy(
backend,
)
- threading.Thread(target=_warm, name="kompress-warmup", daemon=True).start()
+ from headroom.transforms.kompress_compressor import start_kompress_warmup_thread
+
+ # Registered as the model's loader, so a request during the warm-up
+ # doesn't start a second load thread.
+ start_kompress_warmup_thread(compressor.config.model_id, _warm)
return True
async def startup(self):
diff --git a/headroom/transforms/content_router.py b/headroom/transforms/content_router.py
index f73d269ea..4e0de0fc0 100644
--- a/headroom/transforms/content_router.py
+++ b/headroom/transforms/content_router.py
@@ -2742,6 +2742,21 @@ class ContentRouter(Transform):
except Exception:
return True
+ @staticmethod
+ def _kompress_load_in_progress(compressor: Any) -> bool:
+ """Whether a background thread is still fetching or loading ``compressor``'s model.
+
+ Compressors without ``is_loading`` (the remote one is always ready,
+ test doubles) count as not loading.
+ """
+ is_loading = getattr(compressor, "is_loading", None)
+ if is_loading is None:
+ return False
+ try:
+ return bool(is_loading())
+ except Exception:
+ return False
+
def _record_to_toin(
self,
strategy: CompressionStrategy,
@@ -4725,10 +4740,22 @@ class ContentRouter(Transform):
if compressor:
if not compressor.is_ready():
compressor.ensure_background_load()
- # Surface: warn once per ContentRouter instance so operators
- # know compression is degraded — model not cached, or
- # HuggingFace unreachable (corporate firewall, SSL, etc.).
- if not getattr(self, "_kompress_warned", False):
+ # Surface once per ContentRouter instance that requests pass
+ # through uncompressed. While a thread is still fetching or
+ # loading the model (the post-startup warm-up, or the
+ # download just started above) that is expected, so say it
+ # at INFO. With no loader running, e.g. a failed download
+ # in its retry backoff, compression is degraded: warn and
+ # name the usual causes (_background_download has already
+ # logged the actual error).
+ if self._kompress_load_in_progress(compressor):
+ if not getattr(self, "_kompress_loading_noted", False):
+ logger.info(
+ "Kompress model still loading in the background; "
+ "requests pass through uncompressed until it is ready."
+ )
+ self._kompress_loading_noted = True
+ elif not getattr(self, "_kompress_warned", False) and not compressor.is_ready():
logger.warning(
"Kompress model not ready; requests will not be "
"compressed. Check HuggingFace connectivity or "
diff --git a/headroom/transforms/kompress_compressor.py b/headroom/transforms/kompress_compressor.py
index 5897dfded..e5afcd125 100644
--- a/headroom/transforms/kompress_compressor.py
+++ b/headroom/transforms/kompress_compressor.py
@@ -23,6 +23,7 @@ import os
import re
import threading
import time
+from collections.abc import Callable, Iterator
from dataclasses import dataclass
from functools import lru_cache
from typing import Any, Literal
@@ -797,8 +798,46 @@ def _smoke_run(session: Any) -> None:
)
+#: Per-stage load cost: stage name -> (wall-clock ms, loading-thread CPU ms).
+StageTimings = dict[str, tuple[float, float]]
+
+
+def _timed_stage(stage_ms: StageTimings | None, stage: str) -> Iterator[None]:
+ """Add the block's wall-clock and thread-CPU milliseconds to ``stage_ms[stage]``.
+
+ Accumulates rather than overwrites, so a stage that runs once per ONNX
+ candidate reports the cost of every attempt, failed ones included. Records
+ on exceptions too. A no-op when ``stage_ms`` is ``None``.
+
+ CPU is this thread's only (``time.thread_time``): work on onnxruntime's own
+ worker threads is not counted, and wall far above CPU means the thread was
+ waiting (disk, the GIL, a lock) rather than working.
+ """
+ if stage_ms is None:
+ yield
+ return
+ wall_started = time.perf_counter()
+ cpu_started = time.thread_time()
+ try:
+ yield
+ finally:
+ wall_ms = (time.perf_counter() - wall_started) * 1000.0
+ cpu_ms = (time.thread_time() - cpu_started) * 1000.0
+ prev_wall, prev_cpu = stage_ms.get(stage, (0.0, 0.0))
+ stage_ms[stage] = (prev_wall + wall_ms, prev_cpu + cpu_ms)
+
+
+def _format_stage_ms(stage_ms: StageTimings) -> str:
+ return " ".join(f"{stage}={wall:.0f}/{cpu:.0f}" for stage, (wall, cpu) in stage_ms.items())
+
+
def _create_onnx_session(
- model_id: str, providers: list[Any], *, allow_download: bool = True
+ model_id: str,
+ providers: list[Any],
+ *,
+ allow_download: bool = True,
+ stage_ms: StageTimings | None = None,
) -> Any:
"""Resolve and load the model's ONNX artifact, trying candidates in order.
@@ -812,6 +851,10 @@ def _create_onnx_session(
cache only; if none is cached, :class:`KompressModelNotCached` is raised
instead of hitting the network. ``onnxruntime`` is imported only after a
candidate resolves, so a cache-only miss never requires it.
+
+ ``stage_ms``, when given, accumulates the cost of ``artifact_resolve``,
+ ``onnxruntime_import``, ``session_build`` and ``smoke_run`` across every
+ candidate tried. See :func:`_timed_stage`.
"""
last_err: Exception | None = None
cache_miss = False
@@ -819,9 +862,10 @@ def _create_onnx_session(
ort: Any = None
for filename in _onnx_filename_candidates():
try:
- onnx_path = _hf_artifact(
- model_id, filename, allow_network=allow_download and refused is None
- )
+ with _timed_stage(stage_ms, "artifact_resolve"):
+ onnx_path = _hf_artifact(
+ model_id, filename, allow_network=allow_download and refused is None
+ )
except KompressModelNotCached as exc:
# Only _hf_artifact's air-gap translation raises this here. It says
# THIS candidate is not cached; a later one may be. Keep looking,
@@ -834,16 +878,19 @@ def _create_onnx_session(
logger.debug("ONNX artifact %r unavailable for %s: %s", filename, model_id, exc)
continue
if ort is None:
- import onnxruntime
+ with _timed_stage(stage_ms, "onnxruntime_import"):
+ import onnxruntime
ort = onnxruntime
try:
- session = ort.InferenceSession(
- onnx_path,
- _onnx_session_options(ort),
- providers=providers,
- )
- _smoke_run(session)
+ with _timed_stage(stage_ms, "session_build"):
+ session = ort.InferenceSession(
+ onnx_path,
+ _onnx_session_options(ort),
+ providers=providers,
+ )
+ with _timed_stage(stage_ms, "smoke_run"):
+ _smoke_run(session)
return session
except Exception as exc:
last_err = exc
@@ -878,7 +925,12 @@ def _load_kompress_onnx(
if model_id in _kompress_cache:
return _kompress_cache[model_id]
- logger.info("Downloading Kompress ONNX model from %s ...", model_id)
+ # Logged before any cache lookup: the artifacts usually come from disk
+ # (startup prefetch), and _hf_artifact goes to the Hub only on a miss.
+ logger.info("Loading Kompress ONNX model from %s ...", model_id)
+ load_started = time.perf_counter()
+ load_cpu_started = time.thread_time()
+ stage_ms: StageTimings = {}
backend = "onnx_coreml" if use_coreml else "onnx"
providers: list[Any]
@@ -907,15 +959,26 @@ def _load_kompress_onnx(
else:
providers = ["CPUExecutionProvider"]
- session = _create_onnx_session(model_id, providers, allow_download=allow_download)
+ session = _create_onnx_session(
+ model_id, providers, allow_download=allow_download, stage_ms=stage_ms
+ )
model = _OnnxModel(session)
- from transformers import AutoTokenizer
+ with _timed_stage(stage_ms, "transformers_import"):
+ from transformers import AutoTokenizer
- tokenizer = _load_modernbert_tokenizer(AutoTokenizer, allow_download=allow_download)
+ with _timed_stage(stage_ms, "tokenizer_load"):
+ tokenizer = _load_modernbert_tokenizer(AutoTokenizer, allow_download=allow_download)
_kompress_cache[model_id] = (model, tokenizer, backend)
- logger.info("Kompress ONNX loaded: %s backend=%s", model_id, backend)
+ logger.info(
+ "Kompress ONNX loaded: %s backend=%s total_ms=%.0f/%.0f stages_ms(wall/cpu): %s",
+ model_id,
+ backend,
+ (time.perf_counter() - load_started) * 1000.0,
+ (time.thread_time() - load_cpu_started) * 1000.0,
+ _format_stage_ms(stage_ms),
+ )
return model, tokenizer, backend
@@ -1341,6 +1404,48 @@ def ensure_background_download(model_id: str = HF_MODEL_ID, device: str = "auto"
thread.start()
+def is_kompress_load_in_progress(model_id: str = HF_MODEL_ID) -> bool:
+ """True while a registered thread (prefetch, download or warm-up) is working on ``model_id``."""
+ with _download_threads_lock:
+ thread = _download_threads.get(model_id)
+ return thread is not None and thread.is_alive()
+
+
+def start_kompress_warmup_thread(model_id: str, warm: Callable[[], None]) -> threading.Thread:
+ """Run ``warm`` on a daemon ``kompress-warmup`` thread registered as ``model_id``'s loader.
+
+ The proxy's post-startup warm-up loads the model on its own thread. Unless
+ that thread is in the per-model registry, a request arriving mid-warm-up
+ finds the model not ready, sees no live loader, and has
+ :func:`ensure_background_download` start a second thread that only queues
+ on ``_kompress_lock`` and then returns the warm-up's cache entry. Registered
+ before it starts, so the warm-up's initial delay is covered too.
+
+ If a prefetch or download thread is already registered, the warm-up takes
+ its place in the registry and waits for it to finish before running
+ ``warm``. Without the wait, a warm-up that failed while a startup prefetch
+ was still downloading would leave the registry pointing at a dead thread,
+ and the next request would start another download next to the hidden
+ prefetch. Waiting also keeps the warm-up from fetching the same files at
+ the same time. Once the warm-up finishes, a failed one included, the next
+ :func:`ensure_background_download` call replaces the dead entry as usual.
+ """
+ with _download_threads_lock:
+ previous = _download_threads.get(model_id)
+
+ def _run() -> None:
+ # Registry entries are always started threads, so join() is safe;
+ # it returns at once for one that has already finished.
+ if previous is not None:
+ previous.join()
+ warm()
+
+ thread = threading.Thread(target=_run, name="kompress-warmup", daemon=True)
+ _download_threads[model_id] = thread
+ thread.start()
+ return thread
+
+
def prefetch_kompress_artifacts(model_id: str = HF_MODEL_ID) -> bool:
"""Download the model's ONNX artifact to the local cache. No native init.
@@ -1751,6 +1856,10 @@ class KompressCompressor(Transform):
"""
ensure_background_download(self.config.model_id, self.config.device)
+ def is_loading(self) -> bool:
+ """True while a background thread is fetching or loading this model."""
+ return is_kompress_load_in_progress(self.config.model_id)
+
def compress(
self,
content: str,
diff --git a/tests/test_content_router_kompress_not_ready_log.py b/tests/test_content_router_kompress_not_ready_log.py
new file mode 100644
index 000000000..34913e728
--- /dev/null
+++ b/tests/test_content_router_kompress_not_ready_log.py
@@ -0,0 +1,126 @@
+"""A Kompress model that is still loading is expected, not a connectivity problem.
+
+The router used to log "Kompress model not ready ... Check HuggingFace
+connectivity" at WARNING on the first request after every proxy start, because
+the post-startup warm-up takes ~20 s and requests arrive meanwhile. It now says
+"still loading" at INFO while a loader thread runs, and keeps the WARNING for
+when nothing is loading the model (e.g. a failed download in its retry backoff).
+"""
+
+from __future__ import annotations
+
+import logging
+import threading
+
+import pytest
+
+from headroom.transforms import kompress_compressor as kc
+from headroom.transforms.content_router import ContentRouter, ContentRouterConfig
+
+_LOGGER = "headroom.transforms.content_router"
+_CONTENT = "some plain text that the ML model would compress. " * 4
+
+
+class _NotReadyKompress:
+ def __init__(self, *, loading: bool) -> None:
+ self.loading = loading
+ self.background_loads = 0
+
+ def is_ready(self) -> bool:
+ return False
+
+ def ensure_background_load(self) -> None:
+ self.background_loads += 1
+
+ def is_loading(self) -> bool:
+ return self.loading
+
+
+def _router(monkeypatch: pytest.MonkeyPatch, compressor: object) -> ContentRouter:
+ router = ContentRouter(
+ ContentRouterConfig(
+ relevance_split=False,
+ lossless_then_lossy=False,
+ ccr_inject_marker=False,
+ )
+ )
+ monkeypatch.setattr(router, "_get_kompress", lambda: compressor)
+ return router
+
+
+def _logged(caplog: pytest.LogCaptureFixture, level: int, text: str) -> int:
+ return sum(
+ 1
+ for r in caplog.records
+ if r.name == _LOGGER and r.levelno == level and text in r.getMessage()
+ )
+
+
+def test_still_loading_is_info_not_a_connectivity_warning(monkeypatch, caplog):
+ compressor = _NotReadyKompress(loading=True)
+ router = _router(monkeypatch, compressor)
+
+ with caplog.at_level(logging.INFO, logger=_LOGGER):
+ for _ in range(3):
+ assert router._try_ml_compressor(_CONTENT, "ctx")[0] == _CONTENT
+
+ assert _logged(caplog, logging.INFO, "still loading in the background") == 1
+ assert _logged(caplog, logging.WARNING, "Check HuggingFace connectivity") == 0
+ assert compressor.background_loads == 3
+
+
+def test_no_loader_running_still_warns_once(monkeypatch, caplog):
+ router = _router(monkeypatch, _NotReadyKompress(loading=False))
+
+ with caplog.at_level(logging.INFO, logger=_LOGGER):
+ for _ in range(3):
+ router._try_ml_compressor(_CONTENT, "ctx")
+
+ assert _logged(caplog, logging.WARNING, "Check HuggingFace connectivity") == 1
+ assert _logged(caplog, logging.INFO, "still loading in the background") == 0
+
+
+def test_warns_once_the_load_ends_without_a_model(monkeypatch, caplog):
+ """A warm-up that fails leaves no loader running: then it is degraded."""
+ compressor = _NotReadyKompress(loading=True)
+ router = _router(monkeypatch, compressor)
+
+ with caplog.at_level(logging.INFO, logger=_LOGGER):
+ router._try_ml_compressor(_CONTENT, "ctx")
+ compressor.loading = False
+ router._try_ml_compressor(_CONTENT, "ctx")
+
+ assert _logged(caplog, logging.INFO, "still loading in the background") == 1
+ assert _logged(caplog, logging.WARNING, "Check HuggingFace connectivity") == 1
+
+
+def test_compressor_without_is_loading_keeps_the_warning(monkeypatch, caplog):
+ class _Legacy:
+ def is_ready(self) -> bool:
+ return False
+
+ def ensure_background_load(self) -> None:
+ pass
+
+ router = _router(monkeypatch, _Legacy())
+
+ with caplog.at_level(logging.INFO, logger=_LOGGER):
+ router._try_ml_compressor(_CONTENT, "ctx")
+
+ assert _logged(caplog, logging.WARNING, "Check HuggingFace connectivity") == 1
+
+
+def test_registered_warm_up_thread_counts_as_loading():
+ """The real signal: the warm-up thread is the model's registered loader."""
+ model_id = "test/not-ready-log-model"
+ release = threading.Event()
+ try:
+ thread = kc.start_kompress_warmup_thread(model_id, lambda: release.wait(5))
+ assert kc.is_kompress_load_in_progress(model_id)
+ release.set()
+ thread.join(timeout=5)
+ assert not kc.is_kompress_load_in_progress(model_id)
+ finally:
+ release.set()
+ with kc._download_threads_lock:
+ kc._download_threads.pop(model_id, None)
diff --git a/tests/test_kompress_download_backoff.py b/tests/test_kompress_download_backoff.py
index ff83a2089..b6ce3bafa 100644
--- a/tests/test_kompress_download_backoff.py
+++ b/tests/test_kompress_download_backoff.py
@@ -154,3 +154,30 @@ def test_a_live_thread_is_never_duplicated(monkeypatch):
finally:
gate.set()
_drain()
+
+
+def test_warm_up_waits_for_a_live_loader_it_replaces(monkeypatch):
+ """A startup prefetch still downloading stays covered, even if the warm-up fails."""
+ started = _spawned(monkeypatch, fails=False)
+ prefetch_done = threading.Event()
+ prefetch = threading.Thread(target=prefetch_done.wait, args=(10,), daemon=True)
+ with kc._download_threads_lock:
+ prefetch.start()
+ kc._download_threads["some/model"] = prefetch
+
+ # A warm-up that returns without caching the model stands in for a failed one.
+ warmed = threading.Event()
+ warm_up = kc.start_kompress_warmup_thread("some/model", warmed.set)
+ try:
+ kc.ensure_background_download("some/model")
+ assert not warmed.wait(0.2), "warm-up ran while the prefetch was still live"
+ assert started == [], "a request started a download beside the live prefetch"
+ finally:
+ prefetch_done.set()
+ warm_up.join(timeout=10)
+ assert warmed.is_set()
+
+ # Both are done and the model is still not cached: a request may load it now.
+ kc.ensure_background_download("some/model")
+ _drain()
+ assert started == ["some/model"]
diff --git a/tests/test_kompress_failsafe.py b/tests/test_kompress_failsafe.py
index 26e4c7d1b..973ab7bfc 100644
--- a/tests/test_kompress_failsafe.py
+++ b/tests/test_kompress_failsafe.py
@@ -609,6 +609,71 @@ def test_all_artifacts_failing_at_run_raises_rather_than_returning_a_dead_sessio
kc._create_onnx_session("org/model", ["CPUExecutionProvider"])
+# ── Load-stage timings: say where a slow warm-up spent its time ────────────────
+
+
+def _install_ticking_clock(monkeypatch):
+ """Each clock reading is 1s after the last, so every timed stage costs 1000ms."""
+
+ def ticking_clock():
+ now = [0.0]
+
+ def tick() -> float:
+ now[0] += 1.0
+ return now[0]
+
+ return staticmethod(tick)
+
+ clock = type("_Clock", (), {"perf_counter": ticking_clock(), "thread_time": ticking_clock()})
+ monkeypatch.setattr(kc, "time", clock)
+
+
+def test_stage_timings_count_every_candidate_attempt(monkeypatch):
+ """A candidate rejected at smoke-run still costs time, so it shows in the totals."""
+ _install_fake_ort(monkeypatch, run_fails_for={"int8-wo"})
+ _install_ticking_clock(monkeypatch)
+ stage_ms: kc.StageTimings = {}
+
+ kc._create_onnx_session("org/model", ["CPUExecutionProvider"], stage_ms=stage_ms)
+
+ # int8-wo resolved, built and smoke-run (rejected); fp32 the same (accepted).
+ # onnxruntime is imported once, on the first resolved candidate.
+ assert stage_ms == {
+ "artifact_resolve": (2000.0, 2000.0),
+ "onnxruntime_import": (1000.0, 1000.0),
+ "session_build": (2000.0, 2000.0),
+ "smoke_run": (2000.0, 2000.0),
+ }
+
+
+def test_onnx_load_logs_every_stage(monkeypatch, caplog):
+ import sys
+ from types import SimpleNamespace
+
+ _install_fake_ort(monkeypatch, run_fails_for=set())
+ monkeypatch.setitem(sys.modules, "transformers", SimpleNamespace(AutoTokenizer=object()))
+ monkeypatch.setattr(kc, "_kompress_cache", {})
+ monkeypatch.setattr(
+ kc, "_load_modernbert_tokenizer", lambda _auto, *, allow_download: "tokenizer"
+ )
+
+ with caplog.at_level("INFO", logger=kc.logger.name):
+ kc._load_kompress_onnx("org/model")
+
+ loaded = [r.getMessage() for r in caplog.records if "Kompress ONNX loaded" in r.getMessage()]
+ assert len(loaded) == 1
+ for stage in (
+ "artifact_resolve=",
+ "onnxruntime_import=",
+ "session_build=",
+ "smoke_run=",
+ "transformers_import=",
+ "tokenizer_load=",
+ ):
+ assert stage in loaded[0]
+ assert "Downloading" not in caplog.text
+
+
# ── Failure latch: a broken model stops costing us every request ───────────────
diff --git a/tests/test_proxy/test_kompress_background_warmup.py b/tests/test_proxy/test_kompress_background_warmup.py
index 69d68da0b..d03e61372 100644
--- a/tests/test_proxy/test_kompress_background_warmup.py
+++ b/tests/test_proxy/test_kompress_background_warmup.py
@@ -14,10 +14,14 @@ from types import SimpleNamespace
import pytest
from headroom.proxy.server import HeadroomProxy
+from headroom.transforms import kompress_compressor as kc
+
+_MODEL_ID = "test/warmup-model"
class _Kompress:
def __init__(self) -> None:
+ self.config = SimpleNamespace(model_id=_MODEL_ID)
self.preloaded = threading.Event()
def preload(self, *, allow_download: bool = True) -> str:
@@ -37,6 +41,16 @@ def _no_delay(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.delenv("HEADROOM_KOMPRESS_WARMUP", raising=False)
[email protected](autouse=True)
+def _clean_registry():
+ yield
+ with kc._download_threads_lock:
+ thread = kc._download_threads.pop(_MODEL_ID, None)
+ kc._download_failures.pop(_MODEL_ID, None)
+ if thread is not None:
+ thread.join(timeout=5)
+
+
def test_warms_on_a_background_thread(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr("platform.libc_ver", lambda: ("glibc", "2.35"))
monkeypatch.delenv("PYTEST_CURRENT_TEST", raising=False)
@@ -74,3 +88,41 @@ def test_test_processes_do_not_warm_unless_forced(monkeypatch: pytest.MonkeyPatc
monkeypatch.setattr("platform.libc_ver", lambda: ("glibc", "2.35"))
monkeypatch.setenv("PYTEST_CURRENT_TEST", "x")
assert HeadroomProxy._start_kompress_background_warmup(_proxy_with(_Kompress())) is False
+
+
+def test_request_during_warm_up_does_not_start_a_second_loader(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """The warm-up is the model's registered loader until it finishes."""
+ monkeypatch.setattr("platform.libc_ver", lambda: ("glibc", "2.35"))
+ monkeypatch.delenv("PYTEST_CURRENT_TEST", raising=False)
+ monkeypatch.setattr(kc, "_kompress_cache", {})
+ loads: list[str] = []
+ monkeypatch.setattr(
+ kc,
+ "_load_kompress",
+ lambda model_id, device, allow_download=True: (
+ loads.append(model_id) or (None, None, "onnx")
+ ),
+ )
+ release = threading.Event()
+
+ class _SlowKompress(_Kompress):
+ def preload(self, *, allow_download: bool = True) -> str:
+ self.preloaded.set()
+ release.wait(5)
+ raise OSError("warm-up failed")
+
+ kompress = _SlowKompress()
+ assert HeadroomProxy._start_kompress_background_warmup(_proxy_with(kompress)) is True
+ assert kompress.preloaded.wait(5)
+
+ kc.ensure_background_download(_MODEL_ID)
+ assert loads == []
+
+ # Once the warm-up is over (here, failed), a request still starts its own load.
+ release.set()
+ kc._download_threads[_MODEL_ID].join(timeout=5)
+ kc.ensure_background_download(_MODEL_ID)
+ kc._download_threads[_MODEL_ID].join(timeout=5)
+ assert loads == [_MODEL_ID]