Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(proxy): warm deferred kompress model so /health stops reporting backend=null (#2799)

Parideboye2350bc8fdbd
diff --git a/headroom/proxy/kompress_health_detail_policy.py b/headroom/proxy/kompress_health_detail_policy.py
new file mode 100644
index 000000000..80965459b
--- /dev/null
+++ b/headroom/proxy/kompress_health_detail_policy.py
@@ -0,0 +1,70 @@
+"""Policy for the public ``detail`` field on the kompress health check.
+
+``/health`` and ``/readyz`` are auth-exempt, so everything they serialize is
+unauthenticated public data. The kompress check reports ``detail`` because
+``ready=false`` on its own is ambiguous — the model may be warming in the
+background, missing from the local cache, or the ML extras may not be installed
+at all (#2730).
+
+That diagnostic value must not come at the cost of leaking internals. Native
+loader, model and cache failures routinely embed absolute filesystem paths,
+model identifiers, endpoint URLs and configuration values in their exception
+text, and a warmup slot's ``error`` string is populated from exactly those
+failures. So the public field draws from a closed vocabulary: anything not in
+it degrades to ``unavailable`` rather than being echoed.
+
+The precise cause is not lost — it stays in the proxy log and in the warmup
+slot, which ``/debug/warmup`` serializes behind authentication.
+"""
+
+from __future__ import annotations
+
+# The model is loaded and serving.
+KOMPRESS_DETAIL_LOADED = "loaded"
+# The background warm thread is still running.
+KOMPRESS_DETAIL_WARMING = "warming"
+# Cache-only warm found no local artifacts; no network call was made.
+KOMPRESS_DETAIL_NOT_CACHED = "model not cached"
+# The warm attempt raised. The cause is in the log, not here.
+KOMPRESS_DETAIL_WARM_FAILED = "warm failed"
+# The ML extras are absent, or no compressor was ever constructed.
+KOMPRESS_DETAIL_NOT_INSTALLED = "not installed"
+
+# ``ContentRouter.eager_load_compressors()`` source statuses. These are fixed
+# literals in the router, never interpolated, so they are safe to surface.
+KOMPRESS_DETAIL_ENABLED = "enabled"
+KOMPRESS_DETAIL_DEFERRED = "deferred"
+KOMPRESS_DETAIL_UNAVAILABLE = "unavailable"
+
+#: Every value ``/health`` and ``/readyz`` may report for ``kompress.detail``.
+KOMPRESS_HEALTH_DETAILS = frozenset(
+ {
+ KOMPRESS_DETAIL_LOADED,
+ KOMPRESS_DETAIL_WARMING,
+ KOMPRESS_DETAIL_NOT_CACHED,
+ KOMPRESS_DETAIL_WARM_FAILED,
+ KOMPRESS_DETAIL_NOT_INSTALLED,
+ KOMPRESS_DETAIL_ENABLED,
+ KOMPRESS_DETAIL_DEFERRED,
+ KOMPRESS_DETAIL_UNAVAILABLE,
+ }
+)
+
+#: What an out-of-vocabulary value collapses to.
+DEFAULT_KOMPRESS_HEALTH_DETAIL = KOMPRESS_DETAIL_UNAVAILABLE
+
+
+def public_detail(value: str | None) -> str:
+ """Return ``value`` if it is a known detail, else the bounded fallback.
+
+ This is the single choke point for the field. Route *every* candidate
+ through it — including strings that look safe at the call site, such as a
+ warmup slot's ``source_status`` — so a future code path cannot introduce a
+ leak by writing free-form text into the slot.
+ """
+ if not isinstance(value, str):
+ return DEFAULT_KOMPRESS_HEALTH_DETAIL
+ normalized = value.strip().lower()
+ if normalized in KOMPRESS_HEALTH_DETAILS:
+ return normalized
+ return DEFAULT_KOMPRESS_HEALTH_DETAIL
diff --git a/headroom/proxy/server.py b/headroom/proxy/server.py
index 9346e2333..24c3d1a03 100644
--- a/headroom/proxy/server.py
+++ b/headroom/proxy/server.py
@@ -147,6 +147,14 @@ from headroom.proxy.helpers import (
resolve_display_provider,
retry_after_ms,
)
+from headroom.proxy.kompress_health_detail_policy import (
+ KOMPRESS_DETAIL_LOADED,
+ KOMPRESS_DETAIL_NOT_CACHED,
+ KOMPRESS_DETAIL_NOT_INSTALLED,
+ KOMPRESS_DETAIL_WARM_FAILED,
+ KOMPRESS_DETAIL_WARMING,
+ public_detail,
+)
from headroom.proxy.loop_callback_failure_policy import is_known_websocket_callback_failure
from headroom.proxy.loopback_guard import is_loopback_host, is_loopback_host_header
from headroom.proxy.malloc_trim import trim_periodically
@@ -1388,6 +1396,9 @@ class HeadroomProxy(
)
self._compression_quarantine_releases: int = 0
self._compression_metrics_lock = threading.Lock()
+ # Daemon thread that warms the Kompress model after startup when the
+ # eager preload deferred the native load (GH #2730). None until started.
+ self._kompress_warm_thread: threading.Thread | None = None
# Backend for Anthropic API (direct, LiteLLM, or any-llm)
# Supports: "anthropic" (direct), "bedrock", "vertex", "litellm-<provider>", or "anyllm"
@@ -1997,6 +2008,13 @@ class HeadroomProxy(
cost off the request path. Skipped on glibc older than 2.28 (the
affected host family) and when ``HEADROOM_KOMPRESS_WARMUP`` is ``0``;
``1`` forces it. Returns ``True`` when a warm-up thread was started.
+
+ The thread also promotes ``warmup.kompress``. The eager preload
+ deliberately skips ``preload()`` so a native load can never block the
+ port bind (GH #790), which left the slot at ``status="null"``, and
+ ``_reconcile_kompress_health`` only promotes it once a compressor
+ reports ``is_ready()`` — so an idle proxy reported
+ ``kompress: unhealthy, backend: null`` forever (GH #2730).
"""
raw = os.environ.get("HEADROOM_KOMPRESS_WARMUP", "").strip().lower()
if raw in ("0", "false", "no", "off"):
@@ -2043,14 +2061,37 @@ class HeadroomProxy(
except ValueError:
delay = 2.0
+ slot = self.warmup.kompress
+
def _warm() -> None:
time.sleep(max(0.0, delay))
started = time.monotonic()
+ try:
+ from headroom.transforms.kompress_compressor import KompressModelNotCached
+
+ not_cached: tuple[type[BaseException], ...] = (KompressModelNotCached,)
+ except Exception: # ML extras absent; every failure is just "failed"
+ not_cached = ()
try:
backend = compressor.preload(allow_download=True)
- except Exception as exc: # the lazy request path still loads on first use
- logger.warning("Kompress background warm-up failed: %s", exc)
+ except not_cached:
+ # The local loader only raises this when downloads are off, so
+ # with allow_download=True a cold cache downloads instead. This
+ # covers compressors that report a cache miss on their own.
+ slot.info["detail"] = KOMPRESS_DETAIL_NOT_CACHED
+ logger.debug("Kompress background warm-up: model not cached")
return
+ except Exception:
+ # The detail lands on the auth-exempt /health payload, so it
+ # stays a bounded token; loader failures embed paths, model
+ # ids and URLs. The real cause goes to the log only.
+ slot.info["detail"] = KOMPRESS_DETAIL_WARM_FAILED
+ logger.warning("Kompress background warm-up failed", exc_info=True)
+ return
+ if not backend:
+ return
+ slot.info.pop("detail", None)
+ slot.mark_loaded(handle=compressor, backend=backend, source_status="background")
logger.info(
"Kompress: warmed in the background in %.0f ms (backend %s)",
(time.monotonic() - started) * 1000,
@@ -2061,7 +2102,7 @@ class HeadroomProxy(
# Registered as the model's loader, so a request during the warm-up
# doesn't start a second load thread.
- start_kompress_warmup_thread(compressor.config.model_id, _warm)
+ self._kompress_warm_thread = start_kompress_warmup_thread(compressor.config.model_id, _warm)
return True
async def startup(self):
@@ -3168,6 +3209,31 @@ class ActivityMiddleware:
self.proxy._activity_generation += 1
+def _kompress_routers(proxy: HeadroomProxy) -> list[ContentRouter]:
+ """Kompress-enabled ContentRouters across both pipelines, deduped by identity."""
+ routers: list[ContentRouter] = []
+ for pipeline in (proxy.anthropic_pipeline, proxy.openai_pipeline):
+ for transform in getattr(pipeline, "transforms", ()):
+ if (
+ isinstance(transform, ContentRouter)
+ and transform.config.enable_kompress
+ and all(transform is not item for item in routers)
+ ):
+ routers.append(transform)
+ return routers
+
+
+def _kompress_compressors(proxy: HeadroomProxy) -> list[Any]:
+ """Compressor instances held by the kompress-enabled routers, deduped."""
+ compressors: list[Any] = []
+ for router in _kompress_routers(proxy):
+ for name in ("_kompress", "_kompress_remote"):
+ compressor = getattr(router, name, None)
+ if compressor is not None and all(compressor is not item for item in compressors):
+ compressors.append(compressor)
+ return compressors
+
+
def create_app(config: ProxyConfig | None = None) -> FastAPI:
"""Create FastAPI application."""
if not FASTAPI_AVAILABLE:
@@ -3567,30 +3633,13 @@ def create_app(config: ProxyConfig | None = None) -> FastAPI:
return result
def _kompress_health_routers() -> list[ContentRouter]:
- routers: list[ContentRouter] = []
- for pipeline in (proxy.anthropic_pipeline, proxy.openai_pipeline):
- for transform in getattr(pipeline, "transforms", ()):
- if (
- isinstance(transform, ContentRouter)
- and transform.config.enable_kompress
- and all(transform is not item for item in routers)
- ):
- routers.append(transform)
- return routers
+ return _kompress_routers(proxy)
def _reconcile_kompress_health() -> bool:
- routers = _kompress_health_routers()
- if not routers:
+ if not _kompress_health_routers():
return False
- compressors: list[Any] = []
- for router in routers:
- for name in ("_kompress", "_kompress_remote"):
- compressor = getattr(router, name, None)
- if compressor is not None and all(compressor is not item for item in compressors):
- compressors.append(compressor)
-
- for compressor in compressors:
+ for compressor in _kompress_compressors(proxy):
try:
if not compressor.is_ready():
continue
@@ -3624,6 +3673,35 @@ def create_app(config: ProxyConfig | None = None) -> FastAPI:
)
return True
+ def _kompress_detail() -> str:
+ """Human-readable reason behind the kompress readiness bit.
+
+ ``ready=false`` on its own is ambiguous — the model may be warming in
+ the background, missing from the local cache, or the extras may not be
+ installed at all (GH #2730).
+
+ This value is served by the auth-exempt ``/health`` and ``/readyz``, so
+ every candidate goes through :func:`public_detail` and anything outside
+ ``KOMPRESS_HEALTH_DETAILS`` collapses to ``unavailable``. That matters
+ most for ``slot.error``, which carries raw loader exception text.
+ ``/debug/warmup`` and the proxy log still hold the precise cause.
+ """
+ slot = proxy.warmup.kompress
+ if slot.status == "loaded":
+ return KOMPRESS_DETAIL_LOADED
+ detail = slot.info.get("detail")
+ if isinstance(detail, str) and detail:
+ return public_detail(detail)
+ if slot.error:
+ return public_detail(slot.error)
+ warm_thread = getattr(proxy, "_kompress_warm_thread", None)
+ if warm_thread is not None and warm_thread.is_alive():
+ return KOMPRESS_DETAIL_WARMING
+ source_status = slot.info.get("source_status")
+ if isinstance(source_status, str) and source_status:
+ return public_detail(source_status)
+ return KOMPRESS_DETAIL_NOT_INSTALLED
+
def _health_checks() -> dict[str, dict[str, Any]]:
kompress_enabled = _reconcile_kompress_health()
memory_status = (
@@ -3676,6 +3754,7 @@ def create_app(config: ProxyConfig | None = None) -> FastAPI:
ready=proxy.warmup.kompress.status == "loaded",
optional=True,
backend=proxy.warmup.kompress.info.get("backend", None),
+ detail=_kompress_detail(),
),
}
diff --git a/tests/test_kompress_health_detail_policy.py b/tests/test_kompress_health_detail_policy.py
new file mode 100644
index 000000000..937a52114
--- /dev/null
+++ b/tests/test_kompress_health_detail_policy.py
@@ -0,0 +1,38 @@
+"""The public kompress health detail must stay inside a closed vocabulary."""
+
+import pytest
+
+from headroom.proxy.kompress_health_detail_policy import (
+ DEFAULT_KOMPRESS_HEALTH_DETAIL,
+ KOMPRESS_HEALTH_DETAILS,
+ public_detail,
+)
+
+
[email protected]("value", sorted(KOMPRESS_HEALTH_DETAILS))
+def test_known_details_pass_through(value):
+ assert public_detail(value) == value
+
+
+ "value",
+ [
+ "warm failed: /home/opt/models/modernbert.onnx not readable",
+ "OSError: cannot open shared object /usr/lib/libonnxruntime.so",
+ "HTTPError 401 for https://internal.example/models?token=abc123",
+ "",
+ " ",
+ None,
+ 123,
+ ],
+)
+def test_unknown_details_collapse(value):
+ assert public_detail(value) == DEFAULT_KOMPRESS_HEALTH_DETAIL
+
+
+def test_details_are_normalized():
+ assert public_detail(" Model Not Cached ") == "model not cached"
+
+
+def test_default_is_itself_a_known_detail():
+ assert DEFAULT_KOMPRESS_HEALTH_DETAIL in KOMPRESS_HEALTH_DETAILS
diff --git a/tests/test_kompress_preload_deferral.py b/tests/test_kompress_preload_deferral.py
index 2dff7a99d..3ea0f2aad 100644
--- a/tests/test_kompress_preload_deferral.py
+++ b/tests/test_kompress_preload_deferral.py
@@ -6,6 +6,8 @@ the cached Kompress native stack on the blocking startup/lifespan path.
from __future__ import annotations
+from types import SimpleNamespace
+
import pytest
from headroom import onnx_runtime
@@ -96,6 +98,7 @@ def test_load_kompress_auto_does_not_pytorch_download_on_cache_miss(monkeypatch)
class _StubCompressor:
def __init__(self, *, cached: bool):
+ self.config = SimpleNamespace(model_id="org/model")
self._cached = cached
self.preload_calls: list[bool] = []
@@ -260,6 +263,9 @@ async def test_proxy_startup_does_not_enter_cached_kompress_native_loader(monkey
code_aware_enabled=False,
)
)
+ # The background warm thread (GH #2730) is allowed to preload; this test
+ # covers the *awaited* startup path only, so disable it for determinism.
+ monkeypatch.setenv("HEADROOM_KOMPRESS_WARMUP", "0")
router = _router_kompress_only()
stub = _FatalPreloadCompressor(cached=True)
monkeypatch.setattr(router, "_get_kompress", lambda: stub)
@@ -270,5 +276,112 @@ async def test_proxy_startup_does_not_enter_cached_kompress_native_loader(monkey
try:
assert stub.preload_calls == []
assert proxy.warmup.kompress.info["source_status"] == "deferred"
+ assert proxy._kompress_warm_thread is None
+ finally:
+ await proxy.shutdown()
+
+
+def _kompress_proxy(monkeypatch, stub):
+ from headroom.proxy.server import HeadroomProxy, ProxyConfig
+
+ proxy = HeadroomProxy(
+ ProxyConfig(
+ optimize=True,
+ cache_enabled=False,
+ rate_limit_enabled=False,
+ cost_tracking_enabled=False,
+ code_aware_enabled=False,
+ )
+ )
+ router = _router_kompress_only()
+ monkeypatch.setattr(router, "_get_kompress", lambda: stub)
+ proxy.anthropic_pipeline.transforms = [router]
+ proxy.openai_pipeline.transforms = [router]
+ return proxy
+
+
+def _enable_startup_warm(monkeypatch):
+ """Opt the deferred Kompress warm-up in, without the startup delay.
+
+ The warm-up is skipped under pytest unless asked for explicitly, and its
+ 2 s settle delay is only there to keep startup responsive.
+ """
+ monkeypatch.setenv("HEADROOM_KOMPRESS_WARMUP", "1")
+ monkeypatch.setenv("HEADROOM_KOMPRESS_WARMUP_DELAY_SECONDS", "0")
+
+
+async def test_startup_background_warm_promotes_deferred_kompress(monkeypatch):
+ """A cached model must reach the warmup slot without any request (GH #2730)."""
+ pytest.importorskip("httpx")
+ _enable_startup_warm(monkeypatch)
+
+ stub = _StubCompressor(cached=True)
+ proxy = _kompress_proxy(monkeypatch, stub)
+
+ await proxy.startup()
+ try:
+ thread = proxy._kompress_warm_thread
+ assert thread is not None
+ thread.join(timeout=10)
+ assert not thread.is_alive()
+ assert stub.preload_calls == [True]
+ assert proxy.warmup.kompress.status == "loaded"
+ assert proxy.warmup.kompress.info["backend"] == "onnx"
+ assert proxy.warmup.kompress.info["source_status"] == "background"
+ finally:
+ await proxy.shutdown()
+
+
+async def test_startup_background_warm_reports_compressor_cache_miss(monkeypatch):
+ """A compressor that reports a cache miss stays deferred and says why.
+
+ The local loader downloads on a cold cache under ``allow_download=True``;
+ this covers compressors that raise ``KompressModelNotCached`` regardless.
+ """
+ pytest.importorskip("httpx")
+ _enable_startup_warm(monkeypatch)
+
+ stub = _StubCompressor(cached=False)
+ proxy = _kompress_proxy(monkeypatch, stub)
+
+ await proxy.startup()
+ try:
+ thread = proxy._kompress_warm_thread
+ assert thread is not None
+ thread.join(timeout=10)
+ assert stub.preload_calls == [True]
+ assert proxy.warmup.kompress.status == "null"
+ assert proxy.warmup.kompress.info["detail"] == "model not cached"
+ finally:
+ await proxy.shutdown()
+
+
+class _LeakyPreloadCompressor(_StubCompressor):
+ """Fails with an exception carrying a path and a token, as loaders do."""
+
+ def preload(self, *, allow_download: bool = True) -> str:
+ self.preload_calls.append(allow_download)
+ raise RuntimeError("/opt/secret-path/model.onnx token=SENTINEL-LEAK-9f3a")
+
+
+async def test_startup_background_warm_failure_detail_is_bounded(monkeypatch):
+ """The warm failure reason must not reach the auth-exempt health payload."""
+ pytest.importorskip("httpx")
+ _enable_startup_warm(monkeypatch)
+
+ stub = _LeakyPreloadCompressor(cached=True)
+ proxy = _kompress_proxy(monkeypatch, stub)
+
+ await proxy.startup()
+ try:
+ thread = proxy._kompress_warm_thread
+ assert thread is not None
+ thread.join(timeout=10)
+ assert stub.preload_calls == [True]
+ assert proxy.warmup.kompress.status == "null"
+ assert proxy.warmup.kompress.info["detail"] == "warm failed"
finally:
await proxy.shutdown()
diff --git a/tests/test_proxy/test_kompress_background_warmup.py b/tests/test_proxy/test_kompress_background_warmup.py
index 98dcff7d2..3d08aea13 100644
--- a/tests/test_proxy/test_kompress_background_warmup.py
+++ b/tests/test_proxy/test_kompress_background_warmup.py
@@ -29,10 +29,25 @@ class _Kompress:
return "onnx"
+class _Slot:
+ """Minimal stand-in for ``WarmupRegistry``'s kompress component slot."""
+
+ def __init__(self) -> None:
+ self.status = "null"
+ self.info: dict[str, object] = {}
+
+ def mark_loaded(self, *, handle: object, backend: str, **_: object) -> None:
+ self.status = "loaded"
+ self.info["handle"] = handle
+ self.info["backend"] = backend
+
+
def _proxy_with(compressor: object | None) -> SimpleNamespace:
router = SimpleNamespace(_get_kompress=lambda: compressor)
pipeline = SimpleNamespace(transforms=[router])
- return SimpleNamespace(anthropic_pipeline=pipeline, openai_pipeline=pipeline)
+ # The warm-up promotes this slot so /health stops reporting backend=null.
+ warmup = SimpleNamespace(kompress=_Slot())
+ return SimpleNamespace(anthropic_pipeline=pipeline, openai_pipeline=pipeline, warmup=warmup)
@pytest.fixture(autouse=True)
@@ -55,8 +70,16 @@ def test_warms_on_a_background_thread(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr("platform.libc_ver", lambda: ("glibc", "2.35"))
monkeypatch.delenv("PYTEST_CURRENT_TEST", raising=False)
kompress = _Kompress()
- assert HeadroomProxy._start_kompress_background_warmup(_proxy_with(kompress)) is True
+ proxy = _proxy_with(kompress)
+ assert HeadroomProxy._start_kompress_background_warmup(proxy) is True
assert kompress.preloaded.wait(5)
+ # /health reads this slot; without the promotion an idle proxy reports
+ # kompress backend=null forever (GH #2730).
+ assert proxy.warmup.kompress.status == "loaded"
+ assert proxy.warmup.kompress.info["backend"] == "onnx"
+ thread = proxy._kompress_warm_thread
+ thread.join(timeout=5)
+ assert not thread.is_alive()
def test_skipped_on_the_glibc_family_that_crashes(monkeypatch: pytest.MonkeyPatch) -> None:
diff --git a/tests/test_proxy_eager_preload_bind.py b/tests/test_proxy_eager_preload_bind.py
index 4502890a1..0bf6f95f4 100644
--- a/tests/test_proxy_eager_preload_bind.py
+++ b/tests/test_proxy_eager_preload_bind.py
@@ -162,7 +162,10 @@ async def test_startup_reports_deferred_kompress(caplog):
server_mod.logger.removeHandler(caplog.handler)
assert proxy._kompress_status == "deferred"
- assert "Kompress: DEFERRED (model loads on first request)" in caplog.messages
+ # The wording after the prefix is conditional (a background warm-up
+ # thread started or not), so pin the prefix and the real invariant:
+ # deferred must never be reported as "not installed".
+ assert any(message.startswith("Kompress: DEFERRED") for message in caplog.messages)
assert not any("Kompress: not installed" in message for message in caplog.messages)
finally:
await proxy.shutdown()
diff --git a/tests/test_proxy_health.py b/tests/test_proxy_health.py
index d931eeb28..f7f0b4a59 100644
--- a/tests/test_proxy_health.py
+++ b/tests/test_proxy_health.py
@@ -72,6 +72,7 @@ def test_readyz_excludes_kompress_from_aggregate_readiness(monkeypatch):
"status": "degraded",
"optional": True,
"backend": None,
+ "detail": "model not cached",
}
@@ -88,6 +89,7 @@ def test_readyz_promotes_deferred_kompress_after_runtime_load(monkeypatch):
"status": "healthy",
"optional": True,
"backend": "onnx",
+ "detail": "loaded",
}
# The promotion must also clear the startup marker, otherwise the slot
# serializes as loaded-but-deferred in /debug/warmup.
@@ -114,6 +116,7 @@ def test_readyz_promotes_kompress_from_module_cache(monkeypatch, attached):
"status": "healthy",
"optional": True,
"backend": "onnx",
+ "detail": "loaded",
}
assert proxy.warmup.kompress.handle is model
if compressor is not None:
@@ -147,6 +150,7 @@ def test_readyz_keeps_pending_kompress_unloaded(monkeypatch):
"status": "degraded",
"optional": True,
"backend": None,
+ "detail": "not installed",
}
assert compressor.calls == ["is_ready"]
@@ -187,6 +191,7 @@ def test_readyz_disabled_kompress_skips_inspection(monkeypatch):
"status": "disabled",
"optional": True,
"backend": None,
+ "detail": "not installed",
}
assert compressor.calls == []
@@ -209,6 +214,7 @@ def test_readyz_per_provider_kompress_override_reenables_health(monkeypatch):
"status": "healthy",
"optional": True,
"backend": "onnx",
+ "detail": "loaded",
}
assert compressor.calls == ["is_ready", "ready_backend"]
@@ -231,6 +237,7 @@ def test_readyz_never_calls_lazy_kompress_getters(monkeypatch):
"status": "degraded",
"optional": True,
"backend": None,
+ "detail": "not installed",
}
@@ -247,6 +254,7 @@ def test_readyz_never_calls_lazy_kompress_getters(monkeypatch):
"status": "degraded",
"optional": True,
"backend": None,
+ "detail": "not installed",
},
),
(
@@ -259,6 +267,7 @@ def test_readyz_never_calls_lazy_kompress_getters(monkeypatch):
"status": "healthy",
"optional": True,
"backend": "onnx",
+ "detail": "loaded",
},
),
(
@@ -271,6 +280,7 @@ def test_readyz_never_calls_lazy_kompress_getters(monkeypatch):
"status": "healthy",
"optional": True,
"backend": "remote",
+ "detail": "loaded",
},
),
(
@@ -283,6 +293,7 @@ def test_readyz_never_calls_lazy_kompress_getters(monkeypatch):
"status": "healthy",
"optional": True,
"backend": "onnx",
+ "detail": "loaded",
},
),
(
@@ -295,6 +306,7 @@ def test_readyz_never_calls_lazy_kompress_getters(monkeypatch):
"status": "healthy",
"optional": True,
"backend": "existing",
+ "detail": "loaded",
},
),
(
@@ -307,6 +319,7 @@ def test_readyz_never_calls_lazy_kompress_getters(monkeypatch):
"status": "disabled",
"optional": True,
"backend": None,
+ "detail": "not installed",
},
),
],
@@ -325,6 +338,36 @@ def test_readyz_kompress_state_matrix(monkeypatch, slot_status, compressor, disa
assert payload == expected
+# /health and /readyz are auth-exempt, so nothing they serialize may carry raw
+# exception text: loader failures embed absolute paths, model ids and URLs.
+_SENTINEL_DETAIL = "/opt/secret-path/model.onnx token=SENTINEL-LEAK-9f3a"
+
+
[email protected]("endpoint", ["/health", "/readyz"])
+def test_public_health_does_not_leak_warm_failure_text(monkeypatch, endpoint):
+ app, proxy = _health_app(monkeypatch)
+ # What the background warm thread would have written pre-#2799 review.
+ proxy.warmup.kompress.info["detail"] = f"warm failed: {_SENTINEL_DETAIL}"
+
+ response = TestClient(app).get(endpoint)
+
+ assert response.json()["checks"]["kompress"]["detail"] == "unavailable"
+ assert "SENTINEL-LEAK-9f3a" not in response.text
+ assert "secret-path" not in response.text
+
+
[email protected]("endpoint", ["/health", "/readyz"])
+def test_public_health_does_not_leak_slot_error_text(monkeypatch, endpoint):
+ app, proxy = _health_app(monkeypatch)
+ proxy.warmup.kompress.mark_error(f"native load failed: {_SENTINEL_DETAIL}")
+
+ response = TestClient(app).get(endpoint)
+
+ assert response.json()["checks"]["kompress"]["detail"] == "unavailable"
+ assert "SENTINEL-LEAK-9f3a" not in response.text
+ assert "secret-path" not in response.text
+
+
def test_remote_kompress_endpoint_skips_local_model_at_startup(monkeypatch):
"""With HEADROOM_KOMPRESS_ENDPOINT set, startup neither fetches nor warms the
local model, and /readyz still names the remote backend."""