Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
"""Startup must bind its port even when eager preload hangs (#790).
``HeadroomProxy.startup()`` runs inside the ASGI lifespan, which completes
*before* uvicorn binds the socket. The eager compressor/parser preload used to
run synchronously there, so a hang or an uncatchable native stall during a model
load (observed on Windows) left the proxy "never opening its port". The preload
now runs off the event loop under ``asyncio.wait_for`` with
``EAGER_PRELOAD_TIMEOUT_SECONDS``; on timeout startup logs and continues so the
bind still happens and transforms fall back to lazy loading.
"""
from __future__ import annotations
import logging
import threading
import time
import pytest
pytest.importorskip("fastapi")
import headroom.proxy.server as server_mod
from headroom.proxy.server import ProxyConfig, create_app
def _make_proxy(
*,
optimize: bool,
disable_kompress: bool = False,
disable_kompress_anthropic: bool | None = None,
disable_kompress_openai: bool | None = None,
compressors: set[str] | None = None,
):
config = ProxyConfig(
optimize=optimize,
disable_kompress=disable_kompress,
disable_kompress_anthropic=disable_kompress_anthropic,
disable_kompress_openai=disable_kompress_openai,
compressors=compressors,
cache_enabled=False,
rate_limit_enabled=False,
cost_tracking_enabled=False,
log_requests=False,
ccr_inject_tool=False,
ccr_handle_responses=False,
ccr_context_tracking=False,
image_optimize=False,
subscription_tracking_enabled=False,
)
proxy = create_app(config).state.proxy
# Every test here substitutes fake pipelines to control exactly what the
# preload walks. The proxy also eagerly builds the default /v1/compress
# pipeline (a derived ContentRouter, warmed alongside the request
# pipelines), which would inject real transform statuses into those
# assertions — drop it so the fakes remain the only input.
proxy._compress_pipeline_cache = {}
return proxy
class _FastTransform:
def __init__(self, status):
self._status = status
def eager_load_compressors(self):
return self._status
class _RaisingTransform:
def eager_load_compressors(self):
raise RuntimeError("boom")
class _NonDictTransform:
def eager_load_compressors(self):
return "not-a-dict"
class _HangingTransform:
"""Simulates a model load that hangs forever (released via the event)."""
def __init__(self, release: threading.Event):
self._release = release
def eager_load_compressors(self):
# Safety cap so a misbehaving test can never wedge the suite.
self._release.wait(timeout=30)
return {"hang": "done"}
class _FakePipeline:
def __init__(self, transforms):
self.transforms = transforms
def test_eager_preload_dedupes_and_swallows_failures():
proxy = _make_proxy(optimize=False)
shared = _FastTransform({"shared": "enabled"})
proxy.anthropic_pipeline = _FakePipeline([shared, _FastTransform({"kompress": "enabled"})])
# ``shared`` appears in both pipelines and must load exactly once; the
# raising and non-dict transforms must be skipped without aborting.
proxy.openai_pipeline = _FakePipeline([shared, _RaisingTransform(), _NonDictTransform()])
eager_status, statuses = proxy._eager_preload_transforms()
# Keys the preload contributes itself rather than collecting from a
# transform, so this assertion stays about dedupe/swallowing.
non_transform_keys = {"litellm"}
assert {k: v for k, v in eager_status.items() if k not in non_transform_keys} == {
"shared": "enabled",
"kompress": "enabled",
}
assert statuses == [{"shared": "enabled"}, {"kompress": "enabled"}]
assert eager_status["litellm"] in {"ready", "not installed", "skipped"}
async def test_startup_binds_despite_hung_preload(monkeypatch):
monkeypatch.setattr(server_mod, "EAGER_PRELOAD_TIMEOUT_SECONDS", 0.3)
proxy = _make_proxy(optimize=True)
release = threading.Event()
proxy.anthropic_pipeline = _FakePipeline([_HangingTransform(release)])
proxy.openai_pipeline = _FakePipeline([])
try:
start = time.monotonic()
await proxy.startup() # must NOT wait on the hung load
elapsed = time.monotonic() - start
# Returns shortly after the 0.3s preload timeout, far below the 30s hang.
assert elapsed < 10
finally:
release.set()
await proxy.shutdown()
async def test_startup_merges_warmup_for_normal_transforms(monkeypatch):
proxy = _make_proxy(optimize=True)
captured: list[dict] = []
monkeypatch.setattr(proxy.warmup, "merge_transform_status", captured.append)
proxy.anthropic_pipeline = _FakePipeline([_FastTransform({"kompress": "enabled"})])
proxy.openai_pipeline = _FakePipeline([])
try:
await proxy.startup()
assert {"kompress": "enabled"} in captured
assert proxy._kompress_status == "enabled"
finally:
await proxy.shutdown()
async def test_startup_reports_deferred_kompress(caplog):
proxy = _make_proxy(optimize=True)
proxy.anthropic_pipeline = _FakePipeline([_FastTransform({"kompress": "deferred"})])
proxy.openai_pipeline = _FakePipeline([])
try:
# Attach caplog directly so unrelated propagation mutations cannot
# affect this assertion.
server_mod.logger.addHandler(caplog.handler)
try:
with caplog.at_level(logging.INFO, logger=server_mod.logger.name):
await proxy.startup()
finally:
server_mod.logger.removeHandler(caplog.handler)
assert proxy._kompress_status == "deferred"
# The wording after the prefix is conditional (a background warm-up
# thread started or not), so pin the prefix and the real invariant:
# deferred must never be reported as "not installed".
assert any(message.startswith("Kompress: DEFERRED") for message in caplog.messages)
assert not any("Kompress: not installed" in message for message in caplog.messages)
finally:
await proxy.shutdown()
@pytest.mark.parametrize("optimize", [True, False])
async def test_startup_reports_disabled_kompress(caplog, monkeypatch, optimize):
proxy = _make_proxy(optimize=optimize, disable_kompress=True)
monkeypatch.setattr(proxy, "_eager_preload_transforms", lambda: ({}, []))
try:
server_mod.logger.addHandler(caplog.handler)
try:
with caplog.at_level(logging.INFO, logger=server_mod.logger.name):
await proxy.startup()
finally:
server_mod.logger.removeHandler(caplog.handler)
assert "Kompress: DISABLED" in caplog.messages
assert (
"Kompress: not installed (pip install headroom-ai[ml] for ML compression)"
not in caplog.messages
)
finally:
await proxy.shutdown()
async def test_startup_keeps_not_installed_message_when_kompress_is_missing(caplog, monkeypatch):
proxy = _make_proxy(optimize=True)
monkeypatch.setattr(proxy, "_eager_preload_transforms", lambda: ({}, []))
try:
server_mod.logger.addHandler(caplog.handler)
try:
with caplog.at_level(logging.INFO, logger=server_mod.logger.name):
await proxy.startup()
finally:
server_mod.logger.removeHandler(caplog.handler)
assert (
"Kompress: not installed (pip install headroom-ai[ml] for ML compression)"
in caplog.messages
)
finally:
await proxy.shutdown()
async def test_startup_keeps_missing_message_when_provider_override_enables_kompress(
caplog, monkeypatch
):
proxy = _make_proxy(
optimize=True,
disable_kompress=True,
disable_kompress_anthropic=False,
)
monkeypatch.setattr(proxy, "_eager_preload_transforms", lambda: ({}, []))
try:
server_mod.logger.addHandler(caplog.handler)
try:
with caplog.at_level(logging.INFO, logger=server_mod.logger.name):
await proxy.startup()
finally:
server_mod.logger.removeHandler(caplog.handler)
assert "Kompress: DISABLED" not in caplog.messages
assert (
"Kompress: not installed (pip install headroom-ai[ml] for ML compression)"
in caplog.messages
)
finally:
await proxy.shutdown()
async def test_startup_reports_disabled_when_both_providers_disable_kompress(caplog, monkeypatch):
proxy = _make_proxy(
optimize=True,
disable_kompress_anthropic=True,
disable_kompress_openai=True,
)
monkeypatch.setattr(proxy, "_eager_preload_transforms", lambda: ({}, []))
try:
server_mod.logger.addHandler(caplog.handler)
try:
with caplog.at_level(logging.INFO, logger=server_mod.logger.name):
await proxy.startup()
finally:
server_mod.logger.removeHandler(caplog.handler)
assert "Kompress: DISABLED" in caplog.messages
assert (
"Kompress: not installed (pip install headroom-ai[ml] for ML compression)"
not in caplog.messages
)
finally:
await proxy.shutdown()
async def test_startup_reports_disabled_when_compressor_selection_omits_kompress(
caplog, monkeypatch
):
proxy = _make_proxy(optimize=True, compressors={"smart_crusher"})
monkeypatch.setattr(proxy, "_eager_preload_transforms", lambda: ({}, []))
try:
server_mod.logger.addHandler(caplog.handler)
try:
with caplog.at_level(logging.INFO, logger=server_mod.logger.name):
await proxy.startup()
finally:
server_mod.logger.removeHandler(caplog.handler)
assert "Kompress: DISABLED" in caplog.messages
assert (
"Kompress: not installed (pip install headroom-ai[ml] for ML compression)"
not in caplog.messages
)
finally:
await proxy.shutdown()