Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

9.9 KB
Raw Blame
"""Startup must bind its port even when eager preload hangs (#790).

``HeadroomProxy.startup()`` runs inside the ASGI lifespan, which completes
*before* uvicorn binds the socket. The eager compressor/parser preload used to
run synchronously there, so a hang or an uncatchable native stall during a model
load (observed on Windows) left the proxy "never opening its port". The preload
now runs off the event loop under ``asyncio.wait_for`` with
``EAGER_PRELOAD_TIMEOUT_SECONDS``; on timeout startup logs and continues so the
bind still happens and transforms fall back to lazy loading.
"""

from __future__ import annotations

import logging
import threading
import time

import pytest

pytest.importorskip("fastapi")

import headroom.proxy.server as server_mod
from headroom.proxy.server import ProxyConfig, create_app


def _make_proxy(
    *,
    optimize: bool,
    disable_kompress: bool = False,
    disable_kompress_anthropic: bool | None = None,
    disable_kompress_openai: bool | None = None,
    compressors: set[str] | None = None,
):
    config = ProxyConfig(
        optimize=optimize,
        disable_kompress=disable_kompress,
        disable_kompress_anthropic=disable_kompress_anthropic,
        disable_kompress_openai=disable_kompress_openai,
        compressors=compressors,
        cache_enabled=False,
        rate_limit_enabled=False,
        cost_tracking_enabled=False,
        log_requests=False,
        ccr_inject_tool=False,
        ccr_handle_responses=False,
        ccr_context_tracking=False,
        image_optimize=False,
        subscription_tracking_enabled=False,
    )
    proxy = create_app(config).state.proxy
    # Every test here substitutes fake pipelines to control exactly what the
    # preload walks. The proxy also eagerly builds the default /v1/compress
    # pipeline (a derived ContentRouter, warmed alongside the request
    # pipelines), which would inject real transform statuses into those
    # assertions — drop it so the fakes remain the only input.
    proxy._compress_pipeline_cache = {}
    return proxy


class _FastTransform:
    def __init__(self, status):
        self._status = status

    def eager_load_compressors(self):
        return self._status


class _RaisingTransform:
    def eager_load_compressors(self):
        raise RuntimeError("boom")


class _NonDictTransform:
    def eager_load_compressors(self):
        return "not-a-dict"


class _HangingTransform:
    """Simulates a model load that hangs forever (released via the event)."""

    def __init__(self, release: threading.Event):
        self._release = release

    def eager_load_compressors(self):
        # Safety cap so a misbehaving test can never wedge the suite.
        self._release.wait(timeout=30)
        return {"hang": "done"}


class _FakePipeline:
    def __init__(self, transforms):
        self.transforms = transforms


def test_eager_preload_dedupes_and_swallows_failures():
    proxy = _make_proxy(optimize=False)
    shared = _FastTransform({"shared": "enabled"})
    proxy.anthropic_pipeline = _FakePipeline([shared, _FastTransform({"kompress": "enabled"})])
    # ``shared`` appears in both pipelines and must load exactly once; the
    # raising and non-dict transforms must be skipped without aborting.
    proxy.openai_pipeline = _FakePipeline([shared, _RaisingTransform(), _NonDictTransform()])

    eager_status, statuses = proxy._eager_preload_transforms()

    # Keys the preload contributes itself rather than collecting from a
    # transform, so this assertion stays about dedupe/swallowing.
    non_transform_keys = {"litellm"}
    assert {k: v for k, v in eager_status.items() if k not in non_transform_keys} == {
        "shared": "enabled",
        "kompress": "enabled",
    }
    assert statuses == [{"shared": "enabled"}, {"kompress": "enabled"}]
    assert eager_status["litellm"] in {"ready", "not installed", "skipped"}


async def test_startup_binds_despite_hung_preload(monkeypatch):
    monkeypatch.setattr(server_mod, "EAGER_PRELOAD_TIMEOUT_SECONDS", 0.3)
    proxy = _make_proxy(optimize=True)
    release = threading.Event()
    proxy.anthropic_pipeline = _FakePipeline([_HangingTransform(release)])
    proxy.openai_pipeline = _FakePipeline([])

    try:
        start = time.monotonic()
        await proxy.startup()  # must NOT wait on the hung load
        elapsed = time.monotonic() - start
        # Returns shortly after the 0.3s preload timeout, far below the 30s hang.
        assert elapsed < 10
    finally:
        release.set()
        await proxy.shutdown()


async def test_startup_merges_warmup_for_normal_transforms(monkeypatch):
    proxy = _make_proxy(optimize=True)
    captured: list[dict] = []
    monkeypatch.setattr(proxy.warmup, "merge_transform_status", captured.append)
    proxy.anthropic_pipeline = _FakePipeline([_FastTransform({"kompress": "enabled"})])
    proxy.openai_pipeline = _FakePipeline([])

    try:
        await proxy.startup()
        assert {"kompress": "enabled"} in captured
        assert proxy._kompress_status == "enabled"
    finally:
        await proxy.shutdown()


async def test_startup_reports_deferred_kompress(caplog):
    proxy = _make_proxy(optimize=True)
    proxy.anthropic_pipeline = _FakePipeline([_FastTransform({"kompress": "deferred"})])
    proxy.openai_pipeline = _FakePipeline([])

    try:
        # Attach caplog directly so unrelated propagation mutations cannot
        # affect this assertion.
        server_mod.logger.addHandler(caplog.handler)
        try:
            with caplog.at_level(logging.INFO, logger=server_mod.logger.name):
                await proxy.startup()
        finally:
            server_mod.logger.removeHandler(caplog.handler)

        assert proxy._kompress_status == "deferred"
        # The wording after the prefix is conditional (a background warm-up
        # thread started or not), so pin the prefix and the real invariant:
        # deferred must never be reported as "not installed".
        assert any(message.startswith("Kompress: DEFERRED") for message in caplog.messages)
        assert not any("Kompress: not installed" in message for message in caplog.messages)
    finally:
        await proxy.shutdown()


@pytest.mark.parametrize("optimize", [True, False])
async def test_startup_reports_disabled_kompress(caplog, monkeypatch, optimize):
    proxy = _make_proxy(optimize=optimize, disable_kompress=True)
    monkeypatch.setattr(proxy, "_eager_preload_transforms", lambda: ({}, []))

    try:
        server_mod.logger.addHandler(caplog.handler)
        try:
            with caplog.at_level(logging.INFO, logger=server_mod.logger.name):
                await proxy.startup()
        finally:
            server_mod.logger.removeHandler(caplog.handler)

        assert "Kompress: DISABLED" in caplog.messages
        assert (
            "Kompress: not installed (pip install headroom-ai[ml] for ML compression)"
            not in caplog.messages
        )
    finally:
        await proxy.shutdown()


async def test_startup_keeps_not_installed_message_when_kompress_is_missing(caplog, monkeypatch):
    proxy = _make_proxy(optimize=True)
    monkeypatch.setattr(proxy, "_eager_preload_transforms", lambda: ({}, []))

    try:
        server_mod.logger.addHandler(caplog.handler)
        try:
            with caplog.at_level(logging.INFO, logger=server_mod.logger.name):
                await proxy.startup()
        finally:
            server_mod.logger.removeHandler(caplog.handler)

        assert (
            "Kompress: not installed (pip install headroom-ai[ml] for ML compression)"
            in caplog.messages
        )
    finally:
        await proxy.shutdown()


async def test_startup_keeps_missing_message_when_provider_override_enables_kompress(
    caplog, monkeypatch
):
    proxy = _make_proxy(
        optimize=True,
        disable_kompress=True,
        disable_kompress_anthropic=False,
    )
    monkeypatch.setattr(proxy, "_eager_preload_transforms", lambda: ({}, []))

    try:
        server_mod.logger.addHandler(caplog.handler)
        try:
            with caplog.at_level(logging.INFO, logger=server_mod.logger.name):
                await proxy.startup()
        finally:
            server_mod.logger.removeHandler(caplog.handler)

        assert "Kompress: DISABLED" not in caplog.messages
        assert (
            "Kompress: not installed (pip install headroom-ai[ml] for ML compression)"
            in caplog.messages
        )
    finally:
        await proxy.shutdown()


async def test_startup_reports_disabled_when_both_providers_disable_kompress(caplog, monkeypatch):
    proxy = _make_proxy(
        optimize=True,
        disable_kompress_anthropic=True,
        disable_kompress_openai=True,
    )
    monkeypatch.setattr(proxy, "_eager_preload_transforms", lambda: ({}, []))

    try:
        server_mod.logger.addHandler(caplog.handler)
        try:
            with caplog.at_level(logging.INFO, logger=server_mod.logger.name):
                await proxy.startup()
        finally:
            server_mod.logger.removeHandler(caplog.handler)

        assert "Kompress: DISABLED" in caplog.messages
        assert (
            "Kompress: not installed (pip install headroom-ai[ml] for ML compression)"
            not in caplog.messages
        )
    finally:
        await proxy.shutdown()


async def test_startup_reports_disabled_when_compressor_selection_omits_kompress(
    caplog, monkeypatch
):
    proxy = _make_proxy(optimize=True, compressors={"smart_crusher"})
    monkeypatch.setattr(proxy, "_eager_preload_transforms", lambda: ({}, []))

    try:
        server_mod.logger.addHandler(caplog.handler)
        try:
            with caplog.at_level(logging.INFO, logger=server_mod.logger.name):
                await proxy.startup()
        finally:
            server_mod.logger.removeHandler(caplog.handler)

        assert "Kompress: DISABLED" in caplog.messages
        assert (
            "Kompress: not installed (pip install headroom-ai[ml] for ML compression)"
            not in caplog.messages
        )
    finally:
        await proxy.shutdown()