Skip to content

The fastest, litest AI Gateway. Rust core with Python SDK. Call 100+ LLM APIs in OpenAI (or native) format with cost tracking, guardrails, load balancing, and logging [Bedrock, Azure, OpenAI, Anthropic, OpenAI, VertexAI, vLLM, Nvidia NIM]

18.3 KB
Raw Blame
import asyncio
import json
import threading
import uuid
from collections.abc import Callable, Mapping, Sequence
from queue import SimpleQueue
from typing import Final

import httpx
import pytest
from integration._support.client import Gateway, Scenario, eventually, object_value, string_value
from integration._support.database import read_rows
from integration._support.openai_wire import answering_model_discovery, chat_reply
from integration._support.wire import Reply, Request, Wire, wire_server
from pydantic import JsonValue

from litellm.router_strategy.complexity_router.config import DEFAULT_JEV_INSTRUCTIONS

_MODEL: Final = "ai_decide"
_ROUTE: Final = "/api/2.0/ai-functions/ai-decide"
_ENDPOINT: Final = "databricks-openjev-qwen35-4b"
_SECOND_WORKSPACE: Final = "/second-workspace"
_API_KEY: Final = "synthetic-databricks-key"
_MODEL_RULE: Final = (
    "Databricks decisions model must be 'ai_decide' (the ai_decide AI Function) or a bare serving endpoint name "
    "(letters, digits, '-', '_' and '.', with no '/', '?', '#', spaces, or a leading '.'), e.g. "
    "databricks-openjev-qwen35-4b"
)
_REJECTED_AT_LOAD: Final = "The router would drop this deployment at load time, so the write is rejected instead."
_MODEL_REQUIRED: Final = (
    "opensource_classifier_config.model is required for provider 'databricks': 'ai_decide' or a serving endpoint "
    "name, e.g. databricks-openjev-qwen35-4b"
)
_BASE_NEEDS_KEY: Final = (
    "opensource_classifier_config.api_base requires opensource_classifier_config.api_key: DATABRICKS_API_KEY or "
    "DATABRICKS_TOKEN is only sent to DATABRICKS_API_BASE"
)
_BLANK_KEY: Final = (
    "opensource_classifier_config.api_key must be non-empty; omit it to use the provider environment key"
)
_ANSWERS: Final[dict[str, JsonValue]] = {
    "tier": {
        "type": "choice",
        "choice": "COMPLEX",
        "confidence": 0.91,
        "probabilities": {"SIMPLE": 0.03, "MEDIUM": 0.06, "COMPLEX": 0.91},
    }
}
_AI_DECIDE_ANSWER: Final[dict[str, JsonValue]] = {"response": {"answers": _ANSWERS}, "metadata": {"version": "1.0"}}
_ENDPOINT_ANSWER: Final[dict[str, JsonValue]] = {
    "model": "/mosaicml/local_model",
    "answers": _ANSWERS,
    "usage": {"input_tokens": 12, "output_tokens": 1},
}
_TIER_CRITERIA: Final[dict[str, JsonValue]] = {
    "SIMPLE": "Greetings, chitchat, or short factual lookups with known answers",
    "MEDIUM": "Everyday requests needing explanation, light reasoning, or minor technical work",
    "COMPLEX": "Non-trivial code, architecture, multi-step work, or specialized domain depth",
    "REASONING": "Open-ended analysis, proofs, tradeoffs, or tasks requiring careful thought",
}
_ROWS_QUERY: Final = (
    "SELECT request_id, status, metadata->>'internal_call_origin' AS origin "
    'FROM "LiteLLM_SpendLogs" WHERE model_group = %s'
)


def _classifier(request: Request) -> Reply:
    answer: Final = _ENDPOINT_ANSWER if request.target.endswith("/invocations") else _AI_DECIDE_ANSWER
    return Reply(body=json.dumps(answer).encode())


def _tier(text: str) -> Callable[[Request], Reply]:
    def respond(request: Request) -> Reply:
        body: Final = json.loads(request.body)
        return chat_reply(f"{text}-{uuid.uuid4().hex[:8]}", "tier-model", text, stream=bool(body.get("stream")))

    return answering_model_discovery(respond)


def _classifier_config(api_base: str, **overrides: JsonValue) -> dict[str, JsonValue]:
    return {
        "provider": "databricks",
        "model": _MODEL,
        "api_base": api_base,
        "api_key": _API_KEY,
        "timeout_ms": 20000,
        "circuit_breaker_enabled": False,
        **overrides,
    }


def _without(config: Mapping[str, JsonValue], field: str) -> dict[str, JsonValue]:
    return {name: value for name, value in config.items() if name != field}


def _router_config(classifier: Mapping[str, JsonValue], simple: str, complex_: str) -> dict[str, JsonValue]:
    return {
        "classifier_type": "oss_classifier",
        "opensource_classifier_config": dict(classifier),
        "tiers": {"SIMPLE": simple, "MEDIUM": simple, "COMPLEX": complex_, "REASONING": complex_},
    }


def _validate(gateway: Gateway, config: Mapping[str, JsonValue]) -> dict[str, JsonValue]:
    response: Final = gateway.request(
        "POST", "/auto_router/validate_complexity_router_config", {"complexity_router_config": dict(config)}
    )
    assert response.status_code == 200, response.text
    return object_value(response.json())


def _create_router(scenario: Scenario, config: Mapping[str, JsonValue]) -> tuple[str, str]:
    name: Final = f"router-{uuid.uuid4().hex[:10]}"
    created: Final = scenario.gateway.post(
        "/model/new",
        {
            "model_name": name,
            "litellm_params": {"model": "auto_router/complexity_router", "complexity_router_config": dict(config)},
            "model_info": {},
        },
    )
    identity: Final = string_value(object_value(created["model_info"])["id"])
    scenario.cleanups.callback(scenario.delete_model, identity)
    return name, identity


def _stored_classifier(gateway: Gateway, identity: str) -> dict[str, JsonValue]:
    stored: Final = gateway.request("GET", "/model/info", params={"litellm_model_id": identity})
    assert stored.status_code == 200, stored.text
    (entry,) = stored.json()["data"]
    router_config: Final = object_value(object_value(object_value(entry)["litellm_params"])["complexity_router_config"])
    return object_value(router_config["opensource_classifier_config"])


def _judge_targets(judge: Wire) -> tuple[str, ...]:
    return tuple(request.target for request in judge.drain() if request.method == "POST")


def _chat_body(model: str) -> dict[str, JsonValue]:
    return {"model": model, "messages": [{"role": "user", "content": f"design a distributed cache {uuid.uuid4().hex}"}]}


async def _burst(base_url: str, key: str, model: str, count: int) -> tuple[httpx.Response, ...]:
    async with httpx.AsyncClient(
        base_url=base_url, timeout=60, trust_env=False, headers={"Authorization": f"Bearer {key}"}
    ) as client:
        return tuple(
            await asyncio.gather(*(client.post("/v1/chat/completions", json=_chat_body(model)) for _ in range(count)))
        )


def _request_rows(rows: Sequence[dict[str, JsonValue]]) -> tuple[dict[str, JsonValue], ...]:
    return tuple(row for row in rows if row["origin"] != "autorouter_classifier")


def _rows(router: str, *, want: frozenset[str]) -> tuple[dict[str, JsonValue], ...]:
    return tuple(
        eventually(
            lambda: read_rows(_ROWS_QUERY, (router,)),
            lambda found: want <= frozenset(string_value(row["request_id"]) for row in _request_rows(found)),
            seconds=70,
        )
    )


def test_validate_accepts_a_databricks_ai_decide_classifier_with_or_without_the_serving_endpoints_suffix(
    gateway: Gateway,
) -> None:
    for api_base in (
        "https://dbc-example.cloud.databricks.com",
        "https://dbc-example.cloud.databricks.com/serving-endpoints",
    ):
        config: Final = _router_config(_classifier_config(api_base), "simple", "complex")
        assert _validate(gateway, config) == {"valid": True, "error": None}, api_base


def test_validate_accepts_a_databricks_classifier_with_a_bare_endpoint_name_of_any_length(gateway: Gateway) -> None:
    for model in (_ENDPOINT, "e" * 5000):
        config: Final = _router_config(
            _classifier_config("https://dbc-example.cloud.databricks.com/serving-endpoints", model=model),
            "simple",
            "complex",
        )
        assert _validate(gateway, config) == {"valid": True, "error": None}, model[:40]


def test_validate_names_each_databricks_classifier_misconfiguration(gateway: Gateway) -> None:
    base: Final = _classifier_config("https://dbc-example.cloud.databricks.com/serving-endpoints")
    cases: Final[tuple[tuple[str, dict[str, JsonValue], str], ...]] = (
        ("model omitted", _without(base, "model"), f"at opensource_classifier_config: Value error, {_MODEL_REQUIRED}"),
        (
            "api_base without api_key",
            _without(base, "api_key"),
            f"at opensource_classifier_config: Value error, {_BASE_NEEDS_KEY}",
        ),
        ("slash in the name", {**base, "model": "a/b"}, _MODEL_RULE),
        ("space in the name", {**base, "model": "a b"}, _MODEL_RULE),
        ("leading dot", {**base, "model": ".hidden"}, _MODEL_RULE),
        ("empty name", {**base, "model": ""}, _MODEL_RULE),
        (
            "blank api_key",
            {**base, "api_key": ""},
            f"at opensource_classifier_config.api_key: Value error, {_BLANK_KEY}",
        ),
        ("int name", {**base, "model": 5}, "at opensource_classifier_config.model: Input should be a valid string"),
        (
            "list name",
            {**base, "model": ["a"]},
            "at opensource_classifier_config.model: Input should be a valid string",
        ),
        ("null name", {**base, "model": None}, "at opensource_classifier_config.model: Input should be a valid string"),
    )
    for label, classifier, expected in cases:
        verdict: Final = _validate(gateway, _router_config(classifier, "simple", "complex"))
        assert verdict["valid"] is False, (label, verdict)
        error: Final = string_value(verdict["error"])
        assert error.startswith("complexity_router_config is invalid "), (label, error)
        assert expected in error, (label, error)
        assert error.endswith(_REJECTED_AT_LOAD), (label, error)


@pytest.mark.parametrize(
    ("model", "target", "cost"),
    [(_MODEL, _ROUTE, {}), (_ENDPOINT, f"/{_ENDPOINT}/invocations", {"classifier_cost": 0.0})],
    ids=["ai_decide", "serving_endpoint"],
)
def test_test_routing_classifies_through_databricks_without_calling_the_tier_it_picked(
    gateway: Gateway, model: str, target: str, cost: dict[str, JsonValue]
) -> None:
    prompt: Final = f"design a distributed cache {uuid.uuid4().hex}"
    with (
        wire_server(_classifier) as judge,
        wire_server(_tier("simple answer")) as simple,
        wire_server(_tier("complex answer")) as complex_tier,
        gateway.scenario() as scenario,
    ):
        simple_model: Final = scenario.model(model="openai/simple-tier", api_base=simple.url)
        complex_model: Final = scenario.model(model="openai/complex-tier", api_base=complex_tier.url)
        config: Final = _router_config(_classifier_config(judge.url, model=model), simple_model, complex_model)
        response: Final = gateway.request(
            "POST", "/auto_router/test_routing", {"prompt": prompt, "complexity_router_config": config}
        )
        assert response.status_code == 200, response.text
        assert response.json() == {
            "routed_model": complex_model,
            "routed_model_configured": True,
            "routing_decision": {
                "router_model_name": "auto_router_routing_test",
                "router_type": "complexity",
                "routed_model": complex_model,
                "cause": "jev_classifier",
                "tier": "COMPLEX",
                "signals": [
                    "databricks-classifier:COMPLEX",
                    "databricks-confidence=0.910000",
                    "tier-probability:SIMPLE=0.030000",
                    "tier-probability:MEDIUM=0.060000",
                    "tier-probability:COMPLEX=0.910000",
                ],
                "classifier_model": f"databricks/{model}",
                **cost,
                "classifier_probabilities": {"SIMPLE": 0.03, "MEDIUM": 0.06, "COMPLEX": 0.91},
                "classifier_confidence": 0.91,
                "conversation_continuing": False,
            },
        }, response.text
        (call,) = [request for request in judge.drain() if request.method == "POST"]
        assert call.target == target, call.target
        assert call.headers.get("authorization") == f"Bearer {_API_KEY}", call.headers
        assert json.loads(call.body) == {
            "state": f"\nClassify this message:\n{prompt}",
            **({} if model == _MODEL else {"model": model}),
            "questions": {
                "tier": {"type": "choice", "instructions": DEFAULT_JEV_INSTRUCTIONS, "criteria": _TIER_CRITERIA}
            },
        }, call.body
        assert _judge_targets(simple) == () and _judge_targets(complex_tier) == ()


def test_model_new_refuses_a_databricks_classifier_without_a_model_and_stores_nothing(
    gateway: Gateway,
) -> None:
    name: Final = f"router-{uuid.uuid4().hex[:10]}"
    config: Final = _router_config(
        _without(_classifier_config("https://dbc-example.cloud.databricks.com/serving-endpoints"), "model"),
        "simple",
        "complex",
    )
    response: Final = gateway.request(
        "POST",
        "/model/new",
        {
            "model_name": name,
            "litellm_params": {"model": "auto_router/complexity_router", "complexity_router_config": config},
            "model_info": {},
        },
    )
    assert response.status_code == 400, response.text
    error: Final = object_value(object_value(response.json())["error"])
    assert _MODEL_REQUIRED in string_value(error["message"]), error
    assert (error["type"], error["param"]) == ("validation_error", "litellm_params.model"), error
    listed: Final = gateway.get("/model/info")["data"]
    assert isinstance(listed, list)
    assert [entry for entry in map(object_value, listed) if entry["model_name"] == name] == []


def test_model_update_refuses_dropping_the_model_and_keeps_the_stored_classifier_serving(
    gateway: Gateway,
) -> None:
    with (
        wire_server(_classifier) as judge,
        wire_server(_tier("simple answer")) as simple,
        wire_server(_tier("complex answer")) as complex_tier,
        gateway.scenario() as scenario,
    ):
        simple_model: Final = scenario.model(model="openai/simple-tier", api_base=simple.url)
        complex_model: Final = scenario.model(model="openai/complex-tier", api_base=complex_tier.url)
        classifier: Final = _classifier_config(judge.url)
        name, identity = _create_router(scenario, _router_config(classifier, simple_model, complex_model))
        dropped: Final = _router_config(_without(classifier, "model"), simple_model, complex_model)
        response: Final = gateway.request(
            "PATCH",
            f"/model/{identity}/update",
            {"litellm_params": {"model": "auto_router/complexity_router", "complexity_router_config": dropped}},
        )
        assert response.status_code == 400, response.text
        assert _MODEL_REQUIRED in string_value(object_value(object_value(response.json())["error"])["message"]), (
            response.text
        )
        assert _stored_classifier(gateway, identity)["model"] == _MODEL
        chat: Final = gateway.request("POST", "/v1/chat/completions", _chat_body(name))
        assert chat.status_code == 200, chat.text
        assert chat.json()["choices"][0]["message"]["content"] == "complex answer", chat.text
        assert chat.headers["x-litellm-complexity-router-cause"] == "jev_classifier", dict(chat.headers)
        assert _judge_targets(judge) == (_ROUTE,)


async def test_moving_the_workspace_under_a_burst_moves_the_classifier_and_logs_every_request_once(
    gateway: Gateway,
) -> None:
    base_url: Final = str(gateway.client.base_url)
    loop: Final = asyncio.get_running_loop()
    classifying: Final = asyncio.Event()
    release: Final = threading.Event()

    def classify(request: Request) -> Reply:
        loop.call_soon_threadsafe(classifying.set)
        assert release.wait(timeout=30), "The in-flight classifier calls were never released"
        return _classifier(request)

    with (
        wire_server(classify) as judge,
        wire_server(_tier("simple answer")) as simple,
        wire_server(_tier("complex answer")) as complex_tier,
        gateway.scenario() as scenario,
    ):
        simple_model: Final = scenario.model(model="openai/simple-tier", api_base=simple.url)
        complex_model: Final = scenario.model(model="openai/complex-tier", api_base=complex_tier.url)
        classifier: Final = _classifier_config(judge.url)
        name, identity = _create_router(scenario, _router_config(classifier, simple_model, complex_model))
        moved: Final = _router_config(
            {**classifier, "api_base": f"{judge.url}{_SECOND_WORKSPACE}"}, simple_model, complex_model
        )
        async with asyncio.TaskGroup() as requests:
            burst: Final = requests.create_task(_burst(base_url, gateway.key, name, 24))
            try:
                await classifying.wait()
                patched: Final = gateway.request(
                    "PATCH",
                    f"/model/{identity}/update",
                    {"litellm_params": {"model": "auto_router/complexity_router", "complexity_router_config": moved}},
                )
                assert patched.status_code == 200, patched.text
            finally:
                release.set()
            during: Final = await burst
        assert [response.status_code for response in during] == [200] * 24, [response.text for response in during]
        assert set(_judge_targets(judge)) <= {_ROUTE, f"{_SECOND_WORKSPACE}{_ROUTE}"}
        assert _stored_classifier(gateway, identity)["api_base"] == f"{judge.url}{_SECOND_WORKSPACE}"

        convergence_calls: Final[SimpleQueue[str]] = SimpleQueue()

        def classifier_target() -> tuple[str, ...]:
            chat: Final = gateway.request("POST", "/v1/chat/completions", _chat_body(name))
            assert chat.status_code == 200, chat.text
            convergence_calls.put(string_value(chat.json()["id"]))
            return _judge_targets(judge)

        eventually(classifier_target, lambda targets: targets == (f"{_SECOND_WORKSPACE}{_ROUTE}",), seconds=30)
        after: Final = await _burst(base_url, gateway.key, name, 6)
        assert [response.status_code for response in after] == [200] * 6, [response.text for response in after]
        assert set(_judge_targets(judge)) == {f"{_SECOND_WORKSPACE}{_ROUTE}"}
        identities: Final = (
            *tuple(string_value(response.json()["id"]) for response in (*during, *after)),
            *tuple(convergence_calls.get_nowait() for _ in range(convergence_calls.qsize())),
        )
        assert len(set(identities)) == len(identities), identities
        rows: Final = _rows(name, want=frozenset(identities))
        request_rows: Final = _request_rows(rows)
        assert sorted(string_value(row["request_id"]) for row in request_rows) == sorted(identities), request_rows
        assert all(row["status"] == "success" for row in rows), rows