Skip to content

The fastest, litest AI Gateway. Rust core with Python SDK. Call 100+ LLM APIs in OpenAI (or native) format with cost tracking, guardrails, load balancing, and logging [Bedrock, Azure, OpenAI, Anthropic, OpenAI, VertexAI, vLLM, Nvidia NIM]

16.4 KB
Raw Blame
import itertools
import threading
import uuid
from collections.abc import Iterator
from concurrent.futures import ThreadPoolExecutor
from typing import Final

import httpx
import pytest
from pydantic import BaseModel, ConfigDict, JsonValue

from tests.integration._support.client import Gateway, eventually, held, string_value
from tests.integration._support.database import read_rows

IN_FLIGHT_CALLERS_PER_PROXY: Final = 8
BLOCK_CYCLES: Final = 4
PROBES_PER_PROXY: Final = 6


class _ObservedRequest(BaseModel):
    body: dict[str, JsonValue]


class _Observations(BaseModel):
    requests: list[_ObservedRequest]


class _TeamState(BaseModel):
    team_id: str
    blocked: bool


class _TeamInfo(BaseModel):
    team_info: _TeamState


class _ProxyError(BaseModel):
    type: str


class _ErrorResponse(BaseModel):
    error: _ProxyError


class _Message(BaseModel):
    model_config = ConfigDict(frozen=True)
    content: str


class _ChatBody(BaseModel):
    model_config = ConfigDict(frozen=True)
    messages: tuple[_Message, ...]


@pytest.fixture
def upstream(gateway: Gateway) -> Iterator[httpx.Client]:
    with httpx.Client(base_url=gateway.upstream_url, timeout=15, trust_env=False) as client:
        client.delete("/__observations").raise_for_status()
        yield client


def _chat(proxy: Gateway, model: str, key: str, text: str) -> httpx.Response:
    return proxy.request(
        "POST",
        "/v1/chat/completions",
        {"model": model, "messages": [{"role": "user", "content": text}]},
        key=key,
    )


def _models(client: httpx.Client) -> tuple[str, ...]:
    response: Final = client.get("/__observations")
    assert response.status_code == 200, response.text
    observations: Final = _Observations.model_validate_json(response.text)
    return tuple(string_value(request.body["model"]) for request in observations.requests)


def _texts(client: httpx.Client) -> tuple[str, ...]:
    response: Final = client.get("/__observations")
    assert response.status_code == 200, response.text
    observations: Final = _Observations.model_validate_json(response.text)
    return tuple(
        message.content
        for message in itertools.chain.from_iterable(
            _ChatBody.model_validate(request.body).messages for request in observations.requests
        )
    )


def _keep_calling(proxy: Gateway, model: str, key: str, stop: threading.Event) -> tuple[int, ...]:
    return tuple(
        _chat(proxy, model, key, f"in flight {uuid.uuid4().hex}").status_code
        for _ in itertools.takewhile(lambda _: not stop.is_set(), itertools.count())
    )


def _statuses_on_fresh_connections(proxies: tuple[Gateway, ...], model: str, key: str, marker: str) -> tuple[int, ...]:
    return tuple(
        proxy.request(
            "POST",
            "/v1/chat/completions",
            {"model": model, "messages": [{"role": "user", "content": f"{marker} {uuid.uuid4().hex}"}]},
            key=key,
            headers={"connection": "close"},
        ).status_code
        for proxy in proxies * PROBES_PER_PROXY
    )


def _block_while_team_traffic_is_in_flight(
    gateway: Gateway, peer: Gateway, upstream: httpx.Client, model: str, key: str, team: str
) -> tuple[httpx.Response, tuple[int, ...]]:
    stop: Final = threading.Event()
    with ThreadPoolExecutor(max_workers=2 * IN_FLIGHT_CALLERS_PER_PROXY) as pool:
        callers: Final = tuple(
            pool.submit(_keep_calling, proxy, model, key, stop)
            for proxy in (gateway, peer) * IN_FLIGHT_CALLERS_PER_PROXY
        )
        try:
            eventually(
                lambda: len(_models(upstream)), lambda served: served >= 4 * IN_FLIGHT_CALLERS_PER_PROXY, seconds=30
            )
            blocked: Final = gateway.request("POST", "/team/block", {"team_id": team})
        finally:
            stop.set()
        statuses: Final = tuple(itertools.chain.from_iterable(caller.result() for caller in callers))
    return blocked, statuses


def _block_and_unblock_under_team_traffic(
    gateway: Gateway, peer: Gateway, upstream: httpx.Client, model: str, key: str, team: str, cycle: int
) -> None:
    upstream.delete("/__observations").raise_for_status()
    blocked, in_flight = _block_while_team_traffic_is_in_flight(gateway, peer, upstream, model, key, team)
    assert blocked.status_code == 200, blocked.text
    assert in_flight.count(200) >= 4 * IN_FLIGHT_CALLERS_PER_PROXY, repr(in_flight)
    assert set(in_flight) <= {200, 401}, repr(in_flight)

    marker: Final = f"after block {cycle} {uuid.uuid4().hex}"
    refused: Final = held(
        lambda: _statuses_on_fresh_connections((gateway, peer), model, key, marker),
        lambda statuses: set(statuses) == {401},
        holding=5,
        seconds=15,
    )
    assert refused == (401,) * (2 * PROBES_PER_PROXY), repr(refused)
    leaked: Final = tuple(text for text in _texts(upstream) if text.startswith(marker))
    assert leaked == (), repr(leaked)

    unblocked: Final = gateway.request("POST", "/team/unblock", {"team_id": team})
    assert unblocked.status_code == 200, unblocked.text
    eventually(
        lambda: _statuses_on_fresh_connections((gateway, peer), model, key, f"after unblock {cycle}"),
        lambda statuses: set(statuses) == {200},
        seconds=15,
    )


def test_team_block_and_unblock_update_endpoint_info_and_db_state(
    gateway: Gateway, peer: Gateway, upstream: httpx.Client
) -> None:
    with gateway.scenario() as scenario:
        provider_model: Final = f"team-block-{uuid.uuid4().hex}"
        model: Final = scenario.model(model=f"openai/{provider_model}")
        team: Final = scenario.team(models=[model])
        key: Final = scenario.key(team_id=team)

        primary_before: Final = _chat(gateway, model, key, "team block primary warm")
        assert primary_before.status_code == 200, primary_before.text
        peer_before: Final = _chat(peer, model, key, "team block peer warm")
        assert peer_before.status_code == 200, peer_before.text
        observed_before_block: Final = _models(upstream)
        assert observed_before_block == (provider_model, provider_model), repr(observed_before_block)
        upstream.delete("/__observations").raise_for_status()

        blocked_response: Final = gateway.request("POST", "/team/block", {"team_id": team})
        assert blocked_response.status_code == 200, blocked_response.text
        blocked: Final = _TeamState.model_validate_json(blocked_response.text)
        assert blocked.blocked is True, blocked_response.text

        info_response: Final = gateway.request("GET", "/team/info", params={"team_id": team})
        assert info_response.status_code == 200, info_response.text
        info: Final = _TeamInfo.model_validate_json(info_response.text)
        assert info.team_info.blocked is True, info_response.text
        assert info.team_info.team_id == team, info_response.text
        assert read_rows(
            'SELECT blocked FROM "LiteLLM_TeamTable" WHERE team_id = %s',
            (team,),
        ) == [{"blocked": True}], info_response.text

        cold_key: Final = scenario.key(team_id=team)
        cold_denied: Final = _chat(gateway, model, cold_key, "cold team key after block")
        assert cold_denied.status_code == 401, cold_denied.text
        assert _ErrorResponse.model_validate_json(cold_denied.text).error.type == "auth_error", cold_denied.text
        after_cold_denied: Final = _models(upstream)
        assert after_cold_denied == (), repr(after_cold_denied)
        upstream.delete("/__observations").raise_for_status()

        unblocked_response: Final = gateway.request("POST", "/team/unblock", {"team_id": team})
        assert unblocked_response.status_code == 200, unblocked_response.text
        restored_key: Final = scenario.key(team_id=team)
        restored: Final = _chat(gateway, model, restored_key, "new team key after unblock")
        assert restored.status_code == 200, restored.text
        after_unblock: Final = _models(upstream)
        assert after_unblock == (provider_model,), repr(after_unblock)
        unblocked: Final = _TeamState.model_validate_json(unblocked_response.text)
        assert unblocked.blocked is False, unblocked_response.text
        unblocked_info_response: Final = gateway.request("GET", "/team/info", params={"team_id": team})
        assert unblocked_info_response.status_code == 200, unblocked_info_response.text
        unblocked_info: Final = _TeamInfo.model_validate_json(unblocked_info_response.text)
        assert unblocked_info.team_info.blocked is False, unblocked_info_response.text
        assert read_rows(
            'SELECT blocked FROM "LiteLLM_TeamTable" WHERE team_id = %s',
            (team,),
        ) == [{"blocked": False}], unblocked_info_response.text

        assert observed_before_block + after_cold_denied + after_unblock == (
            provider_model,
            provider_model,
            provider_model,
        ), repr((observed_before_block, after_cold_denied, after_unblock))


def test_unblocking_a_team_restores_warmed_keys_on_both_proxies(
    gateway: Gateway, peer: Gateway, upstream: httpx.Client
) -> None:
    with gateway.scenario() as scenario:
        provider_model: Final = f"team-block-{uuid.uuid4().hex}"
        model: Final = scenario.model(model=f"openai/{provider_model}")
        team: Final = scenario.team(models=[model])
        key: Final = scenario.key(team_id=team)

        primary_before: Final = _chat(gateway, model, key, "team unblock primary warm")
        assert primary_before.status_code == 200, primary_before.text
        peer_before: Final = _chat(peer, model, key, "team unblock peer warm")
        assert peer_before.status_code == 200, peer_before.text
        observed_before_block: Final = _models(upstream)
        assert observed_before_block == (provider_model, provider_model), repr(observed_before_block)
        upstream.delete("/__observations").raise_for_status()

        blocked_response: Final = gateway.request("POST", "/team/block", {"team_id": team})
        assert blocked_response.status_code == 200, blocked_response.text
        cold_key: Final = scenario.key(team_id=team)
        cold_denied: Final = _chat(gateway, model, cold_key, "cold team key after block")
        assert cold_denied.status_code == 401, cold_denied.text
        assert _ErrorResponse.model_validate_json(cold_denied.text).error.type == "auth_error", cold_denied.text
        after_cold_denied: Final = _models(upstream)
        assert after_cold_denied == (), repr(after_cold_denied)
        upstream.delete("/__observations").raise_for_status()

        unblocked_response: Final = gateway.request("POST", "/team/unblock", {"team_id": team})
        assert unblocked_response.status_code == 200, unblocked_response.text

        primary_after: Final = _chat(gateway, model, key, "team unblock primary")
        assert primary_after.status_code == 200, primary_after.text
        peer_after: Final = _chat(peer, model, key, "team unblock peer")
        assert peer_after.status_code == 200, peer_after.text
        after_unblock: Final = _models(upstream)
        assert after_unblock == (provider_model, provider_model), repr(after_unblock)


def test_warmed_team_keys_are_refused_on_both_proxies_after_block(
    gateway: Gateway, peer: Gateway, upstream: httpx.Client
) -> None:
    with gateway.scenario() as scenario:
        provider_model: Final = f"team-block-{uuid.uuid4().hex}"
        model: Final = scenario.model(model=f"openai/{provider_model}")
        team: Final = scenario.team(models=[model])
        key: Final = scenario.key(team_id=team)

        primary_before: Final = _chat(gateway, model, key, "team block primary warm")
        assert primary_before.status_code == 200, primary_before.text
        peer_before: Final = _chat(peer, model, key, "team block peer warm")
        assert peer_before.status_code == 200, peer_before.text
        observed_before_block: Final = _models(upstream)
        assert observed_before_block == (provider_model, provider_model), repr(observed_before_block)
        upstream.delete("/__observations").raise_for_status()

        blocked_response: Final = gateway.request("POST", "/team/block", {"team_id": team})
        assert blocked_response.status_code == 200, blocked_response.text

        primary_denied: Final = _chat(gateway, model, key, "blocked team primary")
        peer_denied: Final = _chat(peer, model, key, "blocked team peer")
        after_block: Final = _models(upstream)
        assert (primary_denied.status_code, peer_denied.status_code) == (401, 401), (
            f"primary={primary_denied.text}; peer={peer_denied.text}"
        )
        assert _ErrorResponse.model_validate_json(primary_denied.text).error.type == "auth_error", primary_denied.text
        assert _ErrorResponse.model_validate_json(peer_denied.text).error.type == "auth_error", peer_denied.text
        assert after_block == (), repr(after_block)
        assert observed_before_block + after_block == (provider_model, provider_model), repr(
            (observed_before_block, after_block)
        )


def test_a_new_team_key_is_refused_on_a_peer_that_cached_the_team_before_block(
    gateway: Gateway, peer: Gateway, upstream: httpx.Client
) -> None:
    with gateway.scenario() as scenario:
        provider_model: Final = f"team-block-{uuid.uuid4().hex}"
        model: Final = scenario.model(model=f"openai/{provider_model}")
        team: Final = scenario.team(models=[model])
        warm_key: Final = scenario.key(team_id=team)

        peer_before: Final = _chat(peer, model, warm_key, "team block peer warm")
        assert peer_before.status_code == 200, peer_before.text
        observed_before_block: Final = _models(upstream)
        assert observed_before_block == (provider_model,), repr(observed_before_block)
        upstream.delete("/__observations").raise_for_status()

        blocked_response: Final = gateway.request("POST", "/team/block", {"team_id": team})
        assert blocked_response.status_code == 200, blocked_response.text
        assert read_rows(
            'SELECT blocked FROM "LiteLLM_TeamTable" WHERE team_id = %s',
            (team,),
        ) == [{"blocked": True}], blocked_response.text

        cold_key: Final = scenario.key(team_id=team)
        peer_denied: Final = _chat(peer, model, cold_key, "new team key on peer after block")
        after_block: Final = _models(upstream)
        assert peer_denied.status_code == 401, peer_denied.text
        assert _ErrorResponse.model_validate_json(peer_denied.text).error.type == "auth_error", peer_denied.text
        assert after_block == (), repr(after_block)


def test_a_team_model_alias_still_routes_on_both_proxies_after_block_and_unblock(
    gateway: Gateway, peer: Gateway, upstream: httpx.Client
) -> None:
    with gateway.scenario() as scenario:
        provider_model: Final = f"team-block-{uuid.uuid4().hex}"
        model: Final = scenario.model(model=f"openai/{provider_model}")
        alias: Final = f"alias-{uuid.uuid4().hex}"
        team: Final = scenario.team(models=[model], model_aliases={alias: model})
        key: Final = scenario.key(team_id=team, models=[model])

        warmed: Final = tuple(
            _chat(proxy, alias, key, f"team alias warm {uuid.uuid4().hex}").status_code for proxy in (gateway, peer)
        )
        assert warmed == (200, 200), repr(warmed)
        assert _models(upstream) == (provider_model, provider_model)
        upstream.delete("/__observations").raise_for_status()

        blocked: Final = gateway.request("POST", "/team/block", {"team_id": team})
        assert blocked.status_code == 200, blocked.text
        unblocked: Final = gateway.request("POST", "/team/unblock", {"team_id": team})
        assert unblocked.status_code == 200, unblocked.text

        after_unblock: Final = _statuses_on_fresh_connections((gateway, peer), alias, key, "team alias after unblock")
        assert after_unblock == (200,) * (2 * PROBES_PER_PROXY), repr(after_unblock)
        assert _models(upstream) == (provider_model,) * (2 * PROBES_PER_PROXY)


def test_a_block_issued_while_team_traffic_is_in_flight_holds_on_every_worker(
    gateway: Gateway, peer: Gateway, upstream: httpx.Client
) -> None:
    with gateway.scenario() as scenario:
        provider_model: Final = f"team-block-{uuid.uuid4().hex}"
        model: Final = scenario.model(
            model=f"openai/{provider_model}", input_cost_per_token=0.000001, output_cost_per_token=0.000002
        )
        team: Final = scenario.team(models=[model])
        key: Final = scenario.key(team_id=team)

        for cycle in range(BLOCK_CYCLES):
            _block_and_unblock_under_team_traffic(gateway, peer, upstream, model, key, team, cycle)