Skip to content

The fastest, litest AI Gateway. Rust core with Python SDK. Call 100+ LLM APIs in OpenAI (or native) format with cost tracking, guardrails, load balancing, and logging [Bedrock, Azure, OpenAI, Anthropic, OpenAI, VertexAI, vLLM, Nvidia NIM]

39.0 KB
Raw Blame
import asyncio
import hashlib
import itertools
import json
import re
import threading
import time
import uuid
from collections.abc import Iterator, Mapping, Sequence
from concurrent.futures import ThreadPoolExecutor
from dataclasses import dataclass
from pathlib import Path
from queue import SimpleQueue
from types import MappingProxyType
from typing import Final, Literal, TypeAlias

import anthropic
import httpx
import openai
import pytest
import yaml
from integration._support import device_login as dl
from integration._support import responses_vendor as rv
from integration._support.client import eventually, gateway_from_environment
from integration._support.database import read_rows
from integration._support.process import OwnedProxy, owned_proxy_process
from integration._support.timing import assert_cut_short, timed
from integration._support.wire import Request
from litellm.llms.chatgpt.authenticator import DEVICE_CODE_COOLDOWN_SECONDS
from pydantic import JsonValue

pytestmark: Final = [pytest.mark.timeout(300), pytest.mark.shares_worker_state]

Provider: TypeAlias = Literal["chatgpt", "copilot"]
Endpoint: TypeAlias = Literal["chat", "responses", "messages"]
Client: TypeAlias = Literal["sdk", "async_sdk", "httpx"]

_PROVIDERS: Final[tuple[Provider, ...]] = ("chatgpt", "copilot")
_ENDPOINTS: Final[tuple[Endpoint, ...]] = ("chat", "responses", "messages")
_CLIENTS: Final[tuple[Client, ...]] = ("sdk", "async_sdk", "httpx")
_MODELS: Final[Mapping[tuple[Provider, Endpoint], str]] = MappingProxyType(
    {
        ("chatgpt", "chat"): "chatgpt/gpt-5.6-terra",
        ("chatgpt", "responses"): "chatgpt/gpt-5.6-terra",
        ("chatgpt", "messages"): "chatgpt/gpt-5.6-terra",
        ("copilot", "chat"): "github_copilot/gpt-5.4",
        ("copilot", "responses"): "github_copilot/gpt-5.4",
        ("copilot", "messages"): "github_copilot/claude-sonnet-5.5",
    }
)
_REFUSALS: Final[Mapping[Provider, str]] = MappingProxyType(
    {"chatgpt": dl.CHATGPT_REFUSAL, "copilot": dl.COPILOT_REFUSAL}
)
_STORED_BEARERS: Final[Mapping[Provider, str]] = MappingProxyType(
    {"chatgpt": dl.CHATGPT_STORED, "copilot": dl.COPILOT_STORED_KEY}
)
_CONTROL_MODEL: Final = "device-login-control"
_EXTRA: Final[Mapping[str, JsonValue]] = MappingProxyType({"num_retries": 0, "cache": {"no-cache": True}})
_CALL_ID: Final = "x-litellm-call-id"
_CLIENT_SECONDS: Final = 60


@dataclass(frozen=True, slots=True)
class _Cell:
    provider: Provider
    endpoint: Endpoint
    stream: bool
    client: Client

    def name(self) -> str:
        return f"{self.provider}-{self.endpoint}-{'stream' if self.stream else 'unary'}-{self.client}"


_CELLS: Final = tuple(
    _Cell(provider, endpoint, stream, client)
    for provider, endpoint, stream, client in itertools.product(_PROVIDERS, _ENDPOINTS, (False, True), _CLIENTS)
)


@dataclass(frozen=True, slots=True)
class _Call:
    base_url: str
    key: str
    model: str
    endpoint: Endpoint
    stream: bool
    client: Client
    marker: str
    served: SimpleQueue[str]

    def prompt(self) -> str:
        return f"Reply to marker-{self.marker}"


@dataclass(frozen=True, slots=True)
class _Served:
    status: int
    text: str
    call_id: str


@dataclass(frozen=True, slots=True)
class _Rig:
    owned: OwnedProxy
    peers: dl.Peers
    chatgpt_dir: Path
    copilot_dir: Path

    def base_url(self) -> str:
        return str(self.owned.gateway.client.base_url).rstrip("/")


@dataclass(frozen=True, slots=True)
class _Scene:
    rig: _Rig
    key: str
    log_offset: int
    served: SimpleQueue[str]

    def call(self, cell: _Cell) -> _Call:
        return _Call(
            self.rig.base_url(),
            self.key,
            _MODELS[cell.provider, cell.endpoint],
            cell.endpoint,
            cell.stream,
            cell.client,
            uuid.uuid4().hex,
            self.served,
        )

    def control(self, stream: bool) -> _Call:
        return _Call(
            self.rig.base_url(), self.key, _CONTROL_MODEL, "chat", stream, "httpx", uuid.uuid4().hex, self.served
        )

    def post(self, path: str, body: Mapping[str, JsonValue], *, as_admin: bool = False) -> _Served:
        key: Final = self.rig.owned.gateway.key if as_admin else self.key
        with httpx.Client(base_url=self.rig.base_url(), timeout=_CLIENT_SECONDS, trust_env=False) as client:
            response: Final = client.post(
                path, json=dict(body), headers={"Authorization": f"Bearer {key}", "anthropic-version": "2023-06-01"}
            )
        return _Served(response.status_code, response.text, response.headers.get(_CALL_ID, ""))

    def token_files(self) -> tuple[str, ...]:
        return tuple(sorted(path.name for path in (*self.rig.chatgpt_dir.iterdir(), *self.rig.copilot_dir.iterdir())))

    def log_tail(self) -> str:
        with self.rig.owned.log.open("rb") as log:
            log.seek(self.log_offset)
            return log.read().decode(errors="replace")

    def store_valid_token(self, provider: Provider) -> None:
        if provider == "chatgpt":
            dl.write_chatgpt(self.rig.chatgpt_dir, dl.chatgpt_record(dl.CHATGPT_STORED))
            return
        dl.write_copilot_key(self.rig.copilot_dir, dl.COPILOT_STORED_KEY, self.rig.peers.api.url)


def _config(peers: dl.Peers, directory: Path) -> Path:
    config: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text())
    config["model_list"] = [
        {"model_name": "*", "litellm_params": {"model": "*"}},
        {
            "model_name": _CONTROL_MODEL,
            "litellm_params": {
                "model": "openai/gpt-4o-mini",
                "api_base": f"{peers.api.url}/v1",
                "api_key": dl.CONTROL_KEY,
            },
        },
    ]
    path: Final = directory / "device-login-guard.yaml"
    path.write_text(yaml.safe_dump(config))
    return path


@pytest.fixture(scope="module")
def rig(tmp_path_factory: pytest.TempPathFactory) -> Iterator[_Rig]:
    directory: Final = tmp_path_factory.mktemp("device-login-guard")
    chatgpt_dir: Final = directory / "chatgpt"
    copilot_dir: Final = directory / "copilot"
    chatgpt_dir.mkdir()
    copilot_dir.mkdir()
    with (
        gateway_from_environment() as gateway,
        dl.device_login_peers(directory) as peers,
        owned_proxy_process(
            gateway,
            directory,
            {
                **peers.environment(),
                "CHATGPT_TOKEN_DIR": str(chatgpt_dir),
                "GITHUB_COPILOT_TOKEN_DIR": str(copilot_dir),
            },
            config=_config(peers, directory),
            workers=2,
            remove_environment=("REQUEST_TIMEOUT",),
        ) as owned,
    ):
        yield _Rig(owned, peers, chatgpt_dir, copilot_dir)


@pytest.fixture
def scene(rig: _Rig) -> Iterator[_Scene]:
    rig.peers.reset()
    dl.clear_tokens(rig.chatgpt_dir, rig.copilot_dir)
    served: Final[SimpleQueue[str]] = SimpleQueue()
    with rig.owned.gateway.scenario() as scenario:
        key: Final = scenario.key()
        yield _Scene(rig, key, rig.owned.log.stat().st_size, served)
        _every_served_call_logged(key, served)
    rig.peers.reset()
    dl.clear_tokens(rig.chatgpt_dir, rig.copilot_dir)


def _path(endpoint: Endpoint) -> str:
    match endpoint:
        case "chat":
            return "/v1/chat/completions"
        case "responses":
            return "/v1/responses"
        case "messages":
            return "/v1/messages"


def _raw_body(call: _Call) -> Mapping[str, JsonValue]:
    common: Final[Mapping[str, JsonValue]] = MappingProxyType({"model": call.model, "stream": call.stream, **_EXTRA})
    match call.endpoint:
        case "chat":
            return {**common, "messages": [{"role": "user", "content": call.prompt()}]}
        case "responses":
            return {**common, "input": call.prompt()}
        case "messages":
            return {**common, "max_tokens": 64, "messages": [{"role": "user", "content": call.prompt()}]}


def _httpx(call: _Call, seconds: float = _CLIENT_SECONDS) -> _Served:
    with (
        httpx.Client(base_url=call.base_url, timeout=seconds, trust_env=False) as client,
        client.stream(
            "POST",
            _path(call.endpoint),
            json=_raw_body(call),
            headers={"Authorization": f"Bearer {call.key}", "anthropic-version": "2023-06-01"},
        ) as response,
    ):
        return _record(
            call, _Served(response.status_code, response.read().decode(), response.headers.get(_CALL_ID, ""))
        )


def _refused(error: openai.APIStatusError | anthropic.APIStatusError) -> _Served:
    return _Served(error.status_code, error.response.text, error.response.headers.get(_CALL_ID, ""))


def _sdk(call: _Call) -> _Served:
    try:
        if call.endpoint == "messages":
            with (
                anthropic.Anthropic(
                    base_url=call.base_url, api_key=call.key, max_retries=0, timeout=_CLIENT_SECONDS
                ) as claude,
                claude.messages.with_streaming_response.create(
                    model=call.model,
                    max_tokens=64,
                    messages=[{"role": "user", "content": call.prompt()}],
                    stream=call.stream,
                    extra_body=dict(_EXTRA),
                ) as message,
            ):
                return _Served(message.status_code, message.text(), message.headers.get(_CALL_ID, ""))
        with openai.OpenAI(
            base_url=f"{call.base_url}/v1", api_key=call.key, max_retries=0, timeout=_CLIENT_SECONDS
        ) as client:
            if call.endpoint == "chat":
                with client.chat.completions.with_streaming_response.create(
                    model=call.model,
                    messages=[{"role": "user", "content": call.prompt()}],
                    stream=call.stream,
                    extra_body=dict(_EXTRA),
                ) as completion:
                    return _Served(completion.status_code, completion.text(), completion.headers.get(_CALL_ID, ""))
            with client.responses.with_streaming_response.create(
                model=call.model, input=call.prompt(), stream=call.stream, extra_body=dict(_EXTRA)
            ) as created:
                return _Served(created.status_code, created.text(), created.headers.get(_CALL_ID, ""))
    except (openai.APIStatusError, anthropic.APIStatusError) as error:
        return _refused(error)


async def _async_sdk(call: _Call) -> _Served:
    try:
        if call.endpoint == "messages":
            async with (
                anthropic.AsyncAnthropic(
                    base_url=call.base_url, api_key=call.key, max_retries=0, timeout=_CLIENT_SECONDS
                ) as claude,
                claude.messages.with_streaming_response.create(
                    model=call.model,
                    max_tokens=64,
                    messages=[{"role": "user", "content": call.prompt()}],
                    stream=call.stream,
                    extra_body=dict(_EXTRA),
                ) as message,
            ):
                return _Served(message.status_code, await message.text(), message.headers.get(_CALL_ID, ""))
        async with openai.AsyncOpenAI(
            base_url=f"{call.base_url}/v1", api_key=call.key, max_retries=0, timeout=_CLIENT_SECONDS
        ) as client:
            if call.endpoint == "chat":
                async with client.chat.completions.with_streaming_response.create(
                    model=call.model,
                    messages=[{"role": "user", "content": call.prompt()}],
                    stream=call.stream,
                    extra_body=dict(_EXTRA),
                ) as completion:
                    return _Served(
                        completion.status_code, await completion.text(), completion.headers.get(_CALL_ID, "")
                    )
            async with client.responses.with_streaming_response.create(
                model=call.model, input=call.prompt(), stream=call.stream, extra_body=dict(_EXTRA)
            ) as created:
                return _Served(created.status_code, await created.text(), created.headers.get(_CALL_ID, ""))
    except (openai.APIStatusError, anthropic.APIStatusError) as error:
        return _refused(error)


def _serve(call: _Call) -> _Served:
    match call.client:
        case "sdk":
            return _record(call, _sdk(call))
        case "async_sdk":
            return _record(call, asyncio.run(_async_sdk(call)))
        case "httpx":
            return _httpx(call)


def _error_message(text: str) -> str:
    error: Final = rv.JSON_OBJECT.validate_python(rv.JSON_OBJECT.validate_json(text)["error"])
    return str(error["message"])


def _spend_rows(key: str) -> Sequence[Mapping[str, JsonValue]]:
    return eventually(
        lambda: read_rows(
            'SELECT request_id, status, model FROM "LiteLLM_SpendLogs" WHERE api_key = %s',
            (hashlib.sha256(key.encode()).hexdigest(),),
        ),
        lambda rows: len(rows) >= 1,
        seconds=40,
    )


def _record(call: _Call, served: _Served) -> _Served:
    call.served.put(served.call_id)
    return served


def _every_served_call_logged(key: str, served: SimpleQueue[str]) -> None:
    served_calls: Final = tuple(served.get_nowait() for _ in range(served.qsize()))
    if not served_calls:
        return
    eventually(
        lambda: read_rows(
            'SELECT request_id FROM "LiteLLM_SpendLogs" WHERE api_key = %s',
            (hashlib.sha256(key.encode()).hexdigest(),),
        ),
        lambda rows: len(rows) >= len(served_calls),
        seconds=40,
    )


def _failure_logged(served: _Served) -> None:
    rows: Final = eventually(
        lambda: read_rows('SELECT status FROM "LiteLLM_SpendLogs" WHERE request_id = %s', (served.call_id,)),
        lambda found: len(found) >= 1,
        seconds=40,
    )
    assert [str(row["status"]) for row in rows] == ["failure"], rows


def _carrying(received: tuple[Request, ...], marker: str) -> tuple[Request, ...]:
    return tuple(request for request in received if marker.encode() in request.body)


def _describe(received: tuple[Request, ...]) -> tuple[tuple[str, str], ...]:
    return tuple((request.method, request.target) for request in received)


@pytest.mark.parametrize("cell", _CELLS, ids=_Cell.name)
def test_missing_token_is_refused_with_the_login_instructions_before_any_auth_host_call(
    scene: _Scene, cell: _Cell
) -> None:
    call: Final = scene.call(cell)
    served: Final = _serve(call)
    assert served.status == 400, served.text
    assert _REFUSALS[cell.provider] in _error_message(served.text), served.text
    assert scene.rig.peers.auth_connections() == ()
    assert _describe(scene.rig.peers.api.drain()) == ()
    (row,) = _spend_rows(scene.key)
    assert (row["request_id"], row["status"]) == (served.call_id, "failure"), row


@pytest.mark.parametrize("cell", _CELLS, ids=_Cell.name)
def test_stored_token_serves_the_request_without_an_auth_host_call(scene: _Scene, cell: _Cell) -> None:
    scene.store_valid_token(cell.provider)
    call: Final = scene.call(cell)
    served: Final = _serve(call)
    assert served.status == 200, served.text
    assert set(rv.MARKER.findall(served.text)) == {call.marker}, served.text
    received: Final = scene.rig.peers.api.drain()
    (forwarded,) = _carrying(received, call.marker)
    assert dl.bearer(forwarded) == _STORED_BEARERS[cell.provider], _describe(received)
    assert scene.rig.peers.auth_connections() == ()
    (row,) = _spend_rows(scene.key)
    assert row["status"] == "success", row


def test_expired_chatgpt_token_is_refreshed_once_and_the_request_is_served(scene: _Scene) -> None:
    dl.write_chatgpt(scene.rig.chatgpt_dir, dl.chatgpt_record(dl.CHATGPT_EXPIRED, refresh_token=dl.GOOD_REFRESH))
    call: Final = scene.call(_Cell("chatgpt", "responses", False, "httpx"))
    served: Final = _serve(call)
    assert served.status == 200, served.text
    assert set(rv.MARKER.findall(served.text)) == {call.marker}, served.text
    (refresh,) = scene.rig.peers.auth.drain()
    assert (refresh.method, refresh.target, refresh.headers.get("host")) == (
        "POST",
        "/oauth/token",
        dl.CHATGPT_AUTH_HOST,
    )
    sent: Final = rv.JSON_OBJECT.validate_json(refresh.body)
    assert (sent["grant_type"], sent["refresh_token"]) == ("refresh_token", dl.GOOD_REFRESH), sent
    (forwarded,) = _carrying(scene.rig.peers.api.drain(), call.marker)
    assert dl.bearer(forwarded) == dl.CHATGPT_REFRESHED
    stored: Final = json.loads((scene.rig.chatgpt_dir / "auth.json").read_text())
    assert stored["access_token"] == dl.CHATGPT_REFRESHED, sorted(stored)


def test_revoked_chatgpt_refresh_token_is_refused_after_the_refresh_call_alone(scene: _Scene) -> None:
    dl.write_chatgpt(scene.rig.chatgpt_dir, dl.chatgpt_record(dl.CHATGPT_EXPIRED, refresh_token=dl.REVOKED_REFRESH))
    served: Final = _serve(scene.call(_Cell("chatgpt", "chat", False, "httpx")))
    assert served.status == 400, served.text
    assert dl.CHATGPT_REFUSAL in _error_message(served.text), served.text
    _failure_logged(served)
    reached: Final = scene.rig.peers.auth.drain()
    assert {(request.method, request.target) for request in reached} == {("POST", "/oauth/token")}, _describe(reached)
    assert _describe(scene.rig.peers.api.drain()) == ()


@dataclass(frozen=True, slots=True)
class _Route:
    name: str
    provider: Provider
    path: str
    body: Mapping[str, JsonValue]

    def label(self) -> str:
        return self.name


_ROUTES: Final = (
    _Route("chatgpt-embeddings", "chatgpt", "/v1/embeddings", {"model": "chatgpt/gpt-5.6-terra", "input": "login"}),
    _Route("chatgpt-completions", "chatgpt", "/v1/completions", {"model": "chatgpt/gpt-5.6-terra", "prompt": "login"}),
    _Route(
        "chatgpt-images", "chatgpt", "/v1/images/generations", {"model": "chatgpt/gpt-5.6-terra", "prompt": "login"}
    ),
    _Route(
        "copilot-embeddings",
        "copilot",
        "/v1/embeddings",
        {"model": "github_copilot/text-embedding-3-small", "input": "login"},
    ),
    _Route("copilot-completions", "copilot", "/v1/completions", {"model": "github_copilot/gpt-5.4", "prompt": "login"}),
    _Route(
        "copilot-images", "copilot", "/v1/images/generations", {"model": "github_copilot/gpt-5.4", "prompt": "login"}
    ),
)


@pytest.mark.parametrize("route", _ROUTES, ids=_Route.label)
def test_missing_token_is_refused_on_the_other_model_routes(scene: _Scene, route: _Route) -> None:
    served: Final = scene.post(route.path, {**route.body, **_EXTRA})
    assert served.status == 400, served.text
    assert _REFUSALS[route.provider] in _error_message(served.text), served.text
    assert scene.rig.peers.auth_connections() == ()
    assert _describe(scene.rig.peers.api.drain()) == ()


def test_copilot_embeddings_are_served_with_the_stored_api_key(scene: _Scene) -> None:
    scene.store_valid_token("copilot")
    served: Final = scene.post(
        "/v1/embeddings", {"model": "github_copilot/text-embedding-3-small", "input": "login", **_EXTRA}
    )
    assert served.status == 200, served.text
    (item,) = rv.ITEMS.validate_python(rv.JSON_OBJECT.validate_json(served.text)["data"])
    assert item["embedding"] == [0.25, 0.5, 0.75], served.text
    (forwarded,) = scene.rig.peers.api.drain()
    assert (forwarded.method, dl.bearer(forwarded)) == ("POST", dl.COPILOT_STORED_KEY), forwarded.target
    assert forwarded.target.endswith("/embeddings"), forwarded.target
    assert scene.rig.peers.auth_connections() == ()


@pytest.mark.parametrize("provider", _PROVIDERS)
def test_connection_test_reports_the_refusal_instead_of_starting_a_login(scene: _Scene, provider: Provider) -> None:
    served: Final = scene.post(
        "/health/test_connection",
        {"litellm_params": {"model": _MODELS[provider, "chat"]}, "mode": "chat"},
        as_admin=True,
    )
    assert served.status == 200, served.text
    report: Final = rv.JSON_OBJECT.validate_json(served.text)
    assert report["status"] == "error", served.text
    assert _REFUSALS[provider] in str(rv.JSON_OBJECT.validate_python(report["result"])["error"]), served.text
    assert scene.rig.peers.auth_connections() == ()


@pytest.mark.parametrize("provider", _PROVIDERS)
def test_count_tokens_answers_locally_without_a_login(scene: _Scene, provider: Provider) -> None:
    served: Final = scene.post(
        "/v1/messages/count_tokens",
        {"model": _MODELS[provider, "messages"], "messages": [{"role": "user", "content": "count these tokens"}]},
    )
    assert served.status == 200, served.text
    counted: Final = rv.JSON_OBJECT.validate_json(served.text)["input_tokens"]
    assert isinstance(counted, int) and counted > 0, served.text
    assert scene.rig.peers.auth_connections() == ()


def test_health_check_still_answers_without_a_login(scene: _Scene) -> None:
    response: Final = scene.rig.owned.gateway.request("GET", "/health", None)
    assert response.status_code == 200, response.text
    health: Final = rv.JSON_OBJECT.validate_json(response.text)
    healthy: Final = rv.ITEMS.validate_python(health["healthy_endpoints"])
    assert [endpoint["model"] for endpoint in healthy] == ["openai/gpt-4o-mini"], response.text
    assert scene.rig.peers.auth_connections() == ()
    tail: Final = scene.log_tail()
    assert dl.CHATGPT_USER_CODE not in tail and dl.COPILOT_USER_CODE not in tail, tail[-2000:]


@pytest.mark.parametrize("endpoint", _ENDPOINTS)
@pytest.mark.parametrize("provider", _PROVIDERS)
def test_an_auth_host_ready_to_grant_a_login_is_never_asked_and_no_code_reaches_the_log(
    scene: _Scene, provider: Provider, endpoint: Endpoint
) -> None:
    scene.rig.peers.switches.grant.set()
    served: Final = _serve(scene.call(_Cell(provider, endpoint, False, "httpx")))
    assert served.status == 400, served.text
    assert _REFUSALS[provider] in _error_message(served.text), served.text
    assert scene.rig.peers.auth_connections() == ()
    assert _describe(scene.rig.peers.auth.drain()) == ()
    tail: Final = scene.log_tail()
    assert dl.CHATGPT_USER_CODE not in tail and dl.COPILOT_USER_CODE not in tail, tail[-2000:]
    assert scene.token_files() == ()


@pytest.mark.parametrize("endpoint", _ENDPOINTS)
@pytest.mark.parametrize("provider", _PROVIDERS)
def test_stored_token_the_provider_rejects_surfaces_its_401_without_a_login(
    scene: _Scene, provider: Provider, endpoint: Endpoint
) -> None:
    if provider == "chatgpt":
        dl.write_chatgpt(scene.rig.chatgpt_dir, dl.chatgpt_record(dl.CHATGPT_REJECTED))
    else:
        dl.write_copilot_key(scene.rig.copilot_dir, dl.COPILOT_REJECTED_KEY, scene.rig.peers.api.url)
    call: Final = scene.call(_Cell(provider, endpoint, False, "httpx"))
    served: Final = _serve(call)
    assert served.status == 401, served.text
    assert dl.REJECTED_DETAIL in served.text, served.text
    received: Final = scene.rig.peers.api.drain()
    assert {dl.bearer(request) for request in _carrying(received, call.marker)} == {
        dl.CHATGPT_REJECTED if provider == "chatgpt" else dl.COPILOT_REJECTED_KEY
    }, _describe(received)
    assert scene.rig.peers.auth_connections() == ()


@pytest.mark.parametrize("stale_key", (False, True), ids=("missing-api-key", "expired-api-key"))
def test_copilot_api_key_is_minted_once_from_the_stored_access_token(scene: _Scene, stale_key: bool) -> None:
    dl.write_copilot_access(scene.rig.copilot_dir, dl.COPILOT_ACCESS)
    if stale_key:
        dl.write_copilot_key(scene.rig.copilot_dir, dl.COPILOT_REJECTED_KEY, scene.rig.peers.api.url, -3600)
    call: Final = scene.call(_Cell("copilot", "chat", False, "httpx"))
    served: Final = _serve(call)
    assert served.status == 200, served.text
    assert set(rv.MARKER.findall(served.text)) == {call.marker}, served.text
    (minted,) = scene.rig.peers.auth.drain()
    assert (minted.method, minted.target, minted.headers.get("host"), minted.headers.get("authorization")) == (
        "GET",
        "/copilot_internal/v2/token",
        dl.GITHUB_API_HOST,
        f"token {dl.COPILOT_ACCESS}",
    )
    (forwarded,) = _carrying(scene.rig.peers.api.drain(), call.marker)
    assert dl.bearer(forwarded) == dl.COPILOT_MINTED_KEY
    assert json.loads((scene.rig.copilot_dir / "api-key.json").read_text())["token"] == dl.COPILOT_MINTED_KEY


def test_copilot_access_token_github_rejects_fails_the_request_without_a_device_login(scene: _Scene) -> None:
    dl.write_copilot_access(scene.rig.copilot_dir, dl.COPILOT_REJECTED_ACCESS)
    served: Final = _serve(scene.call(_Cell("copilot", "chat", False, "httpx")))
    assert served.status == 400, served.text
    assert "Failed to refresh API key" in _error_message(served.text), served.text
    _failure_logged(served)
    reached: Final = scene.rig.peers.auth.drain()
    assert {(request.method, request.target) for request in reached} == {("GET", "/copilot_internal/v2/token")}, (
        _describe(reached)
    )
    assert _describe(scene.rig.peers.api.drain()) == ()


def test_expired_copilot_api_key_with_no_access_token_is_refused_before_any_auth_host_call(scene: _Scene) -> None:
    dl.write_copilot_key(scene.rig.copilot_dir, dl.COPILOT_STORED_KEY, scene.rig.peers.api.url, -3600)
    served: Final = _serve(scene.call(_Cell("copilot", "responses", False, "httpx")))
    assert served.status == 400, served.text
    assert dl.COPILOT_REFUSAL in _error_message(served.text), served.text
    assert scene.rig.peers.auth_connections() == ()
    assert _describe(scene.rig.peers.api.drain()) == ()


_UNUSABLE_CHATGPT_FILES: Final[Mapping[str, str]] = MappingProxyType(
    {
        "invalid-json": "{not json",
        "json-list": "[]",
        "empty-file": "",
        "empty-object": "{}",
        "access-token-int": json.dumps({"access_token": 12345}),
        "access-token-list": json.dumps({"access_token": ["a", "b"]}),
        "access-token-empty": json.dumps({"access_token": ""}),
        "access-token-5kb": json.dumps({"access_token": "x" * 5120}),
        "opaque-token-without-expiry": json.dumps({"access_token": "opaque-token"}),
    }
)


@pytest.mark.parametrize("name", tuple(_UNUSABLE_CHATGPT_FILES))
def test_unusable_chatgpt_auth_file_is_refused_like_a_missing_one_each_time(scene: _Scene, name: str) -> None:
    (scene.rig.chatgpt_dir / "auth.json").write_text(_UNUSABLE_CHATGPT_FILES[name])
    first: Final = _serve(scene.call(_Cell("chatgpt", "responses", False, "httpx")))
    second: Final = _serve(scene.call(_Cell("chatgpt", "responses", False, "httpx")))
    for served in (first, second):
        assert served.status == 400, served.text
        assert dl.CHATGPT_REFUSAL in _error_message(served.text), served.text
    assert scene.rig.peers.auth_connections() == ()
    assert _describe(scene.rig.peers.api.drain()) == ()
    assert (scene.rig.chatgpt_dir / "auth.json").read_text() == _UNUSABLE_CHATGPT_FILES[name]


def test_chatgpt_token_with_a_text_expiry_falls_back_to_the_expiry_inside_the_token(scene: _Scene) -> None:
    dl.write_chatgpt(scene.rig.chatgpt_dir, dl.chatgpt_record(dl.CHATGPT_STORED, expires_at="soon"))
    call: Final = scene.call(_Cell("chatgpt", "responses", False, "httpx"))
    served: Final = _serve(call)
    assert served.status == 200, served.text
    (forwarded,) = _carrying(scene.rig.peers.api.drain(), call.marker)
    assert dl.bearer(forwarded) == dl.CHATGPT_STORED
    assert scene.rig.peers.auth_connections() == ()


_PROVIDER_RESOLUTION_ERROR: Final = "GetLLMProvider Exception"


@dataclass(frozen=True, slots=True)
class _UnusableCopilotFiles:
    files: Mapping[str, str]
    answer: str


_UNUSABLE_COPILOT_FILES: Final[Mapping[str, _UnusableCopilotFiles]] = MappingProxyType(
    {
        "empty-access-token": _UnusableCopilotFiles(MappingProxyType({"access-token": ""}), dl.COPILOT_REFUSAL),
        "blank-access-token": _UnusableCopilotFiles(MappingProxyType({"access-token": "  \n"}), dl.COPILOT_REFUSAL),
        "invalid-api-key-json": _UnusableCopilotFiles(
            MappingProxyType({"api-key.json": "{not json"}), dl.COPILOT_REFUSAL
        ),
        "api-key-list": _UnusableCopilotFiles(MappingProxyType({"api-key.json": "[]"}), _PROVIDER_RESOLUTION_ERROR),
        "api-key-expiry-text": _UnusableCopilotFiles(
            MappingProxyType({"api-key.json": json.dumps({"token": "copilot-text-expiry", "expires_at": "soon"})}),
            _PROVIDER_RESOLUTION_ERROR,
        ),
    }
)


@pytest.mark.parametrize("name", tuple(_UNUSABLE_COPILOT_FILES))
def test_unusable_copilot_token_files_are_refused_like_missing_ones_each_time(scene: _Scene, name: str) -> None:
    variant: Final = _UNUSABLE_COPILOT_FILES[name]
    for file_name, content in variant.files.items():
        (scene.rig.copilot_dir / file_name).write_text(content)
    first: Final = _serve(scene.call(_Cell("copilot", "chat", False, "httpx")))
    second: Final = _serve(scene.call(_Cell("copilot", "chat", False, "httpx")))
    for served in (first, second):
        assert served.status == 400, served.text
        assert variant.answer in _error_message(served.text), served.text
        assert dl.COPILOT_USER_CODE not in served.text
    assert scene.rig.peers.auth_connections() == ()
    assert _describe(scene.rig.peers.api.drain()) == ()


@pytest.mark.parametrize("provider", _PROVIDERS)
def test_a_token_mounted_after_a_refusal_serves_without_a_restart_and_removing_it_refuses_again(
    scene: _Scene, provider: Provider
) -> None:
    cell: Final = _Cell(provider, "chat", False, "httpx")
    before: Final = _serve(scene.call(cell))
    assert before.status == 400, before.text
    scene.store_valid_token(provider)
    mounted: Final = scene.call(cell)
    served: Final = _serve(mounted)
    assert served.status == 200, served.text
    assert set(rv.MARKER.findall(served.text)) == {mounted.marker}, served.text
    dl.clear_tokens(scene.rig.chatgpt_dir, scene.rig.copilot_dir)
    after: Final = _serve(scene.call(cell))
    assert after.status == 400, after.text
    assert _REFUSALS[provider] in _error_message(after.text), after.text
    assert scene.rig.peers.auth_connections() == ()


_COOLDOWN_SECONDS_LEFT: Final = 45
_REFUSAL_MARGIN_SECONDS: Final = 15


def test_a_recent_device_code_request_does_not_hold_the_worker_for_the_rest_of_the_cooldown(scene: _Scene) -> None:
    requested_at: Final = time.time() - (DEVICE_CODE_COOLDOWN_SECONDS - _COOLDOWN_SECONDS_LEFT)
    dl.write_chatgpt(scene.rig.chatgpt_dir, {"device_code_requested_at": requested_at})
    remaining: Final = requested_at + DEVICE_CODE_COOLDOWN_SECONDS - time.time()
    refusal: Final = timed(lambda: _serve(scene.call(_Cell("chatgpt", "chat", False, "httpx"))))
    served: Final = refusal.value
    assert served.status == 400, served.text
    assert dl.CHATGPT_REFUSAL in _error_message(served.text), served.text
    assert_cut_short(refusal.seconds, injected_delay=remaining - _REFUSAL_MARGIN_SECONDS)
    assert scene.rig.peers.auth_connections() == ()


_NOT_LIVE: Final = re.compile(r"saved to the database, but the model id\(s\) \['([0-9a-f-]+)'\] are not live")


def test_model_created_without_a_token_is_saved_not_live_and_serves_once_the_token_is_mounted(scene: _Scene) -> None:
    gateway: Final = scene.rig.owned.gateway
    name: Final = f"device-login-db-{uuid.uuid4().hex}"
    identity: Final = str(uuid.uuid4())
    created: Final = gateway.request(
        "POST",
        "/model/new",
        {
            "model_name": name,
            "litellm_params": {"model": "chatgpt/gpt-5.6-terra"},
            "model_info": {"id": identity},
        },
    )
    try:
        assert created.status_code == 500, created.text
        not_live: Final = _NOT_LIVE.search(created.text)
        assert not_live is not None and not_live.group(1) == identity, created.text
        assert scene.rig.peers.auth_connections() == ()
        scene.store_valid_token("chatgpt")
        call: Final = _Call(
            scene.rig.base_url(), scene.key, name, "responses", False, "httpx", uuid.uuid4().hex, SimpleQueue()
        )
        served: Final = eventually(lambda: _serve(call), lambda answer: answer.status == 200, seconds=90)
        assert set(rv.MARKER.findall(served.text)) == {call.marker}, served.text
        assert scene.rig.peers.auth_connections() == ()
    finally:
        gateway.request("POST", "/model/delete", {"id": identity})


async def _send(client: httpx.AsyncClient, call: _Call) -> _Served:
    async with client.stream(
        "POST",
        _path(call.endpoint),
        json=_raw_body(call),
        headers={"Authorization": f"Bearer {call.key}", "anthropic-version": "2023-06-01"},
    ) as response:
        raw: Final = await response.aread()
    return _record(call, _Served(response.status_code, raw.decode(), response.headers.get(_CALL_ID, "")))


async def _burst(calls: tuple[_Call, ...]) -> tuple[_Served, ...]:
    async with httpx.AsyncClient(base_url=calls[0].base_url, timeout=_CLIENT_SECONDS, trust_env=False) as client:
        return tuple(await asyncio.gather(*(_send(client, call) for call in calls)))


def _chat_response_id(call: _Call, served: _Served) -> str:
    if not call.stream:
        return str(rv.JSON_OBJECT.validate_json(served.text)["id"])
    first: Final = next(line for line in served.text.splitlines() if line.startswith("data: {"))
    return str(rv.JSON_OBJECT.validate_json(first[6:])["id"])


_HTTPX_CELLS: Final = tuple(cell for cell in _CELLS if cell.client == "httpx")


def test_a_burst_of_refusals_leaves_other_models_served_and_logs_every_request_once(scene: _Scene) -> None:
    refused_cells: Final = tuple(_HTTPX_CELLS[index % len(_HTTPX_CELLS)] for index in range(30))
    refused_calls: Final = tuple(scene.call(cell) for cell in refused_cells)
    control_calls: Final = tuple(scene.control(stream=index % 2 == 1) for index in range(10))
    answers: Final = asyncio.run(_burst((*refused_calls, *control_calls)))
    refused: Final = answers[:30]
    controls: Final = answers[30:]
    for cell, answer in zip(refused_cells, refused, strict=True):
        assert answer.status == 400, answer.text
        assert _REFUSALS[cell.provider] in _error_message(answer.text), answer.text
    for call, answer in zip(control_calls, controls, strict=True):
        assert answer.status == 200, answer.text
        assert set(rv.MARKER.findall(answer.text)) == {call.marker}, answer.text
    assert scene.rig.peers.auth_connections() == ()
    received: Final = scene.rig.peers.api.drain()
    assert sorted(rv.newest_marker(request.body.decode()) or "" for request in received) == sorted(
        call.marker for call in control_calls
    ), _describe(received)
    assert {dl.bearer(request) for request in received} == {dl.CONTROL_KEY}
    rows: Final = eventually(
        lambda: read_rows(
            'SELECT request_id, status FROM "LiteLLM_SpendLogs" WHERE api_key = %s',
            (hashlib.sha256(scene.key.encode()).hexdigest(),),
        ),
        lambda found: len(found) >= 40,
        seconds=70,
    )
    by_status: Final = {str(row["request_id"]): str(row["status"]) for row in rows}
    assert len(by_status) == len(rows) == 40, rows
    for answer in refused:
        assert by_status.get(answer.call_id) == "failure", (answer.call_id, rows)
    for call, answer in zip(control_calls, controls, strict=True):
        (logged,) = [
            request_id for request_id in by_status if rv.same_response(request_id, _chat_response_id(call, answer))
        ]
        assert by_status[logged] == "success", rows


def test_auth_host_outage_refuses_each_refresh_and_the_first_request_after_it_refreshes_once(scene: _Scene) -> None:
    dl.write_chatgpt(scene.rig.chatgpt_dir, dl.chatgpt_record(dl.CHATGPT_EXPIRED, refresh_token=dl.GOOD_REFRESH))
    chatgpt_cells: Final = tuple(cell for cell in _HTTPX_CELLS if cell.provider == "chatgpt")
    scene.rig.peers.switches.refuse_connect.set()
    during: Final = asyncio.run(_burst(tuple(scene.call(chatgpt_cells[index % 6]) for index in range(12))))
    for answer in during:
        assert answer.status == 400, answer.text
        assert dl.CHATGPT_REFUSAL in _error_message(answer.text), answer.text
    for answer in during:
        _failure_logged(answer)
    assert set(scene.rig.peers.auth_connections()) == {f"{dl.CHATGPT_AUTH_HOST}:443"}
    assert _describe(scene.rig.peers.auth.drain()) == ()
    assert _describe(scene.rig.peers.api.drain()) == ()
    scene.rig.peers.switches.refuse_connect.clear()
    first: Final = scene.call(chatgpt_cells[0])
    recovered: Final = _serve(first)
    assert recovered.status == 200, recovered.text
    (refresh,) = scene.rig.peers.auth.drain()
    assert (refresh.method, refresh.target) == ("POST", "/oauth/token")
    after_calls: Final = tuple(scene.call(chatgpt_cells[index % 6]) for index in range(12))
    after: Final = asyncio.run(_burst(after_calls))
    for call, answer in zip(after_calls, after, strict=True):
        assert answer.status == 200, answer.text
        assert set(rv.MARKER.findall(answer.text)) == {call.marker}, answer.text
    assert _describe(scene.rig.peers.auth.drain()) == ()
    forwarded: Final = scene.rig.peers.api.drain()
    assert {dl.bearer(request) for request in forwarded} == {dl.CHATGPT_REFRESHED}, _describe(forwarded)


def _refresh_through_a_stalled_auth_host(scene: _Scene, stall: threading.Event) -> tuple[dl.Hangup, _Served, _Call]:
    dl.write_chatgpt(scene.rig.chatgpt_dir, dl.chatgpt_record(dl.CHATGPT_EXPIRED, refresh_token=dl.GOOD_REFRESH))
    call: Final = scene.call(_Cell("chatgpt", "responses", False, "httpx"))
    stall.set()
    with ThreadPoolExecutor(max_workers=1) as pool:
        pending: Final = pool.submit(_httpx, call, 240)
        try:
            dropped: Final = eventually(scene.rig.peers.dropped, lambda hangups: len(hangups) >= 1, seconds=70)
        finally:
            scene.rig.peers.switches.refuse_connect.set()
            stall.clear()
        served: Final = pending.result(timeout=240)
    assert served.status == 400, served.text
    _failure_logged(served)
    scene.rig.peers.switches.refuse_connect.clear()
    assert set(scene.rig.peers.auth_connections()) <= {f"{dl.CHATGPT_AUTH_HOST}:443"}
    return dropped[0], served, call


def _assert_refused_then_refreshed(scene: _Scene, served: _Served) -> None:
    assert served.status == 400, served.text
    assert dl.CHATGPT_REFUSAL in _error_message(served.text), served.text
    assert scene.rig.peers.dropped() == ()
    assert _serve(scene.control(stream=False)).status == 200
    recovered: Final = scene.call(_Cell("chatgpt", "responses", False, "httpx"))
    answer: Final = _serve(recovered)
    assert answer.status == 200, answer.text
    assert set(rv.MARKER.findall(answer.text)) == {recovered.marker}, answer.text
    refreshes: Final = [(request.method, request.target) for request in scene.rig.peers.auth.drain()]
    assert refreshes == [("POST", "/oauth/token")], refreshes
    (forwarded,) = _carrying(scene.rig.peers.api.drain(), recovered.marker)
    assert dl.bearer(forwarded) == dl.CHATGPT_REFRESHED


def test_refresh_call_gives_up_on_a_silent_auth_host_after_30_seconds_and_the_worker_recovers(scene: _Scene) -> None:
    dropped, served, _ = _refresh_through_a_stalled_auth_host(scene, scene.rig.peers.switches.hold_connect)
    assert dropped.authority == f"{dl.CHATGPT_AUTH_HOST}:443"
    assert 25 <= dropped.seconds <= 45, dropped
    _assert_refused_then_refreshed(scene, served)


def test_refresh_call_gives_up_on_a_stalled_tls_handshake_at_the_connect_timeout(scene: _Scene) -> None:
    dropped, served, _ = _refresh_through_a_stalled_auth_host(scene, scene.rig.peers.switches.stall_tls)
    assert dropped.authority == f"{dl.CHATGPT_AUTH_HOST}:443"
    assert 3 <= dropped.seconds <= 20, dropped
    _assert_refused_then_refreshed(scene, served)