Skip to content

The fastest, litest AI Gateway. Rust core with Python SDK. Call 100+ LLM APIs in OpenAI (or native) format with cost tracking, guardrails, load balancing, and logging [Bedrock, Azure, OpenAI, Anthropic, OpenAI, VertexAI, vLLM, Nvidia NIM]

49.5 KB
Raw Blame
import json
import os
import time
import uuid
from collections.abc import Callable, Iterator
from dataclasses import dataclass
from hashlib import sha256
from pathlib import Path
from typing import Final, Literal

import httpx
import jwt
import pytest
from cryptography.hazmat.primitives.asymmetric import rsa
from jwt.algorithms import RSAAlgorithm
from pydantic import BaseModel, JsonValue, TypeAdapter
from redis import Redis

from tests.integration._support.client import (
    Gateway,
    Scenario,
    delete_key_if_present,
    eventually,
    gateway_from_environment,
    object_value,
    string_value,
)
from tests.integration._support.database import read_rows, write_rows
from tests.integration._support.process import owned_proxy
from tests.integration._support.wire import Reply, Request, Wire, wire_server

_AUDIENCE: Final = "litellm-integration"
_CLIENT_CLAIM: Final = "client_id"
_END_USER_CLAIM: Final = "eu"
_KEY_ID: Final = "integration-signing-key"
_UPSTREAM_KEY: Final = "synthetic-openai-key"
_CHANNEL: Final = "litellm_proxy.auth_cache_invalidation"
_CHAT_BODY: Final = TypeAdapter(dict[str, JsonValue])
pytestmark: Final = [pytest.mark.timeout(180), pytest.mark.shares_worker_state]


class _Usage(BaseModel):
    prompt_tokens: int
    completion_tokens: int
    total_tokens: int


class _AssistantMessage(BaseModel):
    content: str


class _Choice(BaseModel):
    message: _AssistantMessage


class _ChatResponse(BaseModel):
    id: str
    choices: tuple[_Choice, ...]
    usage: _Usage


class _KeyResponse(BaseModel):
    key: str


class _DeleteResponse(BaseModel):
    deleted_keys: tuple[str, ...]


class _JWTMappingResponse(BaseModel):
    id: str
    jwt_issuer: str | None
    jwt_claim_name: str
    jwt_claim_value: str
    is_active: bool


class _JWTMappingRow(BaseModel):
    id: str
    token: str
    jwt_issuer: str
    is_active: bool


class _SpendRow(BaseModel):
    api_key: str


class _CountRow(BaseModel):
    count: int


class _MappingIdRow(BaseModel):
    id: str


class _ModelEntry(BaseModel):
    id: str


class _ModelList(BaseModel):
    data: tuple[_ModelEntry, ...]


class _UserMessage(BaseModel):
    content: str


class _ChatRequest(BaseModel):
    model: str
    messages: tuple[_UserMessage, ...]


class _ProxyError(BaseModel):
    message: str
    type: str
    param: str
    code: str


class _ErrorResponse(BaseModel):
    error: _ProxyError


_UNKNOWN_KEY_ERROR: Final = _ProxyError(
    message="Authentication Error, Invalid API key provided.", type="token_not_found_in_db", param="key", code="401"
)


@dataclass(frozen=True, slots=True)
class _JWTLifecycleRuntime:
    first: Gateway
    second: Gateway
    upstream: Wire
    model: str
    private_key_a: rsa.RSAPrivateKey
    private_key_b: rsa.RSAPrivateKey
    issuer_a: str
    issuer_b: str


def _proxy_config(
    directory: Path,
    model: str,
    upstream_url: str,
    issuers: tuple[dict[str, JsonValue], ...] = (),
) -> Path:
    config: Final = directory / "jwt_key_mapping_config.yaml"
    config.write_text(
        json.dumps(
            {
                "model_list": (
                    {
                        "model_name": model,
                        "litellm_params": {
                            "model": f"openai/{model}",
                            "api_base": f"{upstream_url}/v1",
                            "api_key": _UPSTREAM_KEY,
                        },
                    },
                ),
                "general_settings": {
                    "master_key": "os.environ/LITELLM_MASTER_KEY",
                    "database_url": "os.environ/DATABASE_URL",
                    "store_model_in_db": True,
                    "disable_model_info_refresh": True,
                    "proxy_batch_write_at": 1,
                    "proxy_batch_polling_interval": 1,
                    "enable_jwt_auth": True,
                    "litellm_jwtauth": {
                        "user_id_jwt_field": "sub",
                        "virtual_key_claim_field": _CLIENT_CLAIM,
                        "unregistered_jwt_client_behavior": "reject",
                        **({"issuers": issuers} if issuers else {}),
                    },
                },
                "litellm_settings": {
                    "enable_redis_auth_cache": True,
                    "cache": True,
                    "cache_params": {
                        "type": "redis",
                        "host": "os.environ/REDIS_HOST",
                        "port": "os.environ/REDIS_PORT",
                    },
                },
            }
        )
    )
    return config


def _issuer_config(issuer: str, jwks_url: str) -> dict[str, JsonValue]:
    return {
        "issuer": issuer,
        "jwks_url": jwks_url,
        "audience": _AUDIENCE,
        "virtual_key_claim_field": _CLIENT_CLAIM,
        "user_id_jwt_field": "sub",
    }


def _jwks_body(private_key: rsa.RSAPrivateKey) -> bytes:
    public_jwk: Final = _CHAT_BODY.validate_json(RSAAlgorithm.to_jwk(private_key.public_key()))
    return json.dumps({"keys": [{**public_jwk, "kid": _KEY_ID, "use": "sig", "alg": "RS256"}]}).encode()


def _jwks_respond(private_key: rsa.RSAPrivateKey) -> Callable[[Request], Reply]:
    body: Final = _jwks_body(private_key)

    def respond(request: Request) -> Reply:
        assert request.method == "GET"
        assert request.target == "/jwks"
        return Reply(body=body)

    return respond


def _signed_jwt(
    private_key: rsa.RSAPrivateKey,
    client_id: str,
    issuer: str | None = None,
    *,
    subject: str | None = None,
    end_user: str | None = None,
) -> str:
    now: Final = int(time.time())
    claims: Final[dict[str, JsonValue]] = {
        "sub": subject if subject is not None else "integration-user-" + uuid.uuid4().hex,
        "client_id": client_id,
        "aud": _AUDIENCE,
        "iat": now,
        "exp": now + 300,
        **({"iss": issuer} if issuer is not None else {}),
        **({_END_USER_CLAIM: end_user} if end_user is not None else {}),
    }
    return jwt.encode(claims, private_key, algorithm="RS256", headers={"kid": _KEY_ID})


def _chat_body(model: str, prompt: str) -> dict[str, JsonValue]:
    return {
        "model": model,
        "messages": [{"role": "user", "content": prompt}],
    }


def _wire_reply(identity: str) -> Reply:
    return Reply(
        body=json.dumps(
            {
                "id": identity,
                "object": "chat.completion",
                "created": 1,
                "model": "integration-jwt-key-mapping",
                "choices": [
                    {
                        "index": 0,
                        "message": {"role": "assistant", "content": "wire response"},
                        "finish_reason": "stop",
                    }
                ],
                "usage": {"prompt_tokens": 20, "completion_tokens": 20, "total_tokens": 40},
            }
        ).encode()
    )


def _chat_respond(backend: str) -> Callable[[Request], Reply]:
    def respond(request: Request) -> Reply:
        if request.method == "GET":
            assert request.target == "/v1/models", request
            assert request.headers["authorization"] == f"Bearer {_UPSTREAM_KEY}"
            assert request.body == b""
            return Reply(
                body=json.dumps(
                    {
                        "object": "list",
                        "data": [
                            {
                                "id": backend,
                                "object": "model",
                                "created": 0,
                                "owned_by": "integration",
                            }
                        ],
                    }
                ).encode()
            )
        assert request.method == "POST", request
        assert request.target == "/v1/chat/completions"
        assert request.headers["authorization"] == f"Bearer {_UPSTREAM_KEY}"
        body: Final = _CHAT_BODY.validate_python(json.loads(request.body))
        parsed: Final = _ChatRequest.model_validate_json(request.body)
        assert len(parsed.messages) == 1
        assert body == _chat_body(backend, parsed.messages[0].content), body
        return _wire_reply("chatcmpl-" + uuid.uuid4().hex)

    return respond


def _chat(gateway: Gateway, model: str, token: str, prompt: str) -> httpx.Response:
    return gateway.request("POST", "/v1/chat/completions", _chat_body(model, prompt), key=token)


def _expected_chat_request(backend: str, prompt: str) -> tuple[str, str, dict[str, JsonValue]]:
    body: Final = _chat_body(backend, prompt)
    return "POST", "/v1/chat/completions", body


def _upstream_request_key(
    request: tuple[str, str, dict[str, JsonValue] | None],
) -> tuple[str, str, str]:
    return request[0], request[1], "" if request[2] is None else json.dumps(request[2], sort_keys=True)


def _assert_upstream_requests(
    upstream: Wire,
    expected: tuple[tuple[str, str, dict[str, JsonValue] | None], ...],
) -> None:
    observed: Final = tuple(
        sorted(
            (
                (
                    request.method,
                    request.target,
                    None if not request.body else _CHAT_BODY.validate_python(json.loads(request.body)),
                )
                for request in upstream.drain()
            ),
            key=_upstream_request_key,
        )
    )
    assert observed == tuple(sorted(expected, key=_upstream_request_key)), observed


def _assert_chat(gateway: Gateway, model: str, token: str, prompt: str, key_hash: str) -> _ChatResponse:
    response: Final = _chat(gateway, model, token, prompt)
    assert response.status_code == 200, response.text
    payload: Final = _ChatResponse.model_validate_json(response.content)
    assert payload.choices[0].message.content == "wire response", response.text
    assert payload.usage == _Usage(prompt_tokens=20, completion_tokens=20, total_tokens=40), response.text
    spend: Final = TypeAdapter(tuple[_SpendRow, ...]).validate_python(
        eventually(
            lambda: read_rows(
                'SELECT api_key FROM "LiteLLM_SpendLogs" WHERE request_id=%s',
                (payload.id,),
            ),
            lambda rows: len(rows) == 1,
            seconds=70,
        )
    )
    assert spend == (_SpendRow(api_key=key_hash),)
    return payload


def _model_is_available(gateway: Gateway, model: str) -> bool:
    response: Final = gateway.request("GET", "/v1/models")
    if response.status_code != 200:
        return False
    payload: Final = _ModelList.model_validate_json(response.content)
    return any(entry.id == model for entry in payload.data)


def _create_key(gateway: Gateway, model: str) -> str:
    response: Final = gateway.request("POST", "/key/generate", {"models": [model]})
    assert response.status_code == 200, response.text
    return _KeyResponse.model_validate_json(response.content).key


def _create_mapping(
    gateway: Gateway,
    key: str,
    client_id: str,
    issuer: str | None = None,
    spelling: Literal["key", "token"] = "key",
) -> _JWTMappingResponse:
    body: Final = {
        "jwt_claim_name": _CLIENT_CLAIM,
        "jwt_claim_value": client_id,
        spelling: key if spelling == "key" else sha256(key.encode()).hexdigest(),
        **({"jwt_issuer": issuer} if issuer is not None else {}),
    }
    response: Final = gateway.request("POST", "/jwt/key/mapping/new", body)
    assert response.status_code == 200, response.text
    return _JWTMappingResponse.model_validate_json(response.content)


def _delete_mapping_if_present(gateway: Gateway, identity: str) -> None:
    rows: Final = TypeAdapter(tuple[_MappingIdRow, ...]).validate_python(
        read_rows('SELECT id FROM "LiteLLM_JWTKeyMapping" WHERE id=%s', (identity,))
    )
    if not rows:
        return
    response: Final = gateway.request("POST", "/jwt/key/mapping/delete", {"id": identity})
    assert response.status_code == 200, response.text


def _assert_unmapped(response: httpx.Response, client_id: str) -> None:
    assert response.status_code == 403, response.text
    error: Final = _ErrorResponse.model_validate_json(response.content)
    expected: Final = f"JWT Key Mapping: No registered mapping for {_CLIENT_CLAIM}='{client_id}'. Access denied."
    assert error.error == _ProxyError(message=expected, type="auth_error", param="None", code="403"), response.text


def _mapping_info(gateway: Gateway, identity: str) -> _JWTMappingResponse:
    response: Final = gateway.request("GET", "/jwt/key/mapping/info", params={"id": identity})
    assert response.status_code == 200, response.text
    return _JWTMappingResponse.model_validate_json(response.content)


def _mapping_row(gateway: Gateway, identity: str) -> _JWTMappingRow:
    rows: Final = TypeAdapter(tuple[_JWTMappingRow, ...]).validate_python(
        read_rows(
            'SELECT id, token, jwt_issuer, is_active FROM "LiteLLM_JWTKeyMapping" WHERE id=%s',
            (identity,),
        )
    )
    assert len(rows) == 1
    return rows[0]


def _spend_count(gateway: Gateway, key_hash: str) -> int:
    rows: Final = TypeAdapter(tuple[_CountRow, ...]).validate_python(
        read_rows('SELECT count(*) AS count FROM "LiteLLM_SpendLogs" WHERE api_key=%s', (key_hash,))
    )
    assert len(rows) == 1
    return rows[0].count


@pytest.fixture(scope="module")
def jwt_lifecycle(tmp_path_factory: pytest.TempPathFactory) -> Iterator[_JWTLifecycleRuntime]:
    directory: Final = tmp_path_factory.mktemp("jwt-key-mapping-lifecycle")
    model: Final = "jwt-key-mapping-" + uuid.uuid4().hex
    issuer_a: Final = "https://idp-a.integration.example"
    issuer_b: Final = "https://idp-b.integration.example"
    private_key_a: Final = rsa.generate_private_key(public_exponent=65537, key_size=2048)
    private_key_b: Final = rsa.generate_private_key(public_exponent=65537, key_size=2048)

    with (
        gateway_from_environment() as base,
        wire_server(_chat_respond(model)) as upstream,
        wire_server(_jwks_respond(private_key_a)) as jwks_a,
        wire_server(_jwks_respond(private_key_b)) as jwks_b,
        Redis(host=os.environ["REDIS_HOST"], port=int(os.environ["REDIS_PORT"])) as cache,
    ):
        issuers: Final = (
            _issuer_config(issuer_a, f"{jwks_a.url}/jwks"),
            _issuer_config(issuer_b, f"{jwks_b.url}/jwks"),
        )
        config: Final = _proxy_config(directory, model, upstream.url, issuers)
        baseline: Final = cache.pubsub_numsub(_CHANNEL)[0][1]
        with (
            owned_proxy(base, directory / "worker-one", {}, config=config) as first,
            owned_proxy(base, directory / "worker-two", {}, config=config) as second,
        ):
            subscribers: Final = eventually(
                lambda: cache.pubsub_numsub(_CHANNEL)[0][1],
                lambda count: count >= baseline + 2,
                seconds=20,
            )
            assert subscribers >= baseline + 2
            eventually(lambda: _model_is_available(first, model), bool, seconds=30)
            eventually(lambda: _model_is_available(second, model), bool, seconds=30)
            _assert_upstream_requests(upstream, ())
            yield _JWTLifecycleRuntime(
                first=first,
                second=second,
                upstream=upstream,
                model=model,
                private_key_a=private_key_a,
                private_key_b=private_key_b,
                issuer_a=issuer_a,
                issuer_b=issuer_b,
            )


def test_updating_a_warmed_mapping_deactivates_and_repoints_it_on_both_workers(
    jwt_lifecycle: _JWTLifecycleRuntime,
) -> None:
    first: Final = jwt_lifecycle.first
    second: Final = jwt_lifecycle.second
    upstream: Final = jwt_lifecycle.upstream
    model: Final = jwt_lifecycle.model
    backend: Final = model
    client_id: Final = "integration-client-" + uuid.uuid4().hex
    issuer: Final = jwt_lifecycle.issuer_a
    prompts: Final = (
        "mapping warm primary " + uuid.uuid4().hex,
        "mapping warm peer " + uuid.uuid4().hex,
        "mapping inactive primary " + uuid.uuid4().hex,
        "mapping inactive peer " + uuid.uuid4().hex,
        "mapping repointed primary " + uuid.uuid4().hex,
        "mapping repointed peer " + uuid.uuid4().hex,
        "mapping restored primary " + uuid.uuid4().hex,
        "mapping restored peer " + uuid.uuid4().hex,
    )
    token: Final = _signed_jwt(jwt_lifecycle.private_key_a, client_id, issuer)
    upstream.drain()
    with first.scenario() as scenario:
        old_key: Final = _create_key(first, model)
        new_key: Final = _create_key(first, model)
        scenario.cleanups.callback(delete_key_if_present, first, old_key)
        scenario.cleanups.callback(delete_key_if_present, first, new_key)
        old_hash: Final = sha256(old_key.encode()).hexdigest()
        new_hash: Final = sha256(new_key.encode()).hexdigest()
        mapping: Final = _create_mapping(first, old_key, client_id, issuer)
        scenario.cleanups.callback(_delete_mapping_if_present, first, mapping.id)
        assert mapping.jwt_issuer == issuer
        assert _mapping_row(first, mapping.id).jwt_issuer == issuer

        first_response: Final = _assert_chat(first, model, token, prompts[0], old_hash)
        second_response: Final = _assert_chat(second, model, token, prompts[1], old_hash)
        assert first_response.id != second_response.id

        deactivated_response: Final = first.request(
            "POST", "/jwt/key/mapping/update", {"id": mapping.id, "is_active": False}
        )
        assert deactivated_response.status_code == 200, deactivated_response.text
        deactivated: Final = _JWTMappingResponse.model_validate_json(deactivated_response.content)
        assert deactivated.is_active is False, deactivated_response.text
        assert _mapping_info(first, mapping.id).is_active is False
        inactive_row: Final = _mapping_row(first, mapping.id)
        assert inactive_row.token == old_hash and inactive_row.is_active is False
        old_spend_count: Final = _spend_count(first, old_hash)

        refused_first: Final = _chat(first, model, token, prompts[2])
        _assert_unmapped(refused_first, client_id)
        refused_second: Final = eventually(
            lambda: _chat(second, model, token, prompts[3]),
            lambda response: response.status_code == 403,
            seconds=3,
        )
        _assert_unmapped(refused_second, client_id)
        assert _spend_count(first, old_hash) == old_spend_count

        reactivated_response: Final = first.request(
            "POST",
            "/jwt/key/mapping/update",
            {"id": mapping.id, "is_active": True, "token": new_hash},
        )
        assert reactivated_response.status_code == 200, reactivated_response.text
        reactivated: Final = _JWTMappingResponse.model_validate_json(reactivated_response.content)
        assert reactivated.is_active is True, reactivated_response.text
        assert _mapping_info(first, mapping.id).is_active is True
        active_row: Final = _mapping_row(first, mapping.id)
        assert active_row.token == new_hash and active_row.is_active is True
        _assert_chat(first, model, token, prompts[4], new_hash)
        _assert_chat(second, model, token, prompts[5], new_hash)

        restored_response: Final = first.request(
            "POST",
            "/jwt/key/mapping/update",
            {"id": mapping.id, "key": old_key},
        )
        assert restored_response.status_code == 200, restored_response.text
        restored: Final = _JWTMappingResponse.model_validate_json(restored_response.content)
        assert restored.is_active is True, restored_response.text
        assert _mapping_info(first, mapping.id).is_active is True
        restored_row: Final = _mapping_row(first, mapping.id)
        assert restored_row.token == old_hash and restored_row.is_active is True
        _assert_chat(first, model, token, prompts[6], old_hash)
        _assert_chat(second, model, token, prompts[7], old_hash)
        served_prompts: Final = prompts[0:2] + prompts[4:8]
        _assert_upstream_requests(upstream, tuple(_expected_chat_request(backend, prompt) for prompt in served_prompts))


def test_reactivating_a_global_mapping_restores_issuer_jwt_callers_on_both_workers(
    jwt_lifecycle: _JWTLifecycleRuntime,
) -> None:
    pytest.skip(
        "BUG: reactivating a global JWT key mapping keeps refusing issuer-scoped JWTs with 403 No registered mapping "
        "because the miss was cached under the issuer key and update evicts only the global key"
    )
    first: Final = jwt_lifecycle.first
    second: Final = jwt_lifecycle.second
    upstream: Final = jwt_lifecycle.upstream
    model: Final = jwt_lifecycle.model
    client_id: Final = "integration-client-" + uuid.uuid4().hex
    prompts: Final = tuple(f"global mapping reactivation {index} {uuid.uuid4().hex}" for index in range(6))
    token: Final = _signed_jwt(jwt_lifecycle.private_key_a, client_id, jwt_lifecycle.issuer_a)
    upstream.drain()

    with first.scenario() as scenario:
        key: Final = _create_key(first, model)
        scenario.cleanups.callback(delete_key_if_present, first, key)
        key_hash: Final = sha256(key.encode()).hexdigest()
        mapping: Final = _create_mapping(first, key, client_id)
        scenario.cleanups.callback(_delete_mapping_if_present, first, mapping.id)
        assert mapping.jwt_issuer is None
        assert _mapping_info(first, mapping.id).jwt_issuer is None
        assert _mapping_row(first, mapping.id) == _JWTMappingRow(
            id=mapping.id,
            token=key_hash,
            jwt_issuer="",
            is_active=True,
        )
        _assert_chat(first, model, token, prompts[0], key_hash)
        _assert_chat(second, model, token, prompts[1], key_hash)

        deactivated_response: Final = first.request(
            "POST",
            "/jwt/key/mapping/update",
            {"id": mapping.id, "is_active": False},
        )
        assert deactivated_response.status_code == 200, deactivated_response.text
        assert _mapping_info(first, mapping.id).is_active is False
        assert _mapping_row(first, mapping.id) == _JWTMappingRow(
            id=mapping.id,
            token=key_hash,
            jwt_issuer="",
            is_active=False,
        )
        _assert_unmapped(_chat(first, model, token, prompts[2]), client_id)
        refused_second: Final = eventually(
            lambda: _chat(second, model, token, prompts[3]),
            lambda response: response.status_code == 403,
            seconds=3,
        )
        _assert_unmapped(refused_second, client_id)
        assert _spend_count(first, key_hash) == 2

        reactivated_response: Final = first.request(
            "POST",
            "/jwt/key/mapping/update",
            {"id": mapping.id, "is_active": True},
        )
        assert reactivated_response.status_code == 200, reactivated_response.text
        reactivated: Final = _JWTMappingResponse.model_validate_json(reactivated_response.content)
        assert reactivated.is_active is True, reactivated_response.text
        assert reactivated.jwt_issuer is None, reactivated_response.text
        assert _mapping_info(first, mapping.id).jwt_issuer is None
        assert _mapping_row(first, mapping.id) == _JWTMappingRow(
            id=mapping.id,
            token=key_hash,
            jwt_issuer="",
            is_active=True,
        )
        _assert_chat(first, model, token, prompts[4], key_hash)
        _assert_chat(second, model, token, prompts[5], key_hash)
        assert _spend_count(first, key_hash) == 4
        _assert_upstream_requests(
            upstream,
            tuple(_expected_chat_request(model, prompt) for prompt in prompts[0:2] + prompts[4:6]),
        )


@pytest.mark.parametrize("scope", ("global", "issuer"))
@pytest.mark.parametrize("spelling", ("body", "path", "plaintext_path"))
def test_regenerating_a_jwt_mapped_key_moves_jwt_callers_to_the_new_hash(
    jwt_lifecycle: _JWTLifecycleRuntime, spelling: str, scope: str
) -> None:
    first: Final = jwt_lifecycle.first
    second: Final = jwt_lifecycle.second
    upstream: Final = jwt_lifecycle.upstream
    model: Final = jwt_lifecycle.model
    backend: Final = model
    client_id: Final = "integration-client-" + uuid.uuid4().hex
    prompts: Final = (
        "regenerate warm primary " + uuid.uuid4().hex,
        "regenerate warm peer " + uuid.uuid4().hex,
        "regenerated primary " + uuid.uuid4().hex,
        "regenerated peer " + uuid.uuid4().hex,
    )
    issuer: Final = jwt_lifecycle.issuer_a if scope == "issuer" else None
    token: Final = _signed_jwt(jwt_lifecycle.private_key_a, client_id, jwt_lifecycle.issuer_a)
    upstream.drain()
    with first.scenario() as scenario:
        old_key: Final = _create_key(first, model)
        scenario.cleanups.callback(delete_key_if_present, first, old_key)
        old_hash: Final = sha256(old_key.encode()).hexdigest()
        mapping: Final = _create_mapping(first, old_key, client_id, issuer)
        scenario.cleanups.callback(_delete_mapping_if_present, first, mapping.id)
        expected_issuer: Final = issuer if issuer is not None else ""
        assert mapping.jwt_issuer == issuer
        assert _mapping_info(first, mapping.id).jwt_issuer == issuer
        assert _mapping_row(first, mapping.id).jwt_issuer == expected_issuer
        _assert_chat(first, model, token, prompts[0], old_hash)
        _assert_chat(second, model, token, prompts[1], old_hash)

        path_key: Final = old_hash if spelling == "path" else old_key
        regenerated_response: Final = (
            first.request("POST", "/key/regenerate", {"key": old_key})
            if spelling == "body"
            else first.request("POST", f"/key/{path_key}/regenerate", {})
        )
        assert regenerated_response.status_code == 200, regenerated_response.text
        new_key: Final = _KeyResponse.model_validate_json(regenerated_response.content).key
        scenario.cleanups.callback(delete_key_if_present, first, new_key)
        new_hash: Final = sha256(new_key.encode()).hexdigest()
        mapped_row: Final = _mapping_row(first, mapping.id)
        assert mapped_row.token == new_hash and mapped_row.is_active is True
        assert mapped_row.jwt_issuer == expected_issuer
        assert _mapping_info(first, mapping.id).jwt_issuer == issuer
        _assert_chat(first, model, token, prompts[2], new_hash)
        _assert_chat(second, model, token, prompts[3], new_hash)
        _assert_upstream_requests(upstream, tuple(_expected_chat_request(backend, prompt) for prompt in prompts))


@pytest.mark.parametrize("scope", ("global", "issuer"))
def test_deleting_a_jwt_mapped_key_stops_its_mapping_on_both_workers(
    jwt_lifecycle: _JWTLifecycleRuntime, scope: str
) -> None:
    first: Final = jwt_lifecycle.first
    second: Final = jwt_lifecycle.second
    upstream: Final = jwt_lifecycle.upstream
    model: Final = jwt_lifecycle.model
    backend: Final = model
    client_id: Final = "integration-client-" + uuid.uuid4().hex
    prompts: Final = (
        "delete mapped warm primary " + uuid.uuid4().hex,
        "delete mapped warm peer " + uuid.uuid4().hex,
        "delete mapped primary " + uuid.uuid4().hex,
        "delete mapped peer " + uuid.uuid4().hex,
    )
    issuer: Final = jwt_lifecycle.issuer_a if scope == "issuer" else None
    token: Final = _signed_jwt(jwt_lifecycle.private_key_a, client_id, jwt_lifecycle.issuer_a)
    upstream.drain()
    with first.scenario() as scenario:
        key: Final = _create_key(first, model)
        scenario.cleanups.callback(delete_key_if_present, first, key)
        key_hash: Final = sha256(key.encode()).hexdigest()
        mapping: Final = _create_mapping(first, key, client_id, issuer)
        scenario.cleanups.callback(_delete_mapping_if_present, first, mapping.id)
        expected_issuer: Final = issuer if issuer is not None else ""
        assert mapping.jwt_issuer == issuer
        assert _mapping_info(first, mapping.id).jwt_issuer == issuer
        assert _mapping_row(first, mapping.id).jwt_issuer == expected_issuer
        warm_first: Final = _assert_chat(first, model, token, prompts[0], key_hash)
        warm_second: Final = _assert_chat(second, model, token, prompts[1], key_hash)
        assert warm_first.id != warm_second.id
        warm_spend_count: Final = _spend_count(first, key_hash)
        assert warm_spend_count == 2

        deletion: Final = first.request("POST", "/key/delete", {"keys": [key]})
        assert deletion.status_code == 200, deletion.text
        assert _DeleteResponse.model_validate_json(deletion.content).deleted_keys == (key,), deletion.text
        assert read_rows('SELECT id FROM "LiteLLM_JWTKeyMapping" WHERE id=%s', (mapping.id,)) == []
        second_refusal: Final = eventually(
            lambda: _chat(second, model, token, prompts[2]),
            lambda response: response.status_code == 403,
            seconds=3,
        )
        _assert_unmapped(second_refusal, client_id)
        first_refusal: Final = _chat(first, model, token, prompts[3])
        _assert_unmapped(first_refusal, client_id)
        assert _spend_count(first, key_hash) == warm_spend_count
        _assert_upstream_requests(upstream, tuple(_expected_chat_request(backend, prompt) for prompt in prompts[:2]))


def test_a_cached_mapping_to_a_key_row_that_is_gone_answers_the_generic_401_on_both_workers(
    jwt_lifecycle: _JWTLifecycleRuntime,
) -> None:
    first: Final = jwt_lifecycle.first
    second: Final = jwt_lifecycle.second
    upstream: Final = jwt_lifecycle.upstream
    model: Final = jwt_lifecycle.model
    client_id: Final = "integration-client-" + uuid.uuid4().hex
    prompts: Final = (
        "stale mapping warm primary " + uuid.uuid4().hex,
        "stale mapping warm peer " + uuid.uuid4().hex,
        "stale mapping primary " + uuid.uuid4().hex,
        "stale mapping peer " + uuid.uuid4().hex,
    )
    token: Final = _signed_jwt(jwt_lifecycle.private_key_a, client_id, jwt_lifecycle.issuer_a)
    upstream.drain()
    with first.scenario() as scenario:
        key: Final = _create_key(first, model)
        scenario.cleanups.callback(delete_key_if_present, first, key)
        key_hash: Final = sha256(key.encode()).hexdigest()
        mapping: Final = _create_mapping(first, key, client_id)
        scenario.cleanups.callback(_delete_mapping_if_present, first, mapping.id)
        _assert_chat(first, model, token, prompts[0], key_hash)
        _assert_chat(second, model, token, prompts[1], key_hash)

        update: Final = first.request("POST", "/key/update", {"key": key, "metadata": {"rotated": "out-of-band"}})
        assert update.status_code == 200, update.text
        write_rows('DELETE FROM "LiteLLM_VerificationToken" WHERE token=%s', (key_hash,))
        assert read_rows('SELECT id FROM "LiteLLM_JWTKeyMapping" WHERE id=%s', (mapping.id,)) == []

        second_refusal: Final = eventually(
            lambda: _chat(second, model, token, prompts[3]),
            lambda response: response.status_code == 401,
            seconds=3,
        )
        first_refusal: Final = _chat(first, model, token, prompts[2])
        for refusal in (first_refusal, second_refusal):
            assert refusal.status_code == 401, refusal.text
            assert _ErrorResponse.model_validate_json(refusal.content).error == _UNKNOWN_KEY_ERROR, refusal.text
            assert key_hash not in refusal.text
        _assert_upstream_requests(upstream, tuple(_expected_chat_request(model, prompt) for prompt in prompts[:2]))


def test_issuer_scoped_mappings_with_the_same_claim_resolve_per_issuer(
    jwt_lifecycle: _JWTLifecycleRuntime,
) -> None:
    candidate: Final = jwt_lifecycle.first
    upstream: Final = jwt_lifecycle.upstream
    model: Final = jwt_lifecycle.model
    backend: Final = model
    client_id: Final = "integration-client-" + uuid.uuid4().hex
    issuer_a: Final = jwt_lifecycle.issuer_a
    issuer_b: Final = jwt_lifecycle.issuer_b
    prompts: Final = tuple(f"issuer mapping {index} {uuid.uuid4().hex}" for index in range(4))
    upstream.drain()

    with candidate.scenario() as scenario:
        key_a: Final = _create_key(candidate, model)
        key_b: Final = _create_key(candidate, model)
        scenario.cleanups.callback(delete_key_if_present, candidate, key_a)
        scenario.cleanups.callback(delete_key_if_present, candidate, key_b)
        hash_a: Final = sha256(key_a.encode()).hexdigest()
        hash_b: Final = sha256(key_b.encode()).hexdigest()
        mapping_a: Final = _create_mapping(candidate, key_a, client_id, issuer_a)
        mapping_b: Final = _create_mapping(candidate, key_b, client_id, issuer_b, spelling="token")
        scenario.cleanups.callback(_delete_mapping_if_present, candidate, mapping_a.id)
        scenario.cleanups.callback(_delete_mapping_if_present, candidate, mapping_b.id)

        _assert_chat(
            candidate,
            model,
            _signed_jwt(jwt_lifecycle.private_key_a, client_id, issuer_a),
            prompts[0],
            hash_a,
        )
        _assert_chat(
            candidate,
            model,
            _signed_jwt(jwt_lifecycle.private_key_b, client_id, issuer_b),
            prompts[1],
            hash_b,
        )
        _assert_chat(
            candidate,
            model,
            _signed_jwt(jwt_lifecycle.private_key_a, client_id, issuer_a),
            prompts[2],
            hash_a,
        )
        _assert_chat(
            candidate,
            model,
            _signed_jwt(jwt_lifecycle.private_key_b, client_id, issuer_b),
            prompts[3],
            hash_b,
        )
        rows: Final = TypeAdapter(tuple[_JWTMappingRow, ...]).validate_python(
            read_rows(
                'SELECT id, token, jwt_issuer, is_active FROM "LiteLLM_JWTKeyMapping" '
                "WHERE jwt_claim_name=%s AND jwt_claim_value=%s ORDER BY jwt_issuer",
                (_CLIENT_CLAIM, client_id),
            )
        )
        assert rows == (
            _JWTMappingRow(id=mapping_a.id, token=hash_a, jwt_issuer=issuer_a, is_active=True),
            _JWTMappingRow(id=mapping_b.id, token=hash_b, jwt_issuer=issuer_b, is_active=True),
        )
        _assert_upstream_requests(
            upstream,
            tuple(_expected_chat_request(backend, prompt) for prompt in prompts),
        )


def test_repointing_a_warmed_mapping_to_a_new_issuer_moves_it_on_both_workers(
    jwt_lifecycle: _JWTLifecycleRuntime,
) -> None:
    first: Final = jwt_lifecycle.first
    second: Final = jwt_lifecycle.second
    upstream: Final = jwt_lifecycle.upstream
    model: Final = jwt_lifecycle.model
    issuer_a: Final = jwt_lifecycle.issuer_a
    issuer_b: Final = jwt_lifecycle.issuer_b
    client_id: Final = "integration-client-" + uuid.uuid4().hex
    prompts: Final = tuple(f"mapping repoint {index} {uuid.uuid4().hex}" for index in range(4))
    upstream.drain()

    with first.scenario() as scenario:
        key: Final = _create_key(first, model)
        scenario.cleanups.callback(delete_key_if_present, first, key)
        key_hash: Final = sha256(key.encode()).hexdigest()
        mapping: Final = _create_mapping(first, key, client_id, issuer_a)
        scenario.cleanups.callback(_delete_mapping_if_present, first, mapping.id)
        assert _mapping_row(first, mapping.id) == _JWTMappingRow(
            id=mapping.id,
            token=key_hash,
            jwt_issuer=issuer_a,
            is_active=True,
        )

        new_identity_token: Final = _signed_jwt(jwt_lifecycle.private_key_b, client_id, issuer_b)
        _assert_unmapped(_chat(first, model, new_identity_token, prompts[0]), client_id)
        _assert_unmapped(_chat(second, model, new_identity_token, prompts[1]), client_id)

        old_token: Final = _signed_jwt(jwt_lifecycle.private_key_a, client_id, issuer_a)
        _assert_chat(first, model, old_token, prompts[2], key_hash)
        _assert_chat(second, model, old_token, prompts[3], key_hash)
        assert _spend_count(first, key_hash) == 2

        update: Final = first.request(
            "POST",
            "/jwt/key/mapping/update",
            {"id": mapping.id, "jwt_issuer": issuer_b},
        )
        assert update.status_code == 200, update.text
        updated: Final = _JWTMappingResponse.model_validate_json(update.content)
        assert updated.jwt_claim_value == client_id, update.text
        assert updated.jwt_issuer == issuer_b, update.text
        assert _mapping_info(first, mapping.id) == updated
        assert _mapping_row(first, mapping.id) == _JWTMappingRow(
            id=mapping.id,
            token=key_hash,
            jwt_issuer=issuer_b,
            is_active=True,
        )

        _assert_chat(first, model, new_identity_token, prompts[0], key_hash)
        _assert_chat(second, model, new_identity_token, prompts[1], key_hash)
        assert _spend_count(first, key_hash) == 4
        _assert_unmapped(_chat(first, model, old_token, prompts[2]), client_id)
        _assert_unmapped(_chat(second, model, old_token, prompts[3]), client_id)
        assert _spend_count(first, key_hash) == 4
        _assert_upstream_requests(
            upstream,
            tuple(_expected_chat_request(model, prompt) for prompt in prompts),
        )


class _EndUserSpendRow(BaseModel):
    api_key: str
    end_user: str | None


@dataclass(frozen=True, slots=True)
class _EndUserClaimRuntime:
    gateway: Gateway
    model: str
    private_key: rsa.RSAPrivateKey


@dataclass(frozen=True, slots=True)
class _MappedClient:
    client_id: str
    key: str
    key_hash: str


def _end_user_claim_config(directory: Path, model: str, upstream_url: str) -> Path:
    config: Final = directory / "jwt_end_user_claim_config.yaml"
    config.write_text(
        json.dumps(
            {
                "model_list": (
                    {
                        "model_name": model,
                        "litellm_params": {
                            "model": "openai/gpt-4o-mini",
                            "api_base": f"{upstream_url}/v1",
                            "api_key": "integration-provider-key",
                            "input_cost_per_token": 0.001,
                            "output_cost_per_token": 0.002,
                        },
                    },
                ),
                "general_settings": {
                    "master_key": "os.environ/LITELLM_MASTER_KEY",
                    "database_url": "os.environ/DATABASE_URL",
                    "store_model_in_db": True,
                    "disable_model_info_refresh": True,
                    "proxy_batch_write_at": 1,
                    "proxy_batch_polling_interval": 1,
                    "enable_jwt_auth": True,
                    "litellm_jwtauth": {
                        "user_id_jwt_field": "sub",
                        "user_id_upsert": True,
                        "virtual_key_claim_field": _CLIENT_CLAIM,
                        "unregistered_jwt_client_behavior": "auto_register",
                        "end_user_id_jwt_field": _END_USER_CLAIM,
                    },
                },
            }
        )
    )
    return config


@pytest.fixture(scope="module")
def end_user_claims(tmp_path_factory: pytest.TempPathFactory) -> Iterator[_EndUserClaimRuntime]:
    directory: Final = tmp_path_factory.mktemp("jwt-end-user-claim")
    model: Final = "jwt-end-user-claim-" + uuid.uuid4().hex
    private_key: Final = rsa.generate_private_key(public_exponent=65537, key_size=2048)
    with gateway_from_environment() as base, wire_server(_jwks_respond(private_key)) as jwks:
        config: Final = _end_user_claim_config(directory, model, base.upstream_url)
        overrides: Final = {"JWT_PUBLIC_KEY_URL": f"{jwks.url}/jwks", "JWT_AUDIENCE": _AUDIENCE}
        with owned_proxy(base, directory / "proxy", overrides, config=config) as candidate:
            eventually(lambda: _model_is_available(candidate, model), bool, seconds=30)
            yield _EndUserClaimRuntime(gateway=candidate, model=model, private_key=private_key)


def _mapped_client(runtime: _EndUserClaimRuntime, scenario: Scenario) -> _MappedClient:
    client_id: Final = "integration-client-" + uuid.uuid4().hex
    key: Final = _create_key(runtime.gateway, runtime.model)
    scenario.cleanups.callback(delete_key_if_present, runtime.gateway, key)
    mapping: Final = _create_mapping(runtime.gateway, key, client_id)
    scenario.cleanups.callback(_delete_mapping_if_present, runtime.gateway, mapping.id)
    return _MappedClient(client_id=client_id, key=key, key_hash=sha256(key.encode()).hexdigest())


def _end_user_chat(
    runtime: _EndUserClaimRuntime, token: str, *, body_user: str | None = None, header_user: str | None = None
) -> httpx.Response:
    return runtime.gateway.request(
        "POST",
        "/v1/chat/completions",
        {
            **_chat_body(runtime.model, "end user claim " + uuid.uuid4().hex),
            **({"user": body_user} if body_user is not None else {}),
        },
        key=token,
        headers={"x-litellm-end-user-id": header_user} if header_user is not None else None,
    )


def _billed_end_user(response: httpx.Response) -> _EndUserSpendRow:
    assert response.status_code == 200, response.text
    identity: Final = _ChatResponse.model_validate_json(response.content).id
    rows: Final = TypeAdapter(tuple[_EndUserSpendRow, ...]).validate_python(
        eventually(
            lambda: read_rows('SELECT api_key, end_user FROM "LiteLLM_SpendLogs" WHERE request_id=%s', (identity,)),
            lambda values: len(values) == 1,
            seconds=70,
        )
    )
    return rows[0]


def _customer_ids(user_id: str) -> list[dict[str, JsonValue]]:
    return read_rows('SELECT user_id FROM "LiteLLM_EndUserTable" WHERE user_id=%s', (user_id,))


def _delete_customer_if_present(gateway: Gateway, user_id: str) -> None:
    if not _customer_ids(user_id):
        return
    response: Final = gateway.request("POST", "/customer/delete", {"user_ids": [user_id]})
    assert response.status_code == 200, response.text


def _forget_auto_registered_client(client_id: str, subject: str) -> None:
    write_rows(
        'DELETE FROM "LiteLLM_VerificationToken" WHERE token IN '
        '(SELECT token FROM "LiteLLM_JWTKeyMapping" WHERE jwt_claim_value = %s)',
        (client_id,),
    )
    write_rows('DELETE FROM "LiteLLM_JWTKeyMapping" WHERE jwt_claim_value = %s', (client_id,))
    write_rows('DELETE FROM "LiteLLM_UserTable" WHERE user_id = %s', (subject,))


def test_a_mapped_jwt_bills_and_registers_its_end_user_claim(end_user_claims: _EndUserClaimRuntime) -> None:
    gateway: Final = end_user_claims.gateway
    end_user: Final = "integration-end-user-" + uuid.uuid4().hex
    with gateway.scenario() as scenario:
        client: Final = _mapped_client(end_user_claims, scenario)
        scenario.cleanups.callback(_delete_customer_if_present, gateway, end_user)
        token: Final = _signed_jwt(end_user_claims.private_key, client.client_id, end_user=end_user)

        billed: Final = _billed_end_user(_end_user_chat(end_user_claims, token))

        assert billed == _EndUserSpendRow(api_key=client.key_hash, end_user=end_user)
        customers: Final = eventually(lambda: _customer_ids(end_user), lambda rows: len(rows) == 1, seconds=70)
        assert customers == [{"user_id": end_user}]


@pytest.mark.parametrize("spelling", ("body", "header"))
def test_a_mapped_jwt_end_user_claim_wins_over_the_caller_supplied_end_user(
    end_user_claims: _EndUserClaimRuntime, spelling: str
) -> None:
    gateway: Final = end_user_claims.gateway
    end_user: Final = "integration-end-user-" + uuid.uuid4().hex
    supplied: Final = "integration-supplied-" + uuid.uuid4().hex
    with gateway.scenario() as scenario:
        client: Final = _mapped_client(end_user_claims, scenario)
        scenario.cleanups.callback(_delete_customer_if_present, gateway, end_user)
        scenario.cleanups.callback(_delete_customer_if_present, gateway, supplied)
        token: Final = _signed_jwt(end_user_claims.private_key, client.client_id, end_user=end_user)

        response: Final = _end_user_chat(
            end_user_claims,
            token,
            body_user=supplied if spelling == "body" else None,
            header_user=supplied if spelling == "header" else None,
        )

        assert _billed_end_user(response) == _EndUserSpendRow(api_key=client.key_hash, end_user=end_user)


@pytest.mark.parametrize("claim", ("", None), ids=("empty", "missing"))
@pytest.mark.parametrize("spelling", ("body", "header"))
def test_a_mapped_jwt_without_an_end_user_claim_falls_back_to_the_caller_supplied_end_user(
    end_user_claims: _EndUserClaimRuntime, spelling: str, claim: str | None
) -> None:
    gateway: Final = end_user_claims.gateway
    supplied: Final = "integration-supplied-" + uuid.uuid4().hex
    with gateway.scenario() as scenario:
        client: Final = _mapped_client(end_user_claims, scenario)
        scenario.cleanups.callback(_delete_customer_if_present, gateway, supplied)
        token: Final = _signed_jwt(end_user_claims.private_key, client.client_id, end_user=claim)

        response: Final = _end_user_chat(
            end_user_claims,
            token,
            body_user=supplied if spelling == "body" else None,
            header_user=supplied if spelling == "header" else None,
        )

        assert _billed_end_user(response) == _EndUserSpendRow(api_key=client.key_hash, end_user=supplied)


def test_an_auto_registered_client_keeps_its_end_user_claim_after_the_first_request(
    end_user_claims: _EndUserClaimRuntime,
) -> None:
    gateway: Final = end_user_claims.gateway
    client_id: Final = "integration-client-" + uuid.uuid4().hex
    subject: Final = "integration-user-" + uuid.uuid4().hex
    end_user: Final = "integration-end-user-" + uuid.uuid4().hex
    with gateway.scenario() as scenario:
        scenario.cleanups.callback(_forget_auto_registered_client, client_id, subject)
        scenario.cleanups.callback(_delete_customer_if_present, gateway, end_user)
        token: Final = _signed_jwt(end_user_claims.private_key, client_id, subject=subject, end_user=end_user)

        registering: Final = _billed_end_user(_end_user_chat(end_user_claims, token))
        mapped: Final = _billed_end_user(_end_user_chat(end_user_claims, token))

        mappings: Final = read_rows(
            'SELECT token FROM "LiteLLM_JWTKeyMapping" WHERE jwt_claim_name=%s AND jwt_claim_value=%s',
            (_CLIENT_CLAIM, client_id),
        )
        assert len(mappings) == 1, mappings
        expected: Final = _EndUserSpendRow(api_key=string_value(mappings[0]["token"]), end_user=end_user)
        assert (registering, mapped) == (expected, expected)


def test_a_mapped_jwt_end_user_claim_naming_an_over_budget_customer_is_refused_like_a_body_user(
    end_user_claims: _EndUserClaimRuntime,
) -> None:
    gateway: Final = end_user_claims.gateway
    end_user: Final = "integration-end-user-" + uuid.uuid4().hex
    with gateway.scenario() as scenario:
        client: Final = _mapped_client(end_user_claims, scenario)
        created: Final = gateway.request(
            "POST", "/customer/new", {"user_id": end_user, "max_budget": 0.05, "spend": 1.0}
        )
        assert created.status_code == 200, created.text
        scenario.cleanups.callback(_delete_customer_if_present, gateway, end_user)
        token: Final = _signed_jwt(end_user_claims.private_key, client.client_id, end_user=end_user)

        by_body: Final = _end_user_chat(end_user_claims, client.key, body_user=end_user)
        by_claim: Final = _end_user_chat(end_user_claims, token)

        assert by_body.status_code == 422, by_body.text
        assert object_value(object_value(by_body.json())["error"])["type"] == "budget_exceeded", by_body.text
        assert (by_claim.status_code, by_claim.json()) == (by_body.status_code, by_body.json())


def test_a_mapped_jwt_end_user_claim_is_budget_checked_instead_of_an_over_budget_body_user(
    end_user_claims: _EndUserClaimRuntime,
) -> None:
    gateway: Final = end_user_claims.gateway
    end_user: Final = "integration-end-user-" + uuid.uuid4().hex
    over_budget: Final = "integration-over-budget-" + uuid.uuid4().hex
    with gateway.scenario() as scenario:
        client: Final = _mapped_client(end_user_claims, scenario)
        created: Final = gateway.request(
            "POST", "/customer/new", {"user_id": over_budget, "max_budget": 0.05, "spend": 1.0}
        )
        assert created.status_code == 200, created.text
        scenario.cleanups.callback(_delete_customer_if_present, gateway, over_budget)
        scenario.cleanups.callback(_delete_customer_if_present, gateway, end_user)
        token: Final = _signed_jwt(end_user_claims.private_key, client.client_id, end_user=end_user)

        refused: Final = _end_user_chat(end_user_claims, client.key, body_user=over_budget)
        served: Final = _end_user_chat(end_user_claims, token, body_user=over_budget)

        assert refused.status_code == 422, refused.text
        assert _billed_end_user(served) == _EndUserSpendRow(api_key=client.key_hash, end_user=end_user)


@pytest.mark.parametrize("spelling", ("body", "header"))
def test_a_mapped_jwt_without_an_end_user_claim_budget_checks_the_caller_supplied_end_user(
    end_user_claims: _EndUserClaimRuntime, spelling: str
) -> None:
    gateway: Final = end_user_claims.gateway
    over_budget: Final = "integration-over-budget-" + uuid.uuid4().hex
    with gateway.scenario() as scenario:
        client: Final = _mapped_client(end_user_claims, scenario)
        created: Final = gateway.request(
            "POST", "/customer/new", {"user_id": over_budget, "max_budget": 0.05, "spend": 1.0}
        )
        assert created.status_code == 200, created.text
        scenario.cleanups.callback(_delete_customer_if_present, gateway, over_budget)
        token: Final = _signed_jwt(end_user_claims.private_key, client.client_id)

        refused: Final = _end_user_chat(
            end_user_claims,
            token,
            body_user=over_budget if spelling == "body" else None,
            header_user=over_budget if spelling == "header" else None,
        )

        assert refused.status_code == 422, refused.text
        assert object_value(object_value(refused.json())["error"])["type"] == "budget_exceeded", refused.text