Skip to content

The fastest, litest AI Gateway. Rust core with Python SDK. Call 100+ LLM APIs in OpenAI (or native) format with cost tracking, guardrails, load balancing, and logging [Bedrock, Azure, OpenAI, Anthropic, OpenAI, VertexAI, vLLM, Nvidia NIM]

17.6 KB
Raw Blame
import uuid
from collections.abc import Mapping
from typing import Final

import httpx
import pytest
from integration._support.client import Gateway, Scenario, delete_key_if_present, object_value, string_value
from integration._support.database import read_rows
from integration.management._batch_file_caps import DOWNLOADS, RECORDS, UPLOADS, config_entry, hashed
from pydantic import JsonValue

TOKENS: Final = "batch_enqueued_token_limit"
LIMITS: Final = (TOKENS, RECORDS, UPLOADS, DOWNLOADS)
FILE_LIMITS: Final = (RECORDS, UPLOADS, DOWNLOADS)
EVERY_LIMIT: Final[Mapping[str, JsonValue]] = {TOKENS: 9000, RECORDS: 7, UPLOADS: 6, DOWNLOADS: 5}
BULK_ROUTE: Final = "/management/v1/users/bulk"


def _refusal(setting: str, entity: str) -> str:
    return (
        f"Only proxy admins can set {setting} on a {entity}. "
        "It limits what the holder can do with batches, so the holder cannot raise it."
    )


def _assert_refused(response: httpx.Response, setting: str, entity: str) -> None:
    assert response.status_code == 403, response.text
    error: Final = object_value(response.json()["error"])
    assert error["message"] == str({"error": _refusal(setting, entity)}), response.text
    assert error["code"] == "403", response.text


def _key_metadata(token: str) -> JsonValue:
    rows: Final = read_rows('SELECT metadata FROM "LiteLLM_VerificationToken" WHERE token = %s', (hashed(token),))
    assert len(rows) == 1, rows
    return rows[0]["metadata"]


def _team_metadata(team: str) -> JsonValue:
    rows: Final = read_rows('SELECT metadata FROM "LiteLLM_TeamTable" WHERE team_id = %s', (team,))
    assert len(rows) == 1, rows
    return rows[0]["metadata"]


def _user_metadata(user: str) -> JsonValue:
    rows: Final = read_rows('SELECT metadata FROM "LiteLLM_UserTable" WHERE user_id = %s', (user,))
    assert len(rows) == 1, rows
    return rows[0]["metadata"]


def _user_key_metadata(user: str) -> list[JsonValue]:
    rows: Final = read_rows('SELECT metadata FROM "LiteLLM_VerificationToken" WHERE user_id = %s', (user,))
    return [row["metadata"] for row in rows]


def _team_admin_key(scenario: Scenario, team: str) -> str:
    return scenario.key(user_id=scenario.member(team, role="admin"), team_id=team)


def _org_admin_key(scenario: Scenario, organization: str) -> str:
    return scenario.key(user_id=scenario.org_member(organization, role="org_admin"))


def _replaceable_key(gateway: Gateway, scenario: Scenario, **fields: JsonValue) -> str:
    key: Final = string_value(gateway.post("/key/generate", fields)["key"])
    scenario.cleanups.callback(delete_key_if_present, gateway, key)
    return key


def _drop_created_key(scenario: Scenario, response: httpx.Response) -> None:
    if response.status_code == 200 and response.json().get("key"):
        scenario.cleanups.callback(scenario.delete_key, string_value(response.json()["key"]))


def _drop_created_user(scenario: Scenario, response: httpx.Response, user: str) -> None:
    if response.status_code == 200:
        scenario.cleanups.callback(scenario.delete_user, user)
    _drop_created_key(scenario, response)


def _drop_bulk_rows(scenario: Scenario, response: httpx.Response) -> None:
    if response.status_code != 200:
        return
    for row in response.json()["data"]:
        if row["success"]:
            scenario.cleanups.callback(scenario.delete_user, string_value(row["user_id"]))
        if row["key"]:
            scenario.cleanups.callback(scenario.delete_key, string_value(row["key"]))


@pytest.mark.parametrize("setting", LIMITS)
def test_team_admin_cannot_put_a_batch_limit_on_a_new_key(gateway: Gateway, setting: str) -> None:
    with gateway.scenario() as scenario:
        team: Final = scenario.team()
        caller: Final = _team_admin_key(scenario, team)
        alias: Final = f"batch-limit-{uuid.uuid4().hex}"
        refused: Final = gateway.request(
            "POST", "/key/generate", {"team_id": team, "key_alias": alias, "metadata": {setting: 5}}, key=caller
        )
        _drop_created_key(scenario, refused)
        _assert_refused(refused, setting, "key")
        assert read_rows('SELECT token FROM "LiteLLM_VerificationToken" WHERE key_alias = %s', (alias,)) == []
        plain: Final = gateway.request("POST", "/key/generate", {"team_id": team, "key_alias": alias}, key=caller)
        _drop_created_key(scenario, plain)
        assert plain.status_code == 200, plain.text
        assert _key_metadata(string_value(plain.json()["key"])) == {}


@pytest.mark.parametrize("setting", LIMITS)
def test_team_admin_cannot_put_a_batch_limit_on_an_existing_key(gateway: Gateway, setting: str) -> None:
    with gateway.scenario() as scenario:
        team: Final = scenario.team()
        caller: Final = _team_admin_key(scenario, team)
        key: Final = scenario.key(team_id=team, metadata={"owner": "batch-limit-audit"})
        refused: Final = gateway.request(
            "POST", "/key/update", {"key": key, "metadata": {"owner": "batch-limit-audit", setting: 5}}, key=caller
        )
        _assert_refused(refused, setting, "key")
        assert _key_metadata(key) == {"owner": "batch-limit-audit"}


@pytest.mark.parametrize("setting", LIMITS)
def test_team_admin_cannot_put_a_batch_limit_on_a_regenerated_key(gateway: Gateway, setting: str) -> None:
    with gateway.scenario() as scenario:
        team: Final = scenario.team()
        caller: Final = _team_admin_key(scenario, team)
        key: Final = _replaceable_key(gateway, scenario, team_id=team)
        refused: Final = gateway.request("POST", "/key/regenerate", {"key": key, "metadata": {setting: 5}}, key=caller)
        _drop_created_key(scenario, refused)
        _assert_refused(refused, setting, "key")
        assert _key_metadata(key) == {}


@pytest.mark.parametrize("setting", LIMITS)
def test_org_admin_cannot_put_a_batch_limit_on_an_existing_team(gateway: Gateway, setting: str) -> None:
    with gateway.scenario() as scenario:
        organization: Final = scenario.organization()
        caller: Final = _org_admin_key(scenario, organization)
        team: Final = scenario.team(organization_id=organization, metadata={"owner": "batch-limit-audit"})
        refused: Final = gateway.request(
            "POST",
            "/team/update",
            {"team_id": team, "metadata": {"owner": "batch-limit-audit", setting: 5}},
            key=caller,
        )
        _assert_refused(refused, setting, "team")
        assert _team_metadata(team) == {"owner": "batch-limit-audit"}


@pytest.mark.parametrize("setting", LIMITS)
def test_org_admin_cannot_put_a_batch_limit_on_a_new_team(gateway: Gateway, setting: str) -> None:
    with gateway.scenario() as scenario:
        organization: Final = scenario.organization()
        caller: Final = _org_admin_key(scenario, organization)
        alias: Final = f"batch-limit-{uuid.uuid4().hex}"
        refused: Final = gateway.request(
            "POST",
            "/team/new",
            {"team_alias": alias, "organization_id": organization, "metadata": {setting: 5}},
            key=caller,
        )
        if refused.status_code == 200:
            scenario.cleanups.callback(scenario.delete_team, string_value(refused.json()["team_id"]))
        _assert_refused(refused, setting, "team")
        assert read_rows('SELECT team_id FROM "LiteLLM_TeamTable" WHERE team_alias = %s', (alias,)) == []
        plain: Final = gateway.request(
            "POST", "/team/new", {"team_alias": alias, "organization_id": organization}, key=caller
        )
        if plain.status_code == 200:
            scenario.cleanups.callback(scenario.delete_team, string_value(plain.json()["team_id"]))
        assert plain.status_code == 200, plain.text
        assert _team_metadata(string_value(plain.json()["team_id"])) == {}


@pytest.mark.parametrize("setting", LIMITS)
def test_org_admin_cannot_put_a_batch_limit_on_a_new_users_key(gateway: Gateway, setting: str) -> None:
    with gateway.scenario() as scenario:
        organization: Final = scenario.organization()
        caller: Final = _org_admin_key(scenario, organization)
        refused_user, keyless_user, plain_user = (f"batch-limit-{uuid.uuid4().hex}" for _ in range(3))
        refused: Final = gateway.request(
            "POST",
            "/user/new",
            {"user_id": refused_user, "organization_id": organization, "metadata": {setting: 5}},
            key=caller,
        )
        _drop_created_user(scenario, refused, refused_user)
        _assert_refused(refused, setting, "key")
        assert read_rows('SELECT user_id FROM "LiteLLM_UserTable" WHERE user_id = %s', (refused_user,)) == []
        assert _user_key_metadata(refused_user) == []
        keyless: Final = gateway.request(
            "POST",
            "/user/new",
            {
                "user_id": keyless_user,
                "organization_id": organization,
                "metadata": {setting: 5},
                "auto_create_key": False,
            },
            key=caller,
        )
        _drop_created_user(scenario, keyless, keyless_user)
        assert keyless.status_code == 200, keyless.text
        assert _user_metadata(keyless_user) == {setting: 5}
        assert _user_key_metadata(keyless_user) == []
        plain: Final = gateway.request(
            "POST", "/user/new", {"user_id": plain_user, "organization_id": organization}, key=caller
        )
        _drop_created_user(scenario, plain, plain_user)
        assert plain.status_code == 200, plain.text
        assert _user_key_metadata(plain_user) == [{}]


@pytest.mark.parametrize("setting", LIMITS)
def test_bulk_user_creation_refuses_only_the_row_that_puts_a_batch_limit_on_a_key(
    gateway: Gateway, setting: str
) -> None:
    with gateway.scenario() as scenario:
        caller: Final = scenario.key(user_id=scenario.user(user_role="internal_user"), allowed_routes=[BULK_ROUTE])
        refused_user, keyless_user, plain_user = (f"batch-limit-{uuid.uuid4().hex}" for _ in range(3))
        response: Final = gateway.request(
            "POST",
            BULK_ROUTE,
            {
                "users": [
                    {"user_id": refused_user, "metadata": {setting: 5}, "auto_create_key": True},
                    {"user_id": keyless_user, "metadata": {setting: 5}},
                    {"user_id": plain_user, "auto_create_key": True},
                ]
            },
            key=caller,
        )
        _drop_bulk_rows(scenario, response)
        assert response.status_code == 200, response.text
        refused, keyless, plain = response.json()["data"]
        assert refused == {
            "user_id": refused_user,
            "user_email": None,
            "success": False,
            "teams": None,
            "key": None,
            "error": _refusal(setting, "key"),
        }, response.text
        assert (keyless["success"], keyless["key"], keyless["error"]) == (True, None, None), response.text
        assert (plain["success"], plain["error"]) == (True, None), response.text
        assert response.json()["meta"] == {"total_requested": 3, "created": 2, "failed": 1}, response.text
        assert read_rows('SELECT user_id FROM "LiteLLM_UserTable" WHERE user_id = %s', (refused_user,)) == []
        assert _user_key_metadata(refused_user) == []
        assert _user_metadata(keyless_user) == {setting: 5}
        assert _user_key_metadata(keyless_user) == []
        assert _user_key_metadata(plain_user) == [{}]
        assert _key_metadata(string_value(plain["key"])) == {}


def test_proxy_admin_sets_every_batch_limit_on_every_route(gateway: Gateway) -> None:
    limits: Final = dict(EVERY_LIMIT)
    with gateway.scenario() as scenario:
        generated: Final = scenario.key(metadata=limits)
        assert _key_metadata(generated) == limits
        updated: Final = scenario.key()
        gateway.post("/key/update", {"key": updated, "metadata": limits})
        assert _key_metadata(updated) == limits
        replaced: Final = _replaceable_key(gateway, scenario)
        regenerated: Final = string_value(gateway.post("/key/regenerate", {"key": replaced, "metadata": limits})["key"])
        scenario.cleanups.callback(scenario.delete_key, regenerated)
        assert _key_metadata(regenerated) == limits
        created_team: Final = scenario.team(metadata=limits)
        assert _team_metadata(created_team) == limits
        updated_team: Final = scenario.team()
        gateway.post("/team/update", {"team_id": updated_team, "metadata": limits})
        assert _team_metadata(updated_team) == limits
        new_user, bulk_user = (f"batch-limit-{uuid.uuid4().hex}" for _ in range(2))
        created_user: Final = gateway.request("POST", "/user/new", {"user_id": new_user, "metadata": limits})
        _drop_created_user(scenario, created_user, new_user)
        assert created_user.status_code == 200, created_user.text
        assert _user_key_metadata(new_user) == [limits]
        bulk: Final = gateway.request(
            "POST", BULK_ROUTE, {"users": [{"user_id": bulk_user, "metadata": limits, "auto_create_key": True}]}
        )
        _drop_bulk_rows(scenario, bulk)
        assert bulk.status_code == 200, bulk.text
        assert bulk.json()["meta"] == {"total_requested": 1, "created": 1, "failed": 0}, bulk.text
        assert _user_key_metadata(bulk_user) == [limits]


RESENDS: Final = (
    pytest.param({}, id="dropped"),
    pytest.param({UPLOADS: 4}, id="lowered"),
    pytest.param({UPLOADS: 6}, id="raised"),
    pytest.param({UPLOADS: "5"}, id="resent-as-a-string"),
    pytest.param({UPLOADS: None}, id="resent-as-null"),
)


@pytest.mark.parametrize("metadata", RESENDS)
def test_team_admin_may_resend_a_stored_key_limit_but_not_change_it(
    gateway: Gateway, metadata: Mapping[str, JsonValue]
) -> None:
    with gateway.scenario() as scenario:
        team: Final = scenario.team()
        caller: Final = _team_admin_key(scenario, team)
        key: Final = scenario.key(team_id=team, metadata={UPLOADS: 5})
        alias: Final = f"batch-limit-{uuid.uuid4().hex}"
        resent: Final = gateway.request(
            "POST", "/key/update", {"key": key, "key_alias": alias, "metadata": {UPLOADS: 5}}, key=caller
        )
        assert resent.status_code == 200, resent.text
        untouched: Final = gateway.request("POST", "/key/update", {"key": key, "key_alias": f"{alias}-2"}, key=caller)
        assert untouched.status_code == 200, untouched.text
        changed: Final = gateway.request("POST", "/key/update", {"key": key, "metadata": dict(metadata)}, key=caller)
        _assert_refused(changed, UPLOADS, "key")
        assert _key_metadata(key) == {UPLOADS: 5}
        assert read_rows('SELECT key_alias FROM "LiteLLM_VerificationToken" WHERE token = %s', (hashed(key),)) == [
            {"key_alias": f"{alias}-2"}
        ]


@pytest.mark.parametrize("metadata", RESENDS)
def test_org_admin_may_resend_a_stored_team_limit_but_not_change_it(
    gateway: Gateway, metadata: Mapping[str, JsonValue]
) -> None:
    with gateway.scenario() as scenario:
        organization: Final = scenario.organization()
        caller: Final = _org_admin_key(scenario, organization)
        team: Final = scenario.team(organization_id=organization, metadata={UPLOADS: 5})
        resent: Final = gateway.request(
            "POST", "/team/update", {"team_id": team, "tpm_limit": 5000, "metadata": {UPLOADS: 5}}, key=caller
        )
        assert resent.status_code == 200, resent.text
        untouched: Final = gateway.request("POST", "/team/update", {"team_id": team, "tpm_limit": 6000}, key=caller)
        assert untouched.status_code == 200, untouched.text
        changed: Final = gateway.request(
            "POST", "/team/update", {"team_id": team, "metadata": dict(metadata)}, key=caller
        )
        _assert_refused(changed, UPLOADS, "team")
        assert read_rows('SELECT metadata, tpm_limit FROM "LiteLLM_TeamTable" WHERE team_id = %s', (team,)) == [
            {"metadata": {UPLOADS: 5}, "tpm_limit": 6000}
        ]


@pytest.mark.parametrize("setting", FILE_LIMITS)
def test_config_list_shows_each_file_limit_as_an_unset_integer(gateway: Gateway, setting: str) -> None:
    entry: Final = config_entry(gateway, setting)
    assert (entry["field_type"], entry["field_value"], entry["stored_in_db"], entry["editable"]) == (
        "Integer",
        None,
        None,
        True,
    ), entry


@pytest.mark.parametrize("setting", FILE_LIMITS)
def test_config_update_refuses_a_zero_file_limit(gateway: Gateway, setting: str) -> None:
    response: Final = gateway.request("POST", "/config/update", {"general_settings": {setting: 0}})
    assert response.status_code == 422, response.text
    assert response.json() == {
        "detail": [
            {
                "type": "greater_than",
                "loc": ["body", "general_settings", setting],
                "msg": "Input should be greater than 0",
            }
        ]
    }, response.text
    assert config_entry(gateway, setting)["field_value"] is None


@pytest.mark.parametrize("setting", FILE_LIMITS)
@pytest.mark.parametrize(
    ("value", "kind"),
    [
        pytest.param(0, "int", id="zero"),
        pytest.param(-1, "int", id="negative"),
        pytest.param("abc", "str", id="word"),
        pytest.param(1.5, "float", id="fraction"),
        pytest.param([5], "list", id="list"),
        pytest.param("", "str", id="empty-string"),
    ],
)
def test_config_field_update_refuses_a_malformed_file_limit(
    gateway: Gateway, value: JsonValue, kind: str, setting: str
) -> None:
    response: Final = gateway.request(
        "POST",
        "/config/field/update",
        {"field_name": setting, "field_value": value, "config_type": "general_settings"},
    )
    assert response.status_code == 400, response.text
    assert response.json() == {"detail": {"error": f"Invalid type of field value=<class '{kind}'> passed in."}}
    assert config_entry(gateway, setting)["field_value"] is None