Skip to content

The fastest, litest AI Gateway. Rust core with Python SDK. Call 100+ LLM APIs in OpenAI (or native) format with cost tracking, guardrails, load balancing, and logging [Bedrock, Azure, OpenAI, Anthropic, OpenAI, VertexAI, vLLM, Nvidia NIM]

20.3 KB
Raw Blame
import json
import os
import stat
import subprocess
import sys
import uuid
from collections.abc import Mapping, Sequence
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
from typing import Final

import httpx
import pytest
from pydantic import JsonValue

from tests.integration._support.client import JSON_OBJECT, Gateway, Scenario, eventually, object_value, string_value
from tests.integration._support.process import owned_proxy, owned_proxy_process

pytestmark: Final = pytest.mark.shares_worker_state

CLIENT_VERSION: Final = "0.159.3"
FRESH_CONNECTION: Final = {"Connection": "close"}
CONSISTENT_READS: Final = 8
CONVERGENCE_SECONDS: Final = 60
BURST: Final = 24
CATALOG_FILENAME: Final = "litellm-models.json"
SHIM_STOCK_ENV: Final = "CODEX_SHIM_STOCK"
SHIM_LAUNCH_ENV: Final = "CODEX_SHIM_LAUNCH"
SHIM_SCRIPT: Final = """\
import json
import os
import sys

argv = sys.argv[1:]
if argv[-2:] == ["debug", "models"]:
    override = next((arg for arg in argv if arg.startswith("model_catalog_json=")), None)
    if override is None:
        sys.stdout.write(open(os.environ[%(stock)r], encoding="utf-8").read())
    else:
        catalog = json.load(open(json.loads(override.partition("=")[2]), encoding="utf-8"))
        if not catalog.get("models"):
            sys.stderr.write("empty model catalog\\n")
            sys.exit(2)
        sys.stdout.write(json.dumps(catalog))
    sys.exit(0)
recorded = {name: os.environ.get(name) for name in ("OPENAI_BASE_URL", "OPENAI_API_KEY", "LITELLM_PROXY_API_KEY")}
with open(os.environ[%(launch)r], "w", encoding="utf-8") as handle:
    json.dump({"argv": argv, "env": recorded}, handle)
""" % {"stock": SHIM_STOCK_ENV, "launch": SHIM_LAUNCH_ENV}


def _generic_tier(identity: str) -> dict[str, JsonValue]:
    return {"id": identity, "name": identity.capitalize(), "description": f"Sends service_tier={identity} upstream"}


def _catalog_response(proxy: Gateway, *, key: str | None = None) -> httpx.Response:
    return proxy.request(
        "GET", "/v1/models", key=key, params={"client_version": CLIENT_VERSION}, headers=FRESH_CONNECTION
    )


def _entries(response: httpx.Response) -> list[dict[str, JsonValue]]:
    assert response.status_code == 200, response.text
    body: Final = JSON_OBJECT.validate_json(response.content)
    assert set(body) == {"models"}, response.text
    models: Final = body["models"]
    assert isinstance(models, list), response.text
    return [object_value(entry) for entry in models]


def _by_slug(entries: Sequence[Mapping[str, JsonValue]]) -> dict[str, dict[str, JsonValue]]:
    return {string_value(entry["slug"]): dict(entry) for entry in entries}


def _write_config(directory: Path, upstream_url: str, tiered: str, plain: str, alias: str) -> Path:
    config: Final = directory / f"codex_catalog_{uuid.uuid4().hex}.yaml"
    params: Final = {
        "model": "openai/gpt-4o-mini",
        "api_base": f"{upstream_url}/v1",
        "api_key": "integration-provider-key",
    }
    config.write_text(
        json.dumps(
            {
                "model_list": [
                    {
                        "model_name": tiered,
                        "litellm_params": params,
                        "model_info": {
                            "service_tiers": ["priority", {"id": "flex", "name": "Flex", "description": "cheap lane"}],
                            "display_name": "YAML Tiered",
                            "max_input_tokens": 4321,
                        },
                    },
                    {"model_name": plain, "litellm_params": params},
                ],
                "router_settings": {"model_group_alias": {alias: tiered}},
                "general_settings": {
                    "master_key": "os.environ/LITELLM_MASTER_KEY",
                    "database_url": "os.environ/DATABASE_URL",
                },
            }
        )
    )
    return config


@pytest.mark.timeout(240)
def test_yaml_tiers_and_alias_in_owned_proxy(gateway: Gateway, tmp_path: Path) -> None:
    tiered: Final = f"codex-yaml-tiered-{uuid.uuid4().hex}"
    plain: Final = f"codex-yaml-plain-{uuid.uuid4().hex}"
    alias: Final = f"codex-yaml-alias-{uuid.uuid4().hex}"
    config: Final = _write_config(tmp_path, gateway.upstream_url, tiered, plain, alias)
    with owned_proxy(gateway, tmp_path, {"STORE_MODEL_IN_DB": "False"}, config=config, workers=2) as candidate:
        readings: Final = tuple(_entries(_catalog_response(candidate)) for _ in range(CONSISTENT_READS))
        assert all(reading == readings[0] for reading in readings), readings
        catalog: Final = _by_slug(readings[0])
        assert sorted(catalog) == sorted((tiered, plain, alias)), sorted(catalog)
        assert catalog[tiered]["service_tiers"] == [
            _generic_tier("priority"),
            {"id": "flex", "name": "Flex", "description": "cheap lane"},
        ], catalog[tiered]
        assert catalog[tiered]["display_name"] == "YAML Tiered", catalog[tiered]
        assert catalog[tiered]["context_window"] == 4321, catalog[tiered]
        assert catalog[plain]["service_tiers"] == [] and catalog[plain]["display_name"] == plain, catalog[plain]
        assert catalog[alias]["service_tiers"] == catalog[tiered]["service_tiers"], catalog[alias]
        assert catalog[alias]["display_name"] == alias, catalog[alias]
        assert catalog[alias]["context_window"] == 4321, catalog[alias]
        assert string_value(catalog[alias]["base_instructions"]), alias


ROOT: Final = Path(os.environ.get("INTEGRATION_PROXY_ROOT") or Path(__file__).resolve().parents[3])
BUNDLED_STOCK_PATH: Final = ROOT / "litellm" / "proxy" / "common_utils" / "codex_bundled_models_0.159.3.json"
STOCK_UPSTREAM: Final = "gpt-5.5"


def _bundled_stock_entry(slug: str) -> dict[str, JsonValue]:
    models: Final = JSON_OBJECT.validate_json(BUNDLED_STOCK_PATH.read_bytes())["models"]
    assert isinstance(models, list), slug
    (entry,) = (object_value(model) for model in models if object_value(model)["slug"] == slug)
    return entry


def _write_team_owned_config(
    directory: Path, upstream_url: str, stock_team: str, plain_team: str, owned: str, mixed: str, alias: str
) -> Path:
    config: Final = directory / f"codex_catalog_teams_{uuid.uuid4().hex}.yaml"
    plain_params: Final = {
        "model": "openai/gpt-4o-mini",
        "api_base": f"{upstream_url}/v1",
        "api_key": "integration-provider-key",
    }
    stock_params: Final = {**plain_params, "model": f"openai/{STOCK_UPSTREAM}"}
    config.write_text(
        json.dumps(
            {
                "model_list": [
                    {"model_name": owned, "litellm_params": stock_params, "model_info": {"team_id": stock_team}},
                    {
                        "model_name": owned,
                        "litellm_params": plain_params,
                        "model_info": {"team_id": plain_team, "service_tiers": ["flex"]},
                    },
                    {"model_name": mixed, "litellm_params": stock_params, "model_info": {"team_id": stock_team}},
                    {"model_name": mixed, "litellm_params": plain_params, "model_info": {"service_tiers": ["flex"]}},
                ],
                "router_settings": {"model_group_alias": {alias: owned}},
                "general_settings": {
                    "master_key": "os.environ/LITELLM_MASTER_KEY",
                    "database_url": "os.environ/DATABASE_URL",
                },
            }
        )
    )
    return config


@pytest.mark.timeout(240)
def test_team_owned_deployments_of_one_name_serve_each_team_its_own_upstream(gateway: Gateway, tmp_path: Path) -> None:
    """Two teams own a deployment of one model name (and of a `model_group_alias` of it): Codex's stock
    entry for the first deployment's upstream goes only to the team whose requests reach it, and a
    caller outside the owning team reads the deployment no team owns."""
    owned: Final = f"codex-team-owned-{uuid.uuid4().hex}"
    mixed: Final = f"codex-team-mixed-{uuid.uuid4().hex}"
    alias: Final = f"codex-team-alias-{uuid.uuid4().hex}"
    stock: Final = _bundled_stock_entry(STOCK_UPSTREAM)
    assert stock["supported_reasoning_levels"] != [] and stock["service_tiers"] != [], stock
    with gateway.scenario() as scenario:
        stock_team: Final = scenario.team()
        plain_team: Final = scenario.team()
        stock_key: Final = scenario.key(team_id=stock_team, models=[owned, alias, mixed])
        plain_key: Final = scenario.key(team_id=plain_team, models=[owned, alias, mixed])
        teamless_key: Final = scenario.key(models=[mixed])
        config: Final = _write_team_owned_config(
            tmp_path, gateway.upstream_url, stock_team, plain_team, owned, mixed, alias
        )
        with owned_proxy(gateway, tmp_path, {"STORE_MODEL_IN_DB": "False"}, config=config, workers=2) as candidate:

            def stable_catalog(key: str) -> dict[str, dict[str, JsonValue]]:
                readings: Final = tuple(
                    _entries(_catalog_response(candidate, key=key)) for _ in range(CONSISTENT_READS)
                )
                assert all(reading == readings[0] for reading in readings), readings
                return _by_slug(readings[0])

            for_stock_team: Final = stable_catalog(stock_key)
            for_plain_team: Final = stable_catalog(plain_key)
            for_teamless: Final = stable_catalog(teamless_key)
        assert sorted(for_stock_team) == sorted(for_plain_team) == sorted((owned, alias, mixed)), sorted(for_stock_team)
        assert sorted(for_teamless) == [mixed], sorted(for_teamless)
        for slug in (owned, alias):
            assert for_stock_team[slug]["supported_reasoning_levels"] == stock["supported_reasoning_levels"], slug
            assert for_stock_team[slug]["model_messages"] == stock["model_messages"], slug
            assert for_stock_team[slug]["service_tiers"] == stock["service_tiers"], slug
            assert for_plain_team[slug]["supported_reasoning_levels"] == [], for_plain_team[slug]
            assert "model_messages" not in for_plain_team[slug], slug
            assert string_value(for_plain_team[slug]["base_instructions"]), slug
            assert for_plain_team[slug]["service_tiers"] == [_generic_tier("flex")], for_plain_team[slug]
        assert for_stock_team[mixed]["supported_reasoning_levels"] == stock["supported_reasoning_levels"], mixed
        assert for_stock_team[mixed]["service_tiers"] == [], for_stock_team[mixed]
        for outsider in (for_plain_team, for_teamless):
            assert outsider[mixed]["supported_reasoning_levels"] == [], outsider[mixed]
            assert "model_messages" not in outsider[mixed], mixed
            assert outsider[mixed]["service_tiers"] == [_generic_tier("flex")], outsider[mixed]


def _catalog_outcome(proxy: Gateway, slugs: Sequence[JsonValue]) -> str:
    try:
        response: Final = _catalog_response(proxy)
    except httpx.TransportError as error:
        return f"transport:{type(error).__name__}"
    assert response.status_code == 200, response.text
    assert sorted(entry["slug"] for entry in _entries(response)) == list(slugs), response.text
    return "ok"


@pytest.mark.timeout(300)
def test_catalog_survives_a_worker_kill(gateway: Gateway, tmp_path: Path) -> None:
    tiered: Final = f"codex-yaml-tiered-{uuid.uuid4().hex}"
    plain: Final = f"codex-yaml-plain-{uuid.uuid4().hex}"
    alias: Final = f"codex-yaml-alias-{uuid.uuid4().hex}"
    config: Final = _write_config(tmp_path, gateway.upstream_url, tiered, plain, alias)
    with owned_proxy_process(gateway, tmp_path, {"STORE_MODEL_IN_DB": "False"}, config=config, workers=2) as owned:
        candidate: Final = owned.gateway
        workers: Final = eventually(owned.worker_pids, lambda pids: len(pids) == 2, seconds=30)
        victim: Final = min(workers)
        slugs: Final = sorted(entry["slug"] for entry in _entries(_catalog_response(candidate)))
        assert slugs == sorted((tiered, plain, alias)), slugs

        with owned.replacing_worker(victim) as kill:

            def attempt(index: int) -> str:
                if index == 2:
                    kill()
                return _catalog_outcome(candidate, slugs)

            with ThreadPoolExecutor(max_workers=BURST) as pool:
                outcomes: Final = tuple(pool.map(attempt, range(BURST)))
        assert outcomes.count("ok") >= 1, outcomes
        assert all(outcome == "ok" or outcome.startswith("transport:") for outcome in outcomes), outcomes
        respawned: Final = eventually(
            owned.worker_pids,
            lambda pids: len(pids) == 2 and victim not in pids,
            seconds=60,
        )
        assert victim not in respawned, respawned

        def settled_burst() -> tuple[str, ...]:
            with ThreadPoolExecutor(max_workers=BURST) as pool:
                return tuple(pool.map(lambda _: _catalog_outcome(candidate, slugs), range(BURST)))

        final: Final = eventually(settled_burst, lambda values: all(value == "ok" for value in values), seconds=40)
        assert final == ("ok",) * BURST, final


def _install_shim(directory: Path, stock: Mapping[str, JsonValue]) -> tuple[Path, dict[str, str]]:
    shim_dir: Final = directory / "bin"
    shim_dir.mkdir()
    script: Final = directory / "codex_shim.py"
    script.write_text(SHIM_SCRIPT, encoding="utf-8")
    shim: Final = shim_dir / "codex"
    shim.write_text(f'#!/bin/sh\nexec "{sys.executable}" "{script}" "$@"\n', encoding="utf-8")
    shim.chmod(shim.stat().st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
    stock_path: Final = directory / "stock.json"
    stock_path.write_text(json.dumps(stock), encoding="utf-8")
    launch_path: Final = directory / "launch.json"
    home: Final = directory / "home"
    home.mkdir()
    codex_home: Final = directory / "codex-home"
    environment: Final = {
        **os.environ,
        "HOME": str(home),
        "CODEX_HOME": str(codex_home),
        "PATH": f"{shim_dir}{os.pathsep}{os.environ['PATH']}",
        SHIM_STOCK_ENV: str(stock_path),
        SHIM_LAUNCH_ENV: str(launch_path),
    }
    return launch_path, environment


def _run_lite_codex(proxy: Gateway, key: str, environment: Mapping[str, str]) -> subprocess.CompletedProcess[str]:
    return subprocess.run(
        [sys.executable, "-I", "-c", "from litellm.proxy.client.cli import cli; cli()", "codex", "--shim-marker"],
        env={**environment, "LITELLM_PROXY_URL": str(proxy.client.base_url).rstrip("/"), "LITELLM_PROXY_API_KEY": key},
        stdin=subprocess.DEVNULL,
        capture_output=True,
        text=True,
        timeout=90,
        check=False,
    )


def _new_model(scenario: Scenario, name: str, model_info: Mapping[str, JsonValue]) -> None:
    created: Final = scenario.gateway.post(
        "/model/new",
        {
            "model_name": name,
            "litellm_params": {
                "model": "openai/gpt-4o-mini",
                "api_key": "integration-provider-key",
                "api_base": f"{scenario.gateway.upstream_url}/v1",
            },
            "model_info": dict(model_info),
        },
    )
    scenario.cleanups.callback(scenario.delete_model, string_value(object_value(created["model_info"])["id"]))


def _launch_record(path: Path) -> dict[str, JsonValue]:
    assert path.exists(), "the shim was never launched"
    return JSON_OBJECT.validate_json(path.read_bytes())


def _catalog_override(argv: Sequence[JsonValue]) -> str | None:
    overrides: Final = [
        string_value(arg) for arg in argv if isinstance(arg, str) and arg.startswith("model_catalog_json=")
    ]
    assert len(overrides) <= 1, argv
    return json.loads(overrides[0].partition("=")[2]) if overrides else None


def test_lite_codex_writes_the_shared_catalog(gateway: Gateway, tmp_path: Path) -> None:
    """`lite codex` writes the installed Codex's own entry for a proxy model whose name is a stock slug and
    the fallback entry for any other; the proxy's configured tiers never reach the file (Decision 17)."""
    stock_slug: Final = f"codex-stock-{uuid.uuid4().hex}"
    stock_tiers: Final = [{"id": "priority", "name": "Fast", "description": "1.5x speed"}]
    stock: Final = {
        "models": [
            {
                "slug": stock_slug,
                "display_name": "Stock Model",
                "priority": 7,
                "visibility": "hide",
                "supported_in_api": False,
                "service_tiers": stock_tiers,
                "default_service_tier": None,
                "supported_reasoning_levels": [{"effort": "high", "description": "thinks longer"}],
            }
        ]
    }
    launch_path, environment = _install_shim(tmp_path, stock)
    with gateway.scenario() as scenario:
        _new_model(scenario, stock_slug, {"service_tiers": ["ultrafast"]})
        fallback: Final = scenario.model(model_info={"service_tiers": ["priority"]})
        key: Final = scenario.key(models=[stock_slug, fallback])
        completed: Final = _run_lite_codex(gateway, key, environment)
        assert completed.returncode == 0, (completed.stdout, completed.stderr)
        assert "not syncing" not in completed.stderr, completed.stderr
        record: Final = _launch_record(launch_path)
        argv: Final = record["argv"]
        assert isinstance(argv, list) and argv[-1] == "--shim-marker", argv
        catalog_path: Final = Path(environment["CODEX_HOME"]) / CATALOG_FILENAME
        assert _catalog_override(argv) == str(catalog_path), argv
        env: Final = object_value(record["env"])
        assert env["OPENAI_BASE_URL"] == str(gateway.client.base_url).rstrip("/") + "/v1", env
        assert env["OPENAI_API_KEY"] == key, env.keys()
        written: Final = JSON_OBJECT.validate_json(catalog_path.read_bytes())
        assert set(written) == {"models"}, written.keys()
        models: Final = written["models"]
        assert isinstance(models, list), written
        file_entries: Final = _by_slug([object_value(entry) for entry in models])
        served: Final = eventually(
            lambda: _by_slug(_entries(_catalog_response(gateway, key=key))),
            lambda value: (
                stock_slug in value
                and fallback in value
                and value[stock_slug]["service_tiers"] == [_generic_tier("ultrafast")]
                and value[fallback]["service_tiers"] == [_generic_tier("priority")]
            ),
            seconds=CONVERGENCE_SECONDS,
        )
        assert list(file_entries) == list(served), (list(file_entries), list(served))
        assert file_entries[stock_slug]["service_tiers"] == stock_tiers, file_entries[stock_slug]
        assert file_entries[stock_slug]["visibility"] == "list", file_entries[stock_slug]
        assert file_entries[stock_slug]["supported_in_api"] is True, file_entries[stock_slug]
        assert file_entries[stock_slug]["priority"] == list(file_entries).index(stock_slug), file_entries[stock_slug]
        assert file_entries[stock_slug]["supported_reasoning_levels"] == [
            {"effort": "high", "description": "thinks longer"}
        ]
        assert file_entries[fallback]["service_tiers"] == [], file_entries[fallback]
        assert file_entries[fallback]["display_name"] == fallback, file_entries[fallback]
        assert string_value(file_entries[fallback]["base_instructions"]), fallback
        assert served[stock_slug]["service_tiers"] == [_generic_tier("ultrafast")], served[stock_slug]
        assert served[fallback]["service_tiers"] == [_generic_tier("priority")], served[fallback]


def test_lite_codex_skips_a_stock_dump_without_display_name(gateway: Gateway, tmp_path: Path) -> None:
    """A `codex debug models` dump that predates `display_name` is reported, not written, and Codex still
    launches without a catalog override."""
    stock: Final = {"models": [{"slug": "gpt-5.5", "priority": 0, "visibility": "list"}]}
    launch_path, environment = _install_shim(tmp_path, stock)
    with gateway.scenario() as scenario:
        model: Final = scenario.model(model_info={"service_tiers": ["priority"]})
        key: Final = scenario.key(models=[model])
        completed: Final = _run_lite_codex(gateway, key, environment)
        assert completed.returncode == 0, (completed.stdout, completed.stderr)
        assert "printed no model catalog" in completed.stderr, completed.stderr
        record: Final = _launch_record(launch_path)
        argv: Final = record["argv"]
        assert isinstance(argv, list) and argv[-1] == "--shim-marker", argv
        assert _catalog_override(argv) is None, argv
        assert not (Path(environment["CODEX_HOME"]) / CATALOG_FILENAME).exists(), environment["CODEX_HOME"]