Skip to content

The fastest, litest AI Gateway. Rust core with Python SDK. Call 100+ LLM APIs in OpenAI (or native) format with cost tracking, guardrails, load balancing, and logging [Bedrock, Azure, OpenAI, Anthropic, OpenAI, VertexAI, vLLM, Nvidia NIM]

11.1 KB
Raw Blame
import os
import subprocess
import sys
import textwrap
import uuid
from collections.abc import Iterator, Mapping
from dataclasses import dataclass
from pathlib import Path
from types import MappingProxyType
from typing import Final, Literal, TypeAlias

import pytest
from integration._support import device_login as dl
from integration._support import responses_vendor as rv
from pydantic import JsonValue

pytestmark: Final = pytest.mark.timeout(300)

Provider: TypeAlias = Literal["chatgpt", "copilot"]

_MODELS: Final[Mapping[Provider, str]] = MappingProxyType(
    {"chatgpt": "chatgpt/gpt-5.6-terra", "copilot": "github_copilot/gpt-5.4"}
)
_REFUSALS: Final[Mapping[Provider, str]] = MappingProxyType(
    {"chatgpt": dl.CHATGPT_REFUSAL, "copilot": dl.COPILOT_REFUSAL}
)
_USER_CODES: Final[Mapping[Provider, str]] = MappingProxyType(
    {"chatgpt": dl.CHATGPT_USER_CODE, "copilot": dl.COPILOT_USER_CODE}
)
_AUTHENTICATORS: Final[Mapping[Provider, str]] = MappingProxyType(
    {"chatgpt": "litellm.llms.chatgpt.authenticator", "copilot": "litellm.llms.github_copilot.authenticator"}
)
_SCRIPT_SECONDS: Final = 60

_SCRIPT: Final = textwrap.dedent(
    """
    import asyncio, json, sys, threading
    import litellm
    from litellm import Router

    variant, model, marker, authenticator_module = sys.argv[1:5]
    messages = [{"role": "user", "content": f"Reply to marker-{marker}"}]

    def sync_completion():
        return litellm.completion(model=model, messages=messages, num_retries=0)

    def server_tool_call_turns(marked):
        call = {"id": "srvtoolu_01DeviceLogin", "type": "function", "function": {"name": "tool_search_tool_bm25", "arguments": "{}"}}
        mark = {"cache_control": {"type": "ephemeral"}} if marked else {}
        result = {"type": "tool_search_tool_result", "tool_use_id": call["id"], "content": {"type": "tool_search_tool_search_result", "tool_references": []}}
        return [
            {"role": "system", "content": "Answer in one sentence."},
            *messages,
            {"role": "assistant", "content": "", "tool_calls": [{**call, **mark}], "provider_specific_fields": {"tool_results": [result]}},
            {"role": "tool", "tool_call_id": call["id"], "content": "found get_weather"},
            {"role": "user", "content": "go on"},
        ]

    def run():
        if variant == "completion":
            return sync_completion()
        if variant in ("marked-server-tool-call", "unmarked-server-tool-call"):
            return litellm.completion(
                model=model,
                messages=server_tool_call_turns(variant == "marked-server-tool-call"),
                num_retries=0,
                cache_control_injection_points=[{"location": "message", "role": "system"}, {"location": "message", "index": -1}],
            )
        if variant == "responses":
            return litellm.responses(model=model, input=f"Reply to marker-{marker}", num_retries=0)
        if variant == "acompletion":
            return asyncio.run(litellm.acompletion(model=model, messages=messages, num_retries=0))
        if variant == "aresponses":
            return asyncio.run(litellm.aresponses(model=model, input=f"Reply to marker-{marker}", num_retries=0))
        if variant == "worker-thread":
            outcome = {}
            def target():
                try:
                    outcome["value"] = sync_completion()
                except Exception as error:
                    outcome["error"] = error
            thread = threading.Thread(target=target)
            thread.start()
            thread.join()
            if "error" in outcome:
                raise outcome["error"]
            return outcome["value"]
        if variant == "batch-completion":
            (only,) = litellm.batch_completion(model=model, messages=[messages], num_retries=0)
            if isinstance(only, Exception):
                raise only
            return only
        if variant == "sync-inside-loop":
            async def main():
                return sync_completion()
            return asyncio.run(main())
        if variant == "one-liner-inside-loop":
            module = __import__(authenticator_module, fromlist=["Authenticator"])
            async def main():
                return module.Authenticator().get_access_token()
            return asyncio.run(main())
        if variant == "router-built-in-async":
            async def main():
                router = Router(model_list=[{"model_name": "login", "litellm_params": {"model": model}}])
                return await router.acompletion(model="login", messages=messages, num_retries=0)
            return asyncio.run(main())
        if variant == "wildcard-router-in-loop":
            router = Router(model_list=[{"model_name": "*", "litellm_params": {"model": "*"}}])
            async def main():
                return await router.acompletion(model=model, messages=messages, num_retries=0)
            return asyncio.run(main())
        raise SystemExit(f"unknown variant {variant}")

    def render(result):
        if hasattr(result, "__next__"):
            return "".join(str(chunk) for chunk in result)
        return str(result)

    try:
        print(json.dumps({"ok": True, "text": render(run())}), flush=True)
    except Exception as error:
        print(json.dumps({"ok": False, "error": f"{type(error).__name__}: {error}"}), flush=True)
    """
)


@dataclass(frozen=True, slots=True)
class _Outcome:
    returncode: int
    stdout: str
    stderr: str

    def verdict(self) -> Mapping[str, JsonValue]:
        lines: Final = [line for line in self.stdout.splitlines() if line.startswith("{")]
        assert lines, (self.stdout, self.stderr)
        return rv.JSON_OBJECT.validate_json(lines[-1])


@dataclass(frozen=True, slots=True)
class _Bench:
    peers: dl.Peers
    chatgpt_dir: Path
    copilot_dir: Path

    def run(self, variant: str, provider: Provider, marker: str) -> _Outcome:
        completed: Final = subprocess.run(
            [sys.executable, "-P", "-c", _SCRIPT, variant, _MODELS[provider], marker, _AUTHENTICATORS[provider]],
            env={
                **os.environ,
                **self.peers.environment(),
                "CHATGPT_TOKEN_DIR": str(self.chatgpt_dir),
                "GITHUB_COPILOT_TOKEN_DIR": str(self.copilot_dir),
                "LITELLM_LOCAL_MODEL_COST_MAP": "True",
            },
            capture_output=True,
            text=True,
            timeout=_SCRIPT_SECONDS,
            check=False,
        )
        return _Outcome(completed.returncode, completed.stdout, completed.stderr)

    def token_files(self) -> tuple[str, ...]:
        return tuple(sorted(path.name for path in (*self.chatgpt_dir.iterdir(), *self.copilot_dir.iterdir())))


@pytest.fixture(scope="module")
def bench(tmp_path_factory: pytest.TempPathFactory) -> Iterator[_Bench]:
    directory: Final = tmp_path_factory.mktemp("device-login-sdk")
    chatgpt_dir: Final = directory / "chatgpt"
    copilot_dir: Final = directory / "copilot"
    chatgpt_dir.mkdir()
    copilot_dir.mkdir()
    with dl.device_login_peers(directory) as peers:
        yield _Bench(peers, chatgpt_dir, copilot_dir)


@pytest.fixture
def clean(bench: _Bench) -> Iterator[_Bench]:
    bench.peers.reset()
    dl.clear_tokens(bench.chatgpt_dir, bench.copilot_dir)
    yield bench
    bench.peers.reset()
    dl.clear_tokens(bench.chatgpt_dir, bench.copilot_dir)


_FIRST_LOGIN_BEARERS: Final[Mapping[Provider, str]] = MappingProxyType(
    {"chatgpt": dl.CHATGPT_FIRST_LOGIN, "copilot": dl.COPILOT_MINTED_KEY}
)


@pytest.mark.parametrize("variant", ("completion", "responses"))
@pytest.mark.parametrize("provider", ("chatgpt", "copilot"))
def test_sync_call_on_the_main_thread_keeps_the_interactive_first_login(
    clean: _Bench, provider: Provider, variant: str
) -> None:
    clean.peers.switches.grant.set()
    marker: Final = uuid.uuid4().hex
    outcome: Final = clean.run(variant, provider, marker)
    verdict: Final = outcome.verdict()
    assert verdict["ok"] is True, (verdict, outcome.stderr[-3000:])
    assert marker in str(verdict["text"]), verdict
    assert _USER_CODES[provider] in outcome.stdout, outcome.stdout
    assert f"{dl.CHATGPT_AUTH_HOST if provider == 'chatgpt' else dl.GITHUB_HOST}:443" in clean.peers.auth_connections()
    forwarded: Final = [request for request in clean.peers.api.drain() if request.method == "POST"]
    assert {dl.bearer(request) for request in forwarded} == {_FIRST_LOGIN_BEARERS[provider]}, [
        (request.method, request.target) for request in forwarded
    ]
    expected_files: Final = ("auth.json",) if provider == "chatgpt" else ("access-token", "api-key.json")
    assert clean.token_files() == expected_files


_IN_LOOP_VARIANTS: Final = (
    "acompletion",
    "aresponses",
    "worker-thread",
    "batch-completion",
    "sync-inside-loop",
    "one-liner-inside-loop",
    "router-built-in-async",
    "wildcard-router-in-loop",
)


@pytest.mark.parametrize("variant", _IN_LOOP_VARIANTS)
@pytest.mark.parametrize("provider", ("chatgpt", "copilot"))
def test_a_call_inside_a_loop_or_worker_thread_is_refused_before_any_auth_host_call(
    clean: _Bench, provider: Provider, variant: str
) -> None:
    clean.peers.switches.grant.set()
    outcome: Final = clean.run(variant, provider, uuid.uuid4().hex)
    verdict: Final = outcome.verdict()
    assert verdict["ok"] is False, (verdict, outcome.stderr[-3000:])
    assert _REFUSALS[provider] in str(verdict["error"]), verdict
    assert _USER_CODES[provider] not in outcome.stdout, outcome.stdout
    assert clean.peers.auth_connections() == ()
    assert clean.peers.api.drain() == ()
    assert clean.token_files() == ()


@pytest.mark.parametrize("provider", ("chatgpt", "copilot"))
def test_sync_call_on_the_main_thread_fails_without_hanging_when_the_auth_host_denies(
    clean: _Bench, provider: Provider
) -> None:
    outcome: Final = clean.run("completion", provider, uuid.uuid4().hex)
    verdict: Final = outcome.verdict()
    assert verdict["ok"] is False, (verdict, outcome.stderr[-3000:])
    assert f"{dl.CHATGPT_AUTH_HOST if provider == 'chatgpt' else dl.GITHUB_HOST}:443" in clean.peers.auth_connections()
    assert clean.peers.api.drain() == ()
    assert clean.token_files() == ()


def _auth_requests(peers: dl.Peers) -> list[tuple[str, str, str]]:
    return [(request.headers.get("host", ""), request.method, request.target) for request in peers.auth.drain()]


@pytest.mark.parametrize("provider", ("chatgpt", "copilot"))
def test_a_marked_server_tool_call_reaches_the_auth_host_like_an_unmarked_one_when_the_login_is_denied(
    clean: _Bench, provider: Provider
) -> None:
    unmarked: Final = clean.run("unmarked-server-tool-call", provider, uuid.uuid4().hex)
    unmarked_auth: Final = _auth_requests(clean.peers)
    marked: Final = clean.run("marked-server-tool-call", provider, uuid.uuid4().hex)
    marked_auth: Final = _auth_requests(clean.peers)
    assert unmarked.verdict()["ok"] is False, (unmarked.verdict(), unmarked.stderr[-3000:])
    assert marked.verdict()["ok"] is False, (marked.verdict(), marked.stderr[-3000:])
    assert unmarked_auth, unmarked.stderr[-3000:]
    assert marked_auth == unmarked_auth
    assert clean.peers.api.drain() == ()
    assert clean.token_files() == ()