Skip to content

The fastest, litest AI Gateway. Rust core with Python SDK. Call 100+ LLM APIs in OpenAI (or native) format with cost tracking, guardrails, load balancing, and logging [Bedrock, Azure, OpenAI, Anthropic, OpenAI, VertexAI, vLLM, Nvidia NIM]

15.6 KB
Raw Blame
from __future__ import annotations

import asyncio
import json
import re
import threading
import uuid
from collections.abc import AsyncIterator, Callable, Generator, Mapping
from contextlib import asynccontextmanager, contextmanager
from dataclasses import dataclass
from pathlib import Path
from queue import SimpleQueue
from types import MappingProxyType
from typing import Final

import httpx
import psutil
import yaml
from integration._support.client import JSON_OBJECT, Gateway, eventually, object_value
from integration._support.wire import Reply, Request, Wire, wire_server
from pydantic import JsonValue

MARKER: Final = re.compile(r"m[0-9a-f]{32}")
TRACEBACK: Final = "Traceback (most recent call last)"
BUG_NOTICE: Final = "This looks like a bug in LiteLLM"
ASSISTANT_MODEL: Final = "scripted-assistant-model"
CONTROL_MODEL: Final = "assistants-route-errors-control"
SESSION_HEADER: Final = MappingProxyType({"x-litellm-session-id": "assistants-route-errors"})
STARTED_WORKER: Final = re.compile(r"Started server process \[(\d+)\]")


@dataclass(frozen=True, slots=True)
class Route:
    name: str
    method: str
    path: str
    reads_body: bool
    carries_marker: bool


ROUTES: Final = (
    Route("get_assistants", "GET", "/v1/assistants", reads_body=False, carries_marker=False),
    Route("create_assistant", "POST", "/v1/assistants", reads_body=True, carries_marker=True),
    Route("delete_assistant", "DELETE", "/v1/assistants/asst_{marker}", reads_body=False, carries_marker=True),
    Route("create_thread", "POST", "/v1/threads", reads_body=False, carries_marker=False),
    Route("get_thread", "GET", "/v1/threads/thread_{marker}", reads_body=False, carries_marker=True),
    Route("add_message", "POST", "/v1/threads/thread_{marker}/messages", reads_body=True, carries_marker=True),
    Route("get_messages", "GET", "/v1/threads/thread_{marker}/messages", reads_body=False, carries_marker=True),
    Route("run_thread", "POST", "/v1/threads/thread_{marker}/runs", reads_body=True, carries_marker=True),
)
ROUTE_IDS: Final = tuple(route.name for route in ROUTES)
MARKED_ROUTES: Final = tuple(route for route in ROUTES if route.carries_marker)
BODY_ROUTES: Final = tuple(route for route in ROUTES if route.reads_body)
ROUTE_BY_NAME: Final = MappingProxyType({route.name: route for route in ROUTES})


@dataclass(frozen=True, slots=True)
class Mapped:
    status: int
    type: str
    param: str | None


STATUS_TABLE: Final = (
    Mapped(400, "scripted_type", "scripted_param"),
    Mapped(401, "authentication_error", None),
    Mapped(403, "permission_error", None),
    Mapped(404, "invalid_request_error", None),
    Mapped(408, "invalid_request_error", None),
    Mapped(409, "invalid_request_error", None),
    Mapped(422, "scripted_type", "scripted_param"),
    Mapped(429, "throttling_error", "scripted_param"),
    Mapped(500, "internal_server_error", "scripted_param"),
    Mapped(502, "internal_server_error", None),
    Mapped(503, "internal_server_error", None),
    Mapped(504, "internal_server_error", None),
)
MAPPED_BY_STATUS: Final = MappingProxyType({mapped.status: mapped for mapped in STATUS_TABLE})
UNMAPPED_500: Final = Mapped(500, "internal_server_error", None)


def new_marker() -> str:
    return f"m{uuid.uuid4().hex}"


def proxy_path(route: Route, marker: str) -> str:
    return route.path.format(marker=marker)


def request_body(route: Route, marker: str) -> dict[str, JsonValue] | None:
    match route.name:
        case "create_assistant":
            return {"model": ASSISTANT_MODEL, "name": marker}
        case "create_thread":
            return {"messages": [{"role": "user", "content": marker}]}
        case "add_message":
            return {"role": "user", "content": marker}
        case "run_thread":
            return {"assistant_id": f"asst_{marker}"}
        case _:
            return None


def marker_of(request: Request) -> str | None:
    found: Final = MARKER.search(request.target) or MARKER.search(request.body.decode(errors="replace"))
    return found.group(0) if found else None


def route_of(request: Request, marker: str) -> Route:
    target: Final = (request.method, request.target.split("?", 1)[0])
    matches: Final = tuple(route for route in ROUTES if (route.method, proxy_path(route, marker)) == target)
    assert len(matches) == 1, target
    return matches[0]


def error_reply(status: int, message: str) -> Reply:
    error: Final = {"message": message, "type": "scripted_type", "param": "scripted_param", "code": "scripted_code"}
    return Reply(status=status, body=json.dumps({"error": error}).encode())


def scripted_message(marker: str, status: int) -> str:
    return f"scripted {marker} status {status}"


def _assistant(marker: str) -> dict[str, JsonValue]:
    return {
        "id": f"asst_{marker}",
        "object": "assistant",
        "created_at": 1,
        "name": marker,
        "description": None,
        "model": ASSISTANT_MODEL,
        "instructions": None,
        "tools": [],
        "metadata": {},
    }


def _thread(marker: str) -> dict[str, JsonValue]:
    return {"id": f"thread_{marker}", "object": "thread", "created_at": 1, "metadata": {}, "tool_resources": None}


def _message(marker: str) -> dict[str, JsonValue]:
    return {
        "id": f"msg_{marker}",
        "object": "thread.message",
        "created_at": 1,
        "thread_id": f"thread_{marker}",
        "role": "user",
        "content": [{"type": "text", "text": {"value": marker, "annotations": []}}],
        "assistant_id": None,
        "run_id": None,
        "attachments": [],
        "metadata": {},
        "status": "completed",
    }


def _run(marker: str, status: str) -> dict[str, JsonValue]:
    return {
        "id": f"run_{marker}",
        "object": "thread.run",
        "created_at": 1,
        "thread_id": f"thread_{marker}",
        "assistant_id": f"asst_{marker}",
        "status": status,
        "model": ASSISTANT_MODEL,
        "instructions": "",
        "tools": [],
        "metadata": {},
        "parallel_tool_calls": True,
    }


def _page(item: dict[str, JsonValue]) -> dict[str, JsonValue]:
    return {"object": "list", "data": [item], "first_id": item["id"], "last_id": item["id"], "has_more": False}


def success_body(route: Route, marker: str) -> dict[str, JsonValue]:
    match route.name:
        case "get_assistants":
            return _page(_assistant(marker))
        case "create_assistant":
            return _assistant(marker)
        case "delete_assistant":
            return {"id": f"asst_{marker}", "object": "assistant.deleted", "deleted": True}
        case "create_thread" | "get_thread":
            return _thread(marker)
        case "add_message":
            return _message(marker)
        case "get_messages":
            return _page(_message(marker))
        case _:
            return _run(marker, "queued")


def run_poll_path(marker: str) -> str:
    return f"/v1/threads/thread_{marker}/runs/run_{marker}"


def success_trail(route: Route, marker: str) -> tuple[tuple[str, str], ...]:
    first: Final = (route.method, proxy_path(route, marker))
    return (first, ("GET", run_poll_path(marker))) if route.name == "run_thread" else (first,)


def chat_completion_body(marker: str) -> dict[str, JsonValue]:
    return {
        "id": f"chatcmpl-{marker}",
        "object": "chat.completion",
        "created": 1,
        "model": "scripted-chat-model",
        "choices": [{"index": 0, "message": {"role": "assistant", "content": marker}, "finish_reason": "stop"}],
        "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2},
    }


def success_peer(marker: str) -> Callable[[Request], Reply]:
    def respond(request: Request) -> Reply:
        if request.target == "/v1/chat/completions":
            return Reply(body=json.dumps(chat_completion_body(marker)).encode())
        if request.method == "GET" and request.target == run_poll_path(marker):
            return Reply(body=json.dumps(_run(marker, "completed")).encode())
        return Reply(body=json.dumps(success_body(route_of(request, marker), marker)).encode())

    return respond


def error_object(response: httpx.Response) -> dict[str, JsonValue]:
    body: Final = JSON_OBJECT.validate_json(response.content)
    assert set(body) == {"error"}, response.text
    return object_value(body["error"])


def assert_clean_message(message: JsonValue) -> str:
    assert isinstance(message, str), message
    assert TRACEBACK not in message, message
    assert BUG_NOTICE not in message, message
    return message


def assert_openai_error(response: httpx.Response, mapped: Mapped) -> str:
    assert response.status_code == mapped.status, response.text
    error: Final = error_object(response)
    assert set(error) == {"message", "type", "param", "code"}, response.text
    assert (error["type"], error["param"], error["code"]) == (mapped.type, mapped.param, str(mapped.status)), (
        response.text
    )
    return assert_clean_message(error["message"])


def assert_mapped_upstream_error(response: httpx.Response, mapped: Mapped, marker: str) -> None:
    message: Final = assert_openai_error(response, mapped)
    assert marker in message, response.text


def is_model_listing(request: Request) -> bool:
    return request.method == "GET" and request.target.split("?", 1)[0].endswith("/models")


def provider_requests(received: tuple[Request, ...]) -> tuple[Request, ...]:
    return tuple(request for request in received if not is_model_listing(request))


def upstream_trail(received: tuple[Request, ...]) -> tuple[tuple[str, str], ...]:
    return tuple((request.method, request.target.split("?", 1)[0]) for request in provider_requests(received))


@contextmanager
def assistants_wire(respond: Callable[[Request], Reply], port: int = 0) -> Generator[Wire, None, None]:
    def answer(request: Request) -> Reply:
        if is_model_listing(request):
            return Reply(body=json.dumps({"object": "list", "data": []}).encode())
        return respond(request)

    with wire_server(answer, port=port) as wire:
        yield wire


def assert_reached_upstream_once(received: tuple[Request, ...], route: Route, marker: str) -> None:
    assert upstream_trail(received) == ((route.method, proxy_path(route, marker)),), received


def owned_config(
    directory: Path,
    model_list: tuple[Mapping[str, JsonValue], ...],
    assistant_settings: Mapping[str, JsonValue],
    general_settings: Mapping[str, JsonValue] = MappingProxyType({}),
) -> Path:
    config: Final = JSON_OBJECT.validate_python(yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()))
    merged: Final = {
        **config,
        "model_list": [dict(model) for model in model_list],
        "general_settings": {**object_value(config["general_settings"]), **general_settings},
        "router_settings": {**object_value(config["router_settings"]), "num_retries": 0},
        "assistant_settings": dict(assistant_settings),
    }
    path: Final = directory / f"assistants-route-errors-{uuid.uuid4().hex}.yaml"
    path.write_text(yaml.safe_dump(merged))
    return path


def openai_assistants_config(directory: Path, upstream_port: int, timeout_seconds: int) -> Path:
    api_base: Final = f"http://127.0.0.1:{upstream_port}/v1"
    deployment: Final[dict[str, JsonValue]] = {
        "api_base": api_base,
        "api_key": "sk-scripted-assistants",
        "max_retries": 0,
        "timeout": timeout_seconds,
    }
    control: Final[dict[str, JsonValue]] = {
        "model_name": CONTROL_MODEL,
        "litellm_params": {"model": "openai/scripted-chat-model", **deployment},
    }
    return owned_config(directory, (control,), {"custom_llm_provider": "openai", "litellm_params": deployment})


def worker_pids(log: Path) -> tuple[int, ...]:
    return tuple(int(pid) for pid in STARTED_WORKER.findall(log.read_text()))


def live_worker_pids(log: Path) -> tuple[int, ...]:
    return tuple(pid for pid in worker_pids(log) if psutil.pid_exists(pid))


def open_upstream_connections(pid: int, upstream_port: int) -> int:
    return sum(
        1
        for connection in psutil.Process(pid).net_connections(kind="tcp")
        if connection.status == psutil.CONN_ESTABLISHED and connection.raddr and connection.raddr.port == upstream_port
    )


def held_upstream_connections(workers: tuple[int, ...], upstream_port: int, expected: int) -> Mapping[int, int]:
    return eventually(
        lambda: MappingProxyType({pid: open_upstream_connections(pid, upstream_port) for pid in workers}),
        lambda held_by: sum(held_by.values()) == expected,
        seconds=10,
    )


def held_error_peer(
    status_by_marker: Mapping[str, int], arrived: SimpleQueue[str], release: threading.Event
) -> Callable[[Request], Reply]:
    def respond(request: Request) -> Reply:
        marker: Final = marker_of(request)
        assert marker is not None, request.target
        arrived.put(marker)
        assert release.wait(timeout=60), "The burst was never released"
        return error_reply(status_by_marker[marker], scripted_message(marker, status_by_marker[marker]))

    return respond


@dataclass(frozen=True, slots=True)
class Call:
    route: Route
    marker: str


@dataclass(frozen=True, slots=True)
class Answer:
    call: Call
    response: httpx.Response


async def _answer(client: httpx.AsyncClient, call: Call) -> Answer:
    response: Final = await client.request(
        call.route.method, proxy_path(call.route, call.marker), json=request_body(call.route, call.marker)
    )
    return Answer(call, response)


def _async_client(gateway: Gateway) -> httpx.AsyncClient:
    return httpx.AsyncClient(
        base_url=str(gateway.client.base_url),
        headers={"Authorization": f"Bearer {gateway.key}"},
        timeout=60,
        trust_env=False,
    )


def _answers_of(results: tuple[Answer | BaseException, ...]) -> tuple[Answer, ...]:
    for result in results:
        assert not isinstance(result, BaseException) or isinstance(result, httpx.TransportError), repr(result)
    return tuple(result for result in results if isinstance(result, Answer))


async def answer_all(
    gateway: Gateway, calls: tuple[Call, ...], *, tolerate_transport_errors: bool = False
) -> tuple[Answer, ...]:
    async with _async_client(gateway) as client:
        results: Final = await asyncio.gather(
            *(_answer(client, call) for call in calls), return_exceptions=tolerate_transport_errors
        )
    return _answers_of(tuple(results))


async def _send_one_by_one(
    client: httpx.AsyncClient,
    calls: tuple[Call, ...],
    arrived: SimpleQueue[str],
    enough: Callable[[], bool],
) -> tuple[asyncio.Task[Answer], ...]:
    if not calls:
        return ()
    sent: Final = asyncio.create_task(_answer(client, calls[0]))
    assert await asyncio.to_thread(arrived.get, True, 30) == calls[0].marker
    if enough():
        return (sent,)
    return (sent, *await _send_one_by_one(client, calls[1:], arrived, enough))


@dataclass(frozen=True, slots=True)
class Held:
    calls: tuple[Call, ...]
    pending: tuple[asyncio.Task[Answer], ...]

    async def answers(self) -> tuple[Answer, ...]:
        return _answers_of(tuple(await asyncio.gather(*self.pending, return_exceptions=True)))


@asynccontextmanager
async def held_one_by_one(
    gateway: Gateway, calls: tuple[Call, ...], arrived: SimpleQueue[str], enough: Callable[[], bool]
) -> AsyncIterator[Held]:
    async with _async_client(gateway) as client:
        pending: Final = await _send_one_by_one(client, calls, arrived, enough)
        yield Held(calls[: len(pending)], pending)


def assert_answered_with_its_own_marker(answer: Answer, mapped: Mapped) -> None:
    assert_mapped_upstream_error(answer.response, mapped, answer.call.marker)
    assert set(MARKER.findall(answer.response.text)) == {answer.call.marker}, answer.response.text