Skip to content

The fastest, litest AI Gateway. Rust core with Python SDK. Call 100+ LLM APIs in OpenAI (or native) format with cost tracking, guardrails, load balancing, and logging [Bedrock, Azure, OpenAI, Anthropic, OpenAI, VertexAI, vLLM, Nvidia NIM]

5.8 KB
Raw Blame
import json
from collections.abc import Callable, Mapping
from types import MappingProxyType
from typing import Final
from urllib.parse import urlsplit

from integration._support.anthropic_sse import message_stream
from integration._support.openai_wire import chat_reply
from integration._support.upstream import aws_event_stream_frame
from integration._support.wire import Reply, Request
from integration.providers._api_base_trailing_slash_support import VECTOR, marker_of, reply_text, request_json
from pydantic import JsonValue

COUNTED_TOKENS: Final = 7
RELEVANCE: Final = 0.75
_USAGE: Final = {"input_tokens": 3, "output_tokens": 2}


def _json(payload: Mapping[str, JsonValue]) -> Reply:
    return Reply(body=json.dumps(payload).encode())


def _path(request: Request) -> str:
    return urlsplit(request.target).path


def _wants_stream(request: Request) -> bool:
    return request_json(request).get("stream") is True


def anthropic_message(marker: str) -> dict[str, JsonValue]:
    return {
        "id": f"msg_{marker}",
        "type": "message",
        "role": "assistant",
        "model": "scripted",
        "content": [{"type": "text", "text": reply_text(marker)}],
        "stop_reason": "end_turn",
        "stop_sequence": None,
        "usage": dict(_USAGE),
    }


def _anthropic(request: Request) -> Reply:
    if _path(request).endswith("/count_tokens"):
        return _json({"input_tokens": COUNTED_TOKENS})
    marker: Final = marker_of(request)
    if _wants_stream(request):
        return Reply(
            content_type="text/event-stream", chunks=message_stream(f"msg_{marker}", "scripted", reply_text(marker))
        )
    return _json(anthropic_message(marker))


def _gemini_candidate(text: str, finished: bool) -> dict[str, JsonValue]:
    finish: Final[dict[str, JsonValue]] = {"finishReason": "STOP"} if finished else {}
    return {"content": {"role": "model", "parts": [{"text": text}]}, "index": 0, **finish}


def _gemini_frame(text: str, finished: bool) -> bytes:
    usage: Final = {"promptTokenCount": 3, "candidatesTokenCount": 2, "totalTokenCount": 5}
    frame: Final = {"candidates": [_gemini_candidate(text, finished)], "usageMetadata": usage}
    return b"data: " + json.dumps(frame).encode() + b"\r\n\r\n"


def _gemini(request: Request) -> Reply:
    path: Final = _path(request)
    marker: Final = marker_of(request)
    if path.endswith(":batchEmbedContents"):
        return _json({"embeddings": [{"values": list(VECTOR)}]})
    if path.endswith(":streamGenerateContent"):
        return Reply(
            content_type="text/event-stream",
            chunks=(_gemini_frame("reply ", False), _gemini_frame(marker, True)),
        )
    return _json(
        {
            "candidates": [_gemini_candidate(reply_text(marker), True)],
            "usageMetadata": {"promptTokenCount": 3, "candidatesTokenCount": 2, "totalTokenCount": 5},
        }
    )


def _embedding_list(model: str) -> Reply:
    return _json(
        {
            "object": "list",
            "data": [{"object": "embedding", "index": 0, "embedding": list(VECTOR)}],
            "model": model,
            "usage": {"prompt_tokens": 2, "total_tokens": 2},
        }
    )


def _openai_like(model: str) -> Callable[[Request], Reply]:
    def respond(request: Request) -> Reply:
        path: Final = _path(request)
        marker: Final = marker_of(request)
        if path.endswith("/embeddings"):
            return _embedding_list(model)
        if path.endswith("/audio/transcriptions"):
            return _json({"text": reply_text(marker), "model": model})
        return chat_reply(f"chatcmpl-{marker}", model, reply_text(marker), stream=_wants_stream(request))

    return respond


def _converse_event(kind: str, payload: Mapping[str, JsonValue]) -> bytes:
    headers: Final = {":event-type": kind, ":content-type": "application/json", ":message-type": "event"}
    return aws_event_stream_frame(headers, json.dumps(payload, separators=(",", ":")).encode())


def _bedrock_converse(request: Request) -> Reply:
    marker: Final = marker_of(request)
    usage: Final[dict[str, JsonValue]] = {"inputTokens": 3, "outputTokens": 2, "totalTokens": 5}
    if _path(request).endswith("/converse-stream"):
        events: Final[tuple[tuple[str, Mapping[str, JsonValue]], ...]] = (
            ("messageStart", {"role": "assistant"}),
            ("contentBlockDelta", {"delta": {"text": "reply "}, "contentBlockIndex": 0}),
            ("contentBlockDelta", {"delta": {"text": marker}, "contentBlockIndex": 0}),
            ("contentBlockStop", {"contentBlockIndex": 0}),
            ("messageStop", {"stopReason": "end_turn"}),
            ("metadata", {"usage": usage, "metrics": {"latencyMs": 1}}),
        )
        return Reply(
            content_type="application/vnd.amazon.eventstream",
            chunks=tuple(_converse_event(kind, payload) for kind, payload in events),
        )
    return _json(
        {
            "output": {"message": {"role": "assistant", "content": [{"text": reply_text(marker)}]}},
            "stopReason": "end_turn",
            "usage": usage,
            "metrics": {"latencyMs": 1},
        }
    )


def _cohere_rerank(request: Request) -> Reply:
    return _json(
        {
            "id": f"rerank-{marker_of(request)}",
            "results": [{"index": 0, "relevance_score": RELEVANCE}],
            "meta": {"billed_units": {"search_units": 1}},
        }
    )


_BY_PREFIX: Final[Mapping[str, Callable[[Request], Reply]]] = MappingProxyType(
    {
        "an": _anthropic,
        "bi": lambda request: _json(anthropic_message(marker_of(request))),
        "bc": _bedrock_converse,
        "gm": _gemini,
        "mi": _openai_like("mistral-small-latest"),
        "db": _openai_like("databricks-meta-llama-3-3-70b-instruct"),
        "co": _cohere_rerank,
    }
)


def join_peer(request: Request) -> Reply:
    return _BY_PREFIX[request.target.split("/")[1]](request)