The fastest, litest AI Gateway. Rust core with Python SDK. Call 100+ LLM APIs in OpenAI (or native) format with cost tracking, guardrails, load balancing, and logging [Bedrock, Azure, OpenAI, Anthropic, OpenAI, VertexAI, vLLM, Nvidia NIM]
Python ai-gatewayanthropicazure-openaibedrockgatewaylangchainlitellmllmllm-gatewayllmopsmcp-gatewayopenaiopenai-proxyrustrust-aivertex-ai
import json
from typing import Final
import httpx
import pytest
import respx
import litellm
from openai.types import CreateEmbeddingResponse
_SDK_EMBEDDING_MODELS: Final = (
"openai/sdk-compat",
"mistral/sdk-compat",
"fireworks_ai/accounts/fireworks/models/sdk-compat",
"together_ai/sdk-compat",
"nvidia_nim/sdk-compat",
)
def _mock_openai_embedding_route(
respx_mock: respx.MockRouter, api_base: str = "https://api.openai.com/v1"
) -> respx.Route:
return respx_mock.post(f"{api_base}/embeddings").mock(
return_value=httpx.Response(
200,
json={
"object": "list",
"data": [
{"object": "embedding", "index": 0, "embedding": [0.1, 0.2, 0.3]},
{"object": "embedding", "index": 1, "embedding": [0.4, 0.5, 0.6]},
],
"model": "text-embedding-3-small",
"usage": {"prompt_tokens": 2, "total_tokens": 2},
},
)
)
@pytest.fixture(autouse=True)
def clear_default_encoding_format_env(monkeypatch: pytest.MonkeyPatch):
monkeypatch.delenv("LITELLM_DEFAULT_EMBEDDING_ENCODING_FORMAT", raising=False)
@pytest.mark.parametrize("model", _SDK_EMBEDDING_MODELS)
def test_embedding_sdk_providers_omit_encoding_format_when_client_omits_it(
respx_mock: respx.MockRouter, model: str
) -> None:
provider, upstream_model = model.split("/", 1)
api_base: Final = f"https://{provider.replace('_', '-')}.example/v1"
mock_route: Final = _mock_openai_embedding_route(respx_mock, api_base)
response: Final = litellm.embedding(model=model, input=["hello"], api_key="sk-test", api_base=api_base)
request_body: Final = json.loads(mock_route.calls.last.request.read())
assert "encoding_format" not in request_body
assert request_body["model"] == upstream_model
assert request_body["input"] == ["hello"]
assert mock_route.calls.last.request.headers["authorization"] == "Bearer sk-test"
assert response.data[0]["embedding"] == [0.1, 0.2, 0.3]
def test_embedding_openai_forwards_explicit_encoding_format(respx_mock: respx.MockRouter) -> None:
mock_route: Final = _mock_openai_embedding_route(respx_mock)
litellm.embedding(
model="openai/text-embedding-3-small", input=["hello"], api_key="sk-test", encoding_format="base64"
)
request_body: Final = json.loads(mock_route.calls.last.request.read())
assert request_body["encoding_format"] == "base64"
def test_embedding_openai_explicit_encoding_format_wins_over_env_var(
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setenv("LITELLM_DEFAULT_EMBEDDING_ENCODING_FORMAT", "float")
mock_route: Final = _mock_openai_embedding_route(respx_mock)
litellm.embedding(
model="openai/text-embedding-3-small", input=["hello"], api_key="sk-test", encoding_format="base64"
)
request_body: Final = json.loads(mock_route.calls.last.request.read())
assert request_body["encoding_format"] == "base64"
@pytest.mark.parametrize("env_value", ["float", "base64"])
def test_embedding_openai_env_var_sets_default_encoding_format(
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, env_value: str
) -> None:
monkeypatch.setenv("LITELLM_DEFAULT_EMBEDDING_ENCODING_FORMAT", env_value)
mock_route: Final = _mock_openai_embedding_route(respx_mock)
litellm.embedding(model="openai/text-embedding-3-small", input=["hello"], api_key="sk-test")
request_body: Final = json.loads(mock_route.calls.last.request.read())
assert request_body["encoding_format"] == env_value
@pytest.mark.parametrize("env_none", ["none", "NONE", " none "])
def test_embedding_openai_env_none_omits_encoding_format(
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, env_none: str
) -> None:
monkeypatch.setenv("LITELLM_DEFAULT_EMBEDDING_ENCODING_FORMAT", env_none)
mock_route: Final = _mock_openai_embedding_route(respx_mock)
litellm.embedding(model="openai/text-embedding-3-small", input=["hello"], api_key="sk-test")
request_body: Final = json.loads(mock_route.calls.last.request.read())
assert "encoding_format" not in request_body
@pytest.mark.asyncio
@pytest.mark.parametrize("model", _SDK_EMBEDDING_MODELS)
async def test_aembedding_sdk_providers_omit_encoding_format_when_client_omits_it(
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, model: str
) -> None:
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
provider, upstream_model = model.split("/", 1)
api_base: Final = f"https://{provider.replace('_', '-')}.example/v1"
mock_route: Final = _mock_openai_embedding_route(respx_mock, api_base)
response: Final = await litellm.aembedding(model=model, input=["hello"], api_key="sk-test", api_base=api_base)
request_body: Final = json.loads(mock_route.calls.last.request.read())
assert "encoding_format" not in request_body
assert request_body["model"] == upstream_model
assert request_body["input"] == ["hello"]
assert mock_route.calls.last.request.headers["authorization"] == "Bearer sk-test"
assert response.data[0]["embedding"] == [0.1, 0.2, 0.3]
def test_embedding_openai_omitted_encoding_format_maps_provider_errors(
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
) -> None:
respx_mock.post("https://api.openai.com/v1/embeddings").mock(
return_value=httpx.Response(
429,
headers={"retry-after": "42", "x-should-retry": "false"},
json={"error": {"message": "rate limited", "type": "rate_limit_error"}},
)
)
with pytest.raises(litellm.RateLimitError) as exc_info:
litellm.embedding(
model="openai/text-embedding-3-small", input=["hello"], api_key="sk-test", max_retries=0
)
assert int(exc_info.value.litellm_response_headers["retry-after"]) == 42
@pytest.mark.parametrize(
("model", "url", "extra_params"),
(
("openai/text-embedding-3-small", "https://api.openai.com/v1/embeddings", {}),
(
"azure/migration-embedding-deployment",
"https://migration.openai.azure.com/openai/deployments/migration-embedding-deployment/embeddings",
{"api_base": "https://migration.openai.azure.com", "api_version": "2024-10-21"},
),
),
)
def test_embedding_response_preserves_rate_limit_headers(
respx_mock: respx.MockRouter, model: str, url: str, extra_params: dict[str, str]
) -> None:
respx_mock.post(url__startswith=url).mock(
return_value=httpx.Response(
200,
json={
"object": "list",
"data": [{"object": "embedding", "index": 0, "embedding": [0.1, 0.2, 0.3]}],
"model": "text-embedding-3-small",
"usage": {"prompt_tokens": 1, "total_tokens": 1},
},
headers={
"x-ratelimit-limit-requests": "10",
"x-ratelimit-remaining-requests": "9",
"x-ratelimit-remaining-tokens": "99",
},
)
)
response: Final = litellm.embedding(model=model, input=["Hello world"], api_key="sk-test", **extra_params)
additional_headers: Final = response._hidden_params["additional_headers"]
assert {
name: additional_headers[name]
for name in (
"x-ratelimit-remaining-requests",
"x-ratelimit-remaining-tokens",
"llm_provider-x-ratelimit-remaining-requests",
"llm_provider-x-ratelimit-remaining-tokens",
)
} == {
"x-ratelimit-remaining-requests": "9",
"x-ratelimit-remaining-tokens": "99",
"llm_provider-x-ratelimit-remaining-requests": "9",
"llm_provider-x-ratelimit-remaining-tokens": "99",
}
def _openai_sdk_response_keys(route: respx.Route) -> frozenset[str]:
return frozenset(dict(CreateEmbeddingResponse.model_validate(route.return_value.json())))
def test_openai_embedding_returns_sdk_shaped_vectors(respx_mock: respx.MockRouter) -> None:
route: Final = _mock_openai_embedding_route(respx_mock)
response: Final = litellm.embedding(
model="openai/text-embedding-3-small",
input=["first", "second"],
api_key="sk-test",
)
assert frozenset(dict(response)) - {"_response_ms"} == _openai_sdk_response_keys(route)
assert [row["embedding"] for row in response.data] == [[0.1, 0.2, 0.3], [0.4, 0.5, 0.6]]
assert len(route.calls) == 1
def test_openai_embedding_forwards_dimensions(respx_mock: respx.MockRouter) -> None:
route: Final = _mock_openai_embedding_route(respx_mock)
response: Final = litellm.embedding(
model="openai/text-embedding-3-small",
input=["first", "second"],
api_key="sk-test",
dimensions=5,
)
body: Final = json.loads(route.calls.last.request.read())
assert body["dimensions"] == 5
assert body["input"] == ["first", "second"]
assert frozenset(dict(response)) - {"_response_ms"} == _openai_sdk_response_keys(route)
assert len(response.data) == 2
@pytest.mark.asyncio
async def test_aembedding_openai_returns_the_mocked_vector(
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
_mock_openai_embedding_route(respx_mock)
response: Final = await litellm.aembedding(
model="openai/text-embedding-3-small",
input=["good morning from litellm", "this is another item"],
api_key="sk-test",
)
cost: Final = litellm.completion_cost(
completion_response=response,
custom_cost_per_token={"input_cost_per_token": 0.001, "output_cost_per_token": 0.0},
)
assert [row["embedding"] for row in response.data] == [[0.1, 0.2, 0.3], [0.4, 0.5, 0.6]]
assert cost == pytest.approx(0.002)