The fastest, litest AI Gateway. Rust core with Python SDK. Call 100+ LLM APIs in OpenAI (or native) format with cost tracking, guardrails, load balancing, and logging [Bedrock, Azure, OpenAI, Anthropic, OpenAI, VertexAI, vLLM, Nvidia NIM]
Python ai-gatewayanthropicazure-openaibedrockgatewaylangchainlitellmllmllm-gatewayllmopsmcp-gatewayopenaiopenai-proxyrustrust-aivertex-ai
"""
Test for google_endpoints/endpoints.py
"""
from typing import Final
import pytest
from dotenv import load_dotenv
from pydantic import JsonValue
from starlette.requests import Request
from litellm.proxy.google_endpoints.endpoints import _google_count_tokens_request, google_count_tokens
from litellm.types.llms.vertex_ai import TokenCountDetailsResponse
@pytest.mark.parametrize(
("system_key", "schema_key"),
(("systemInstruction", "parameters"), ("system_instruction", "parametersJsonSchema")),
)
def test_google_count_tokens_translates_system_and_function_declarations(
system_key: str, schema_key: str
) -> None:
schema: Final = {"type": "object", "properties": {"query": {"type": "string"}}}
request_body: Final[dict[str, JsonValue]] = {
"contents": [{"role": "user", "parts": [{"text": "Look this up"}]}],
system_key: {"parts": [{"text": "Use the provided tool."}]},
"tools": [
{
"functionDeclarations": [
{
"name": "lookup",
"description": "Find a record",
schema_key: schema,
}
]
}
],
}
request: Final = _google_count_tokens_request("claude-sonnet", request_body)
assert request.system == [{"type": "text", "text": "Use the provided tool."}]
assert request.tools == [
{
"name": "lookup",
"description": "Find a record",
"input_schema": schema,
}
]
assert request.messages == [{"role": "user", "content": "Look this up"}]
@pytest.mark.parametrize(
("schema_key", "expected_schema"),
(
("parameters", {"type": "object", "properties": {"query": {"type": "string"}}}),
("parametersJsonSchema", {"type": "OBJECT", "properties": {"query": {"type": "STRING"}}}),
),
)
def test_google_count_tokens_normalizes_parameters_only(
schema_key: str, expected_schema: dict[str, JsonValue]
) -> None:
request: Final = _google_count_tokens_request(
"claude-sonnet",
{
"contents": [{"role": "user", "parts": [{"text": "Look this up"}]}],
"tools": [
{
"functionDeclarations": [
{
"name": "lookup",
schema_key: {
"type": "OBJECT",
"properties": {"query": {"type": "STRING"}},
},
}
]
}
],
},
)
assert request.tools == [{"name": "lookup", "input_schema": expected_schema}]
load_dotenv()
@pytest.mark.asyncio
async def test_proxy_gemini_to_openai_like_model_token_counting():
"""
Test the token counting endpoint for proxing gemini to openai-like models.
"""
response: TokenCountDetailsResponse = await google_count_tokens(
request=Request(
scope={
"type": "http",
"parsed_body": (
["contents"],
{"contents": [{"parts": [{"text": "Hello, how are you?"}]}]},
),
}
),
model_name="volcengine/foo",
)
assert response.get("totalTokens") > 0