The fastest, litest AI Gateway. Rust core with Python SDK. Call 100+ LLM APIs in OpenAI (or native) format with cost tracking, guardrails, load balancing, and logging [Bedrock, Azure, OpenAI, Anthropic, OpenAI, VertexAI, vLLM, Nvidia NIM]
Python ai-gatewayanthropicazure-openaibedrockgatewaylangchainlitellmllmllm-gatewayllmopsmcp-gatewayopenaiopenai-proxyrustrust-aivertex-ai
from __future__ import annotations
import asyncio
import json
import re
import threading
import time
import uuid
from collections.abc import Callable, Iterable, Iterator, Mapping
from dataclasses import dataclass
from itertools import chain
from pathlib import Path
from queue import SimpleQueue
from types import MappingProxyType
from typing import Final, Literal
from urllib.parse import urlsplit
import anthropic
import httpx
import jwt
import openai
import psutil
import pytest
import yaml
from cryptography.hazmat.primitives.asymmetric import rsa
from integration._support.client import (
JSON_OBJECT,
Gateway,
Scenario,
eventually,
gateway_from_environment,
object_value,
string_value,
)
from integration._support.database import read_rows
from integration._support.ports import allocate_port
from integration._support.process import graceful_stop_seconds, owned_proxy, owned_proxy_process
from integration._support.wire import Reply, Request, Wire, wire_server
from pydantic import JsonValue
TEAM_API_KEY: Final = "integration-rag-team-key"
WILDCARD_API_KEY: Final = "integration-rag-wildcard-key"
SHARED_API_KEY: Final = "integration-rag-shared-key"
ENVIRONMENT_API_KEY: Final = "integration-rag-environment-key"
CALLER_API_KEY: Final = "integration-rag-caller-key"
FORWARDED_API_KEY: Final = "integration-rag-forwarded-key"
CREDENTIAL_API_KEY: Final = "integration-rag-credential-key"
GEMINI_API_KEY: Final = "integration-rag-gemini-key"
AZURE_API_KEY: Final = "integration-rag-azure-key"
JWT_KEY_ID: Final = "integration-rag-team-provider-jwt"
WILDCARD_MODEL: Final = "openai/gpt-4o-mini"
BARE_WILDCARD_MODEL: Final = "gpt-4o-mini"
OPENAI: Final[Mapping[str, JsonValue]] = MappingProxyType({"custom_llm_provider": "openai"})
DOCUMENT: Final = ("document.txt", b"team provider credentials", "text/plain")
BATCH_INPUT_ROW: Final[Mapping[str, JsonValue]] = MappingProxyType(
{
"custom_id": "integration-rag-batch",
"method": "POST",
"url": "/v1/chat/completions",
"body": {"model": BARE_WILDCARD_MODEL, "messages": [{"role": "user", "content": "batch input"}]},
}
)
BATCH_INPUT: Final = ("batch.jsonl", (json.dumps(dict(BATCH_INPUT_ROW)) + "\n").encode(), "application/jsonl")
FAILING_STORE_NAMES: Final = MappingProxyType({"rag-upstream-401": 401, "rag-upstream-429": 429})
OWNED_PROXY_CELL_SECONDS: Final = 2 * graceful_stop_seconds() + 120
STARTED_WORKER: Final = re.compile(r"Started server process \[(\d+)\]")
PREFIXED: Final = re.compile(r"^/(team|wildcard|shared|environment|credential|caller|gemini|azure|bedrock)(/[^?]*)")
STORE_FILES: Final = re.compile(r"^/v1/vector_stores/([^/]+)/files$")
FILE_CONTENT: Final = re.compile(r"^/v1/files/([^/]+)/content$")
FILE_ITEM: Final = re.compile(r"^/v1/files/([^/]+)$")
BATCH_ITEM: Final = re.compile(r"^/v1/batches/([^/]+)$")
BATCH_CANCEL: Final = re.compile(r"^/v1/batches/([^/]+)/cancel$")
BEDROCK_DATA_SOURCES: Final = re.compile(r"^/knowledgebases/([^/]+)/datasources/$")
BEDROCK_DATA_SOURCE: Final = re.compile(r"^/knowledgebases/([^/]+)/datasources/([^/]+)$")
EMPTY_KNOWLEDGE_BASE: Final = "EMPTYKB000"
BUCKETLESS_KNOWLEDGE_BASE: Final = "NOBUCKET00"
WEB_DATA_SOURCE: Final = "WEBSOURCE0"
STORE_SEARCH: Final = re.compile(r"^(?:/v1|/openai)/vector_stores/([^/]+)/search$")
GEMINI_UPLOAD: Final = re.compile(r"^/upload/v1beta/fileSearchStores/([^:]+):uploadToFileSearchStore$")
JWT_TEAM_COUNT: Final = 10
AWS_AMBIENT_CREDENTIALS: Final = (
"AWS_PROFILE",
"AWS_SESSION_TOKEN",
"AWS_BEARER_TOKEN_BEDROCK",
"AWS_ROLE_ARN",
"AWS_ROLE_NAME",
"AWS_SESSION_NAME",
"AWS_WEB_IDENTITY_TOKEN_FILE",
)
pytestmark: Final = [pytest.mark.timeout(OWNED_PROXY_CELL_SECONDS), pytest.mark.shares_worker_state]
Signature = tuple[str, str, str, str | None]
@dataclass(frozen=True, slots=True)
class Call:
prefix: str
method: str
route: str
authorization: str | None
google_key: str | None
azure_key: str | None
@property
def signature(self) -> Signature:
return (self.prefix, self.method, self.route, self.authorization)
@dataclass(frozen=True, slots=True)
class Tenants:
shared_model: str
global_model: str
team_a: str
team_a_model: str
team_a_key: str
restricted_key: str
viewer_key: str
jwt_member: str
team_w: str
team_w_key: str
wildcard_bare_key: str
wildcard_prefixed_key: str
wildcard_global_key: str
team_s: str
team_s_key: str
teamless_key: str
jwt_teams: tuple[str, ...]
jwt_team_model: str
jwt_multi_member: str
azure_restricted_key: str
credential_name: str
unreachable_credential_name: str
@dataclass(frozen=True, slots=True)
class RagRig:
gateway: Gateway
upstream: Wire
outage: threading.Event
tenants: Tenants
signing_key: rsa.RSAPrivateKey
scenario: Scenario
def jwt(self, subject: str, *team_ids: JsonValue) -> str:
issued_at: Final = int(time.time())
return jwt.encode(
{"sub": subject, "groups": list(team_ids), "iat": issued_at, "exp": issued_at + 300},
self.signing_key,
algorithm="RS256",
headers={"kid": JWT_KEY_ID},
)
def calls(self) -> tuple[Call, ...]:
return tuple(_call(request) for request in self.upstream.drain())
def signatures(self) -> tuple[Signature, ...]:
return tuple(call.signature for call in self.calls())
def _call(request: Request) -> Call:
matched: Final = PREFIXED.match(request.target)
assert matched is not None, request.target
return Call(
prefix=matched[1],
method=request.method,
route=matched[2],
authorization=request.headers.get("authorization"),
google_key=request.headers.get("x-goog-api-key"),
azure_key=request.headers.get("api-key"),
)
def _json_body(request: Request) -> Mapping[str, JsonValue]:
if not request.body or not request.headers.get("content-type", "").startswith("application/json"):
return MappingProxyType({})
return object_value(json.loads(request.body))
def _json_reply(payload: Mapping[str, JsonValue], status: int = 200) -> Reply:
return Reply(status=status, body=json.dumps(payload).encode())
def _error_reply(status: int) -> Reply:
return _json_reply(
{"error": {"message": f"scripted provider {status}", "type": "invalid_request_error", "code": str(status)}},
status,
)
def _vector_store_reply(request: Request, outage: threading.Event) -> Reply:
if outage.is_set():
return _error_reply(503)
name: Final = str(_json_body(request).get("name"))
failing: Final = FAILING_STORE_NAMES.get(name)
if failing is not None:
return _error_reply(failing)
return _json_reply(
{
"id": f"vs_{uuid.uuid4().hex}",
"object": "vector_store",
"created_at": int(time.time()),
"name": name,
"usage_bytes": 0,
"status": "completed",
"file_counts": {"in_progress": 0, "completed": 0, "failed": 0, "cancelled": 0, "total": 0},
}
)
def _owned_by(prefix: str, identity: str) -> bool:
return identity.startswith((f"file-{prefix}-", f"batch_{prefix}_"))
def _file_object(identity: str) -> dict[str, JsonValue]:
return {
"id": identity,
"object": "file",
"bytes": len(DOCUMENT[1]),
"created_at": int(time.time()),
"filename": DOCUMENT[0],
"purpose": "assistants",
"status": "processed",
}
def _file_reply(prefix: str) -> Reply:
return _json_reply(_file_object(f"file-{prefix}-{uuid.uuid4().hex}"))
def _batch_input_reply() -> Reply:
return Reply(status=200, body=BATCH_INPUT[1])
def _batch_object(identity: str, input_file_id: str, status: str) -> dict[str, JsonValue]:
return {
"id": identity,
"object": "batch",
"endpoint": "/v1/chat/completions",
"input_file_id": input_file_id,
"completion_window": "24h",
"status": status,
"created_at": int(time.time()),
"errors": None,
"output_file_id": None,
"error_file_id": None,
"request_counts": {"total": 0, "completed": 0, "failed": 0},
"metadata": None,
}
def _batch_reply(request: Request, prefix: str) -> Reply:
body: Final = _json_body(request)
return _json_reply(
_batch_object(f"batch_{prefix}_{uuid.uuid4().hex}", str(body.get("input_file_id")), "validating")
)
def _list_reply() -> Reply:
return _json_reply({"object": "list", "data": [], "first_id": None, "last_id": None, "has_more": False})
def _owned_reply(prefix: str, identity: str, payload: Mapping[str, JsonValue]) -> Reply:
return _json_reply(payload) if _owned_by(prefix, identity) else _error_reply(404)
def _openai_get(route: str, prefix: str) -> Reply:
if route in {"/v1/files", "/v1/batches"}:
return _list_reply()
content: Final = FILE_CONTENT.match(route)
if content is not None:
return _batch_input_reply() if _owned_by(prefix, content[1]) else _error_reply(404)
file_item: Final = FILE_ITEM.match(route)
if file_item is not None:
return _owned_reply(prefix, file_item[1], _file_object(file_item[1]))
batch_item: Final = BATCH_ITEM.match(route)
if batch_item is not None:
return _owned_reply(prefix, batch_item[1], _batch_object(batch_item[1], f"file-{prefix}-input", "validating"))
return _error_reply(404)
def _openai_delete(route: str, prefix: str) -> Reply:
file_item: Final = FILE_ITEM.match(route)
if file_item is None:
return _error_reply(404)
return _owned_reply(prefix, file_item[1], {"id": file_item[1], "object": "file", "deleted": True})
def _embedding_reply(request: Request) -> Reply:
body: Final = _json_body(request)
inputs: Final = body.get("input")
count: Final = len(inputs) if isinstance(inputs, list) else 1
return _json_reply(
{
"object": "list",
"data": [{"object": "embedding", "index": index, "embedding": [0.1, 0.2, 0.3]} for index in range(count)],
"model": str(body.get("model")),
"usage": {"prompt_tokens": count, "total_tokens": count},
}
)
def _bedrock_reply(request: Request, route: str) -> Reply:
listed: Final = BEDROCK_DATA_SOURCES.match(route)
if request.method == "POST" and listed is not None:
summary: Final[dict[str, JsonValue]] = {
"knowledgeBaseId": listed[1],
"dataSourceId": WEB_DATA_SOURCE,
"name": "integration-web-source",
"status": "AVAILABLE",
"updatedAt": "2026-01-01T00:00:00Z",
}
return _json_reply({"dataSourceSummaries": [] if listed[1] == EMPTY_KNOWLEDGE_BASE else [summary]})
detail: Final = BEDROCK_DATA_SOURCE.match(route)
if request.method == "GET" and detail is not None:
return _json_reply(
{
"dataSource": {
"knowledgeBaseId": detail[1],
"dataSourceId": detail[2],
"name": "integration-web-source",
"status": "AVAILABLE",
"dataSourceConfiguration": {"type": "WEB"},
"createdAt": "2026-01-01T00:00:00Z",
"updatedAt": "2026-01-01T00:00:00Z",
}
}
)
return _error_reply(404)
def _attach_reply(request: Request, store_id: str) -> Reply:
return _json_reply(
{
"id": str(_json_body(request).get("file_id")),
"object": "vector_store.file",
"created_at": int(time.time()),
"vector_store_id": store_id,
"status": "completed",
"last_error": None,
"usage_bytes": 0,
}
)
def _search_reply(request: Request) -> Reply:
return _json_reply(
{
"object": "vector_store.search_results.page",
"search_query": _json_body(request).get("query", ""),
"data": [],
"has_more": False,
"next_page": None,
}
)
def _sse(payload: Mapping[str, JsonValue]) -> bytes:
return f"data: {json.dumps(payload)}\n\n".encode()
def _chunk(identity: str, model: str, delta: Mapping[str, JsonValue], finish_reason: str | None) -> bytes:
return _sse(
{
"id": identity,
"object": "chat.completion.chunk",
"created": int(time.time()),
"model": model,
"choices": [{"index": 0, "delta": dict(delta), "finish_reason": finish_reason}],
}
)
def _chat_reply(request: Request) -> Reply:
body: Final = _json_body(request)
model: Final = str(body.get("model"))
identity: Final = f"chatcmpl-{uuid.uuid4().hex}"
if body.get("stream") is True:
return Reply(
content_type="text/event-stream",
chunks=(
_chunk(identity, model, {"role": "assistant", "content": "integration response"}, None),
_chunk(identity, model, {}, "stop"),
b"data: [DONE]\n\n",
),
)
return _json_reply(
{
"id": identity,
"object": "chat.completion",
"created": int(time.time()),
"model": model,
"choices": [
{
"index": 0,
"message": {"role": "assistant", "content": "integration response"},
"finish_reason": "stop",
}
],
"usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2},
}
)
def _responses_reply(request: Request) -> Reply:
identity: Final = uuid.uuid4().hex
completed: Final = _responses_object(identity, str(_json_body(request).get("model")))
if _json_body(request).get("stream") is not True:
return _json_reply(completed)
events: Final[tuple[tuple[str, Mapping[str, JsonValue]], ...]] = (
(
"response.created",
{
"type": "response.created",
"sequence_number": 0,
"response": {**completed, "status": "in_progress", "output": []},
},
),
(
"response.output_text.delta",
{
"type": "response.output_text.delta",
"sequence_number": 1,
"item_id": f"msg_{identity}",
"output_index": 0,
"content_index": 0,
"delta": "integration response",
"logprobs": [],
},
),
("response.completed", {"type": "response.completed", "sequence_number": 2, "response": completed}),
)
return Reply(
content_type="text/event-stream",
chunks=tuple(f"event: {name}\ndata: {json.dumps(payload)}\n\n".encode() for name, payload in events),
)
def _responses_object(identity: str, model: str) -> dict[str, JsonValue]:
return {
"id": f"resp_{identity}",
"object": "response",
"created_at": int(time.time()),
"status": "completed",
"model": model,
"output": [
{
"type": "message",
"id": f"msg_{identity}",
"status": "completed",
"role": "assistant",
"content": [{"type": "output_text", "text": "integration response", "annotations": []}],
}
],
"parallel_tool_calls": False,
"tool_choice": "auto",
"tools": [],
"usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2},
}
def _gemini_reply(request: Request, route: str) -> Reply:
if request.method == "POST" and route == "/v1beta/fileSearchStores":
return _json_reply({"name": f"fileSearchStores/rag-{uuid.uuid4().hex}"})
upload: Final = GEMINI_UPLOAD.match(route)
if request.method == "POST" and upload is not None:
session_url: Final = f"http://{request.headers['host']}/gemini/upload-session/{upload[1]}"
return Reply(body=b"{}", headers={"x-goog-upload-url": session_url})
if request.method == "PUT" and route.startswith("/upload-session/"):
return _json_reply({"name": f"documents/{uuid.uuid4().hex}"})
return _error_reply(404)
def _openai_reply(request: Request, prefix: str, route: str, outage: threading.Event) -> Reply:
if request.method == "GET":
return _openai_get(route, prefix)
if request.method == "DELETE":
return _openai_delete(route, prefix)
if request.method != "POST":
return _error_reply(404)
if route == "/v1/vector_stores":
return _vector_store_reply(request, outage)
if route in {"/v1/files", "/openai/files"}:
return _file_reply(prefix)
if route == "/v1/batches":
return _batch_reply(request, prefix)
cancelled: Final = BATCH_CANCEL.match(route)
if cancelled is not None:
return _owned_reply(prefix, cancelled[1], _batch_object(cancelled[1], f"file-{prefix}-input", "cancelling"))
if route == "/v1/chat/completions":
return _chat_reply(request)
if route == "/v1/responses":
return _responses_reply(request)
if route == "/v1/embeddings":
return _embedding_reply(request)
attach: Final = STORE_FILES.match(route)
if attach is not None:
return _attach_reply(request, attach[1])
if STORE_SEARCH.match(route) is not None:
return _search_reply(request)
return _error_reply(404)
def _responder(outage: threading.Event) -> Callable[[Request], Reply]:
def respond(request: Request) -> Reply:
matched: Final = PREFIXED.match(request.target)
if matched is None:
return _error_reply(404)
if matched[1] == "gemini":
return _gemini_reply(request, matched[2])
if matched[1] == "bedrock":
return _bedrock_reply(request, matched[2])
return _openai_reply(request, matched[1], matched[2], outage)
return respond
def _proxy_config(directory: Path) -> Path:
base: Final = object_value(yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()))
general_settings: Final = object_value(base["general_settings"])
config: Final = {
**base,
"general_settings": {
**general_settings,
"enable_jwt_auth": True,
"litellm_jwtauth": {"user_id_jwt_field": "sub", "team_ids_jwt_field": "groups"},
"forward_llm_provider_auth_headers": True,
},
"files_settings": [{"custom_llm_provider": "openai"}, {"custom_llm_provider": "azure"}],
}
path: Final = directory / "rag_team_provider_credentials.yaml"
path.write_text(yaml.safe_dump(config))
return path
def _provider_environment(upstream: str) -> Mapping[str, str]:
return MappingProxyType(
{
"OPENAI_API_KEY": ENVIRONMENT_API_KEY,
"OPENAI_API_BASE": f"{upstream}/environment/v1",
"OPENAI_BASE_URL": f"{upstream}/environment/v1",
"GEMINI_API_KEY": GEMINI_API_KEY,
"GEMINI_API_BASE": f"{upstream}/gemini",
"AZURE_API_KEY": AZURE_API_KEY,
"AZURE_API_BASE": f"{upstream}/azure",
"AZURE_API_VERSION": "2025-04-01-preview",
}
)
def _aws_environment(upstream: str) -> Mapping[str, str]:
return MappingProxyType(
{
"AWS_ENDPOINT_URL_BEDROCK_AGENT": f"{upstream}/bedrock",
"AWS_ACCESS_KEY_ID": "integration-rag-aws-access-key",
"AWS_SECRET_ACCESS_KEY": "integration-rag-aws-secret-key",
"AWS_REGION_NAME": "us-east-1",
"AWS_DEFAULT_REGION": "us-east-1",
"AWS_EC2_METADATA_DISABLED": "true",
}
)
def _jwt_team_key(team_id: str) -> str:
return f"integration-rag-jwt-{team_id}"
def _jwt_team(gateway: Gateway, scenario: Scenario, upstream: str, model: str, member: str) -> str:
team_id: Final = scenario.team(models=[model])
_deployment(
gateway,
scenario,
name=model,
model="openai/gpt-4o-mini",
api_key=_jwt_team_key(team_id),
api_base=f"{upstream}/team/v1",
team_id=team_id,
)
gateway.post("/team/member_add", {"team_id": team_id, "member": {"user_id": member, "role": "user"}})
return team_id
def _deployment(
gateway: Gateway,
scenario: Scenario,
*,
name: str,
model: str,
api_key: str,
api_base: str,
team_id: str | None,
model_id: str | None = None,
) -> str:
created: Final = gateway.post(
"/model/new",
{
"model_name": name,
"litellm_params": {"model": model, "api_key": api_key, "api_base": api_base},
"model_info": ({} if team_id is None else {"team_id": team_id})
| ({} if model_id is None else {"id": model_id}),
},
)
model_id: Final = string_value(object_value(created["model_info"])["id"])
scenario.cleanups.callback(scenario.delete_model, model_id)
return model_id
def _delete_credential(gateway: Gateway, name: str) -> None:
response: Final = gateway.request("DELETE", f"/credentials/{name}")
assert response.status_code == 200, response.text
def _credential(gateway: Gateway, scenario: Scenario, api_base: str) -> str:
name: Final = f"integration-rag-credential-{uuid.uuid4().hex}"
gateway.post(
"/credentials",
{
"credential_name": name,
"credential_values": {"api_key": CREDENTIAL_API_KEY, "api_base": api_base},
"credential_info": {},
},
)
scenario.cleanups.callback(_delete_credential, gateway, name)
return name
def _tenants(gateway: Gateway, scenario: Scenario, upstream: str) -> Tenants:
shared_model: Final = f"integration-rag-shared-{uuid.uuid4().hex}"
_deployment(
gateway,
scenario,
name=shared_model,
model="openai/gpt-4o-mini",
api_key=SHARED_API_KEY,
api_base=f"{upstream}/shared/v1",
team_id=None,
)
global_model: Final = f"openai/integration-rag-global-{uuid.uuid4().hex}"
_deployment(
gateway,
scenario,
name=global_model,
model="openai/gpt-4o-mini",
api_key=SHARED_API_KEY,
api_base=f"{upstream}/shared/v1",
team_id=None,
)
team_a_model: Final = f"integration-rag-team-{uuid.uuid4().hex}"
team_a: Final = scenario.team(models=[team_a_model, shared_model])
_deployment(
gateway,
scenario,
name=team_a_model,
model="openai/gpt-4o-mini",
api_key=TEAM_API_KEY,
api_base=f"{upstream}/team/v1",
team_id=team_a,
)
viewer: Final = scenario.user(user_role="internal_user_viewer")
gateway.post("/team/member_add", {"team_id": team_a, "member": {"user_id": viewer, "role": "user"}})
team_w: Final = scenario.team(models=[shared_model])
_deployment(
gateway,
scenario,
name="openai/*",
model="openai/*",
api_key=WILDCARD_API_KEY,
api_base=f"{upstream}/wildcard/v1",
team_id=team_w,
)
team_s: Final = scenario.team(models=[shared_model])
team_z: Final = scenario.team(models=[shared_model])
_deployment(
gateway,
scenario,
name="azure/*",
model="azure/*",
api_key=WILDCARD_API_KEY,
api_base=f"{upstream}/azure",
team_id=team_z,
)
jwt_team_model: Final = f"integration-rag-jwt-{uuid.uuid4().hex}"
jwt_multi_member: Final = scenario.user(user_role="internal_user")
return Tenants(
shared_model=shared_model,
global_model=global_model,
team_a=team_a,
team_a_model=team_a_model,
team_a_key=scenario.key(team_id=team_a),
restricted_key=scenario.key(team_id=team_a, models=[shared_model]),
viewer_key=scenario.key(user_id=viewer, team_id=team_a),
jwt_member=scenario.member(team_a),
team_w=team_w,
team_w_key=scenario.key(team_id=team_w),
wildcard_bare_key=scenario.key(team_id=team_w, models=[BARE_WILDCARD_MODEL]),
wildcard_prefixed_key=scenario.key(team_id=team_w, models=[WILDCARD_MODEL]),
wildcard_global_key=scenario.key(team_id=team_w, models=[global_model]),
team_s=team_s,
team_s_key=scenario.key(team_id=team_s),
teamless_key=scenario.key(),
jwt_teams=tuple(
_jwt_team(gateway, scenario, upstream, jwt_team_model, jwt_multi_member) for _ in range(JWT_TEAM_COUNT)
),
jwt_team_model=jwt_team_model,
jwt_multi_member=jwt_multi_member,
azure_restricted_key=scenario.key(team_id=team_z, models=["azure/gpt-4o-mini"]),
credential_name=_credential(gateway, scenario, f"{upstream}/credential/v1"),
unreachable_credential_name=_credential(gateway, scenario, _closed_url()),
)
@pytest.fixture(scope="module")
def rag_rig(tmp_path_factory: pytest.TempPathFactory) -> Iterator[RagRig]:
signing_key: Final = rsa.generate_private_key(public_exponent=65537, key_size=2048)
jwks_body: Final = json.dumps(
{
"keys": [
{
**json.loads(jwt.algorithms.RSAAlgorithm.to_jwk(signing_key.public_key())),
"kid": JWT_KEY_ID,
"use": "sig",
"alg": "RS256",
}
]
}
).encode()
def jwks_reply(request: Request) -> Reply:
assert request.method == "GET", request
return Reply(body=jwks_body)
outage: Final = threading.Event()
directory: Final = tmp_path_factory.mktemp("rag_team_provider_credentials")
with (
gateway_from_environment() as admin_gateway,
admin_gateway.scenario() as scenario,
wire_server(_responder(outage)) as upstream,
wire_server(jwks_reply) as jwks,
):
tenants: Final = _tenants(admin_gateway, scenario, upstream.url)
with (
owned_proxy(
admin_gateway,
directory,
{
**_provider_environment(upstream.url),
**_aws_environment(upstream.url),
"JWT_PUBLIC_KEY_URL": jwks.url,
},
config=_proxy_config(directory),
remove_environment=("GOOGLE_API_KEY", *AWS_AMBIENT_CREDENTIALS),
workers=2,
) as gateway,
gateway.scenario() as owned_scenario,
):
yield RagRig(gateway, upstream, outage, tenants, signing_key, owned_scenario)
@pytest.fixture(autouse=True)
def _drain_calls_an_earlier_failed_cell_left(request: pytest.FixtureRequest) -> None:
if "rag_rig" in request.fixturenames:
request.getfixturevalue("rag_rig").upstream.drain()
def _ingest_request(name: str, vector_store: Mapping[str, JsonValue]) -> Mapping[str, str]:
return MappingProxyType(
{"request": json.dumps({"ingest_options": {"name": name, "vector_store": dict(vector_store)}})}
)
def _ingest(
gateway: Gateway,
key: str,
vector_store: Mapping[str, JsonValue] = OPENAI,
*,
name: str = "litellm-rag-ingest",
path: str = "/v1/rag/ingest",
) -> httpx.Response:
return gateway.request_multipart(path, _ingest_request(name, vector_store), {"file": DOCUMENT}, key=key)
def _query(
gateway: Gateway,
key: str,
model: str,
retrieval_config: Mapping[str, JsonValue],
*,
stream: bool = False,
path: str = "/v1/rag/query",
) -> httpx.Response:
return gateway.request(
"POST",
path,
{
"model": model,
"messages": [{"role": "user", "content": f"query team vector store {uuid.uuid4().hex}"}],
"retrieval_config": {"top_k": 1, **retrieval_config},
**({"stream": True} if stream else {}),
},
key=key,
)
def _ingest_signatures(prefix: str, api_key: str, store_id: str) -> tuple[Signature, ...]:
bearer: Final = f"Bearer {api_key}"
return (
(prefix, "POST", "/v1/vector_stores", bearer),
(prefix, "POST", "/v1/files", bearer),
(prefix, "POST", f"/v1/vector_stores/{store_id}/files", bearer),
)
def _search_signature(prefix: str, api_key: str, store_id: str) -> Signature:
return (prefix, "POST", f"/v1/vector_stores/{store_id}/search", f"Bearer {api_key}")
def _chat_signature(prefix: str, api_key: str) -> Signature:
return (prefix, "POST", "/v1/chat/completions", f"Bearer {api_key}")
def _completed_store(rig: RagRig, response: httpx.Response) -> str:
assert response.status_code == 200, response.text
body: Final = object_value(response.json())
assert body["status"] == "completed", response.text
store_id: Final = string_value(body["vector_store_id"])
rig.scenario.cleanups.callback(rig.gateway.post, "/vector_store/delete", {"vector_store_id": store_id})
return store_id
def _store_rows(store_id: str) -> list[dict[str, JsonValue]]:
return read_rows(
'SELECT vector_store_id, team_id, litellm_params, vector_store_metadata FROM "LiteLLM_ManagedVectorStoresTable" '
"WHERE vector_store_id = %s",
(store_id,),
)
def _closed_url() -> str:
return f"http://127.0.0.1:{allocate_port()}/v1"
def test_viewer_ingest_and_team_and_jwt_query_use_team_provider_credentials(rag_rig: RagRig) -> None:
tenants: Final = rag_rig.tenants
store_id: Final = _completed_store(rag_rig, _ingest(rag_rig.gateway, tenants.viewer_key))
assert rag_rig.signatures() == _ingest_signatures("team", TEAM_API_KEY, store_id)
team_query: Final = _query(
rag_rig.gateway, tenants.team_a_key, tenants.team_a_model, {**OPENAI, "vector_store_id": store_id}
)
assert team_query.status_code == 200, team_query.text
assert rag_rig.signatures() == (
_search_signature("team", TEAM_API_KEY, store_id),
_chat_signature("team", TEAM_API_KEY),
)
jwt_query: Final = _query(
rag_rig.gateway,
rag_rig.jwt(tenants.jwt_member, tenants.team_a),
tenants.team_a_model,
{**OPENAI, "vector_store_id": store_id},
)
assert jwt_query.status_code == 200, jwt_query.text
assert rag_rig.signatures() == (
_search_signature("team", TEAM_API_KEY, store_id),
_chat_signature("team", TEAM_API_KEY),
)
@pytest.mark.parametrize(
("name", "status"), (("rag-upstream-401", 401), ("rag-upstream-429", 429)), ids=("provider_401", "provider_429")
)
def test_failed_ingest_returns_the_provider_status_without_saving_a_store(
rag_rig: RagRig, name: str, status: int
) -> None:
before: Final = _store_rows("")
response: Final = _ingest(rag_rig.gateway, rag_rig.tenants.team_a_key, name=name)
assert response.status_code == status, response.text
assert f"scripted provider {status}" in response.text, response.text
assert rag_rig.signatures() == (("team", "POST", "/v1/vector_stores", f"Bearer {TEAM_API_KEY}"),)
assert _store_rows("") == before
def test_unreachable_credential_api_base_fails_ingest_without_saving_a_store(rag_rig: RagRig) -> None:
before: Final = _store_rows("")
response: Final = _ingest(
rag_rig.gateway,
rag_rig.tenants.team_a_key,
{**OPENAI, "litellm_credential_name": rag_rig.tenants.unreachable_credential_name},
)
assert response.status_code == 500, response.text
assert rag_rig.calls() == ()
assert _store_rows("") == before
def test_rag_alias_ingest_and_query_use_the_team_wildcard_deployment(rag_rig: RagRig) -> None:
store_id: Final = _completed_store(
rag_rig, _ingest(rag_rig.gateway, rag_rig.tenants.team_w_key, path="/rag/ingest")
)
assert rag_rig.signatures() == _ingest_signatures("wildcard", WILDCARD_API_KEY, store_id)
rows: Final = _store_rows(store_id)
assert len(rows) == 1, rows
assert WILDCARD_API_KEY not in json.dumps(rows), rows
response: Final = _query(
rag_rig.gateway,
rag_rig.tenants.team_w_key,
WILDCARD_MODEL,
{**OPENAI, "vector_store_id": store_id},
path="/rag/query",
)
assert response.status_code == 200, response.text
assert rag_rig.signatures() == (
_search_signature("wildcard", WILDCARD_API_KEY, store_id),
_chat_signature("wildcard", WILDCARD_API_KEY),
)
def test_jwt_team_member_ingest_uses_team_provider_credentials(rag_rig: RagRig) -> None:
token: Final = rag_rig.jwt(rag_rig.tenants.jwt_member, rag_rig.tenants.team_a)
store_id: Final = _completed_store(rag_rig, _ingest(rag_rig.gateway, token))
assert rag_rig.signatures() == _ingest_signatures("team", TEAM_API_KEY, store_id)
rows: Final = _store_rows(store_id)
assert [row["team_id"] for row in rows] == [rag_rig.tenants.team_a], rows
def test_caller_credential_name_wins_over_the_team_deployment(rag_rig: RagRig) -> None:
store_id: Final = _completed_store(
rag_rig,
_ingest(
rag_rig.gateway,
rag_rig.tenants.team_a_key,
{**OPENAI, "litellm_credential_name": rag_rig.tenants.credential_name},
),
)
assert rag_rig.signatures() == _ingest_signatures("credential", CREDENTIAL_API_KEY, store_id)
@pytest.mark.parametrize("field", ("api_key", "api_base"))
@pytest.mark.parametrize("value", ("set", "null", "empty"))
def test_caller_api_key_or_api_base_on_ingest_is_rejected_before_any_upstream_call(
rag_rig: RagRig, field: Literal["api_key", "api_base"], value: Literal["set", "null", "empty"]
) -> None:
set_values: Final = MappingProxyType({"api_key": CALLER_API_KEY, "api_base": f"{rag_rig.upstream.url}/caller/v1"})
sent: Final[JsonValue] = MappingProxyType({"set": set_values[field], "null": None, "empty": ""})[value]
response: Final = _ingest(rag_rig.gateway, rag_rig.tenants.team_a_key, {**OPENAI, field: sent})
assert response.status_code == 400, response.text
assert f"'{field}' cannot be set in ingest_options.vector_store" in response.text, response.text
assert rag_rig.calls() == ()
@pytest.mark.parametrize("blank", (None, ""), ids=("null", "empty"))
def test_blank_caller_credential_name_falls_back_to_the_team_deployment(rag_rig: RagRig, blank: str | None) -> None:
tenants: Final = rag_rig.tenants
store_id: Final = _completed_store(
rag_rig, _ingest(rag_rig.gateway, tenants.team_a_key, {**OPENAI, "litellm_credential_name": blank})
)
assert rag_rig.signatures() == _ingest_signatures("team", TEAM_API_KEY, store_id)
response: Final = _query(
rag_rig.gateway, tenants.team_a_key, tenants.team_a_model, {**OPENAI, "vector_store_id": store_id}
)
assert response.status_code == 200, response.text
assert rag_rig.signatures() == (
_search_signature("team", TEAM_API_KEY, store_id),
_chat_signature("team", TEAM_API_KEY),
)
@pytest.mark.parametrize("caller", ("teamless_key", "team_s_key", "restricted_key"))
def test_ingest_without_a_usable_team_deployment_uses_the_environment_credentials(
rag_rig: RagRig, caller: Literal["teamless_key", "team_s_key", "restricted_key"]
) -> None:
key: Final = getattr(rag_rig.tenants, caller)
store_id: Final = _completed_store(rag_rig, _ingest(rag_rig.gateway, key))
assert rag_rig.signatures() == _ingest_signatures("environment", ENVIRONMENT_API_KEY, store_id)
@pytest.mark.parametrize(
("caller", "prefix", "api_key"),
(("team_a_key", "team", TEAM_API_KEY), ("team_s_key", "shared", SHARED_API_KEY)),
ids=("team_deployment", "shared_deployment"),
)
def test_files_upload_keeps_the_accessible_deployment_fallback(
rag_rig: RagRig, caller: Literal["team_a_key", "team_s_key"], prefix: str, api_key: str
) -> None:
response: Final = rag_rig.gateway.request_multipart(
"/v1/files", {"purpose": "assistants"}, {"file": DOCUMENT}, key=getattr(rag_rig.tenants, caller)
)
assert response.status_code == 200, response.text
assert rag_rig.signatures() == ((prefix, "POST", "/v1/files", f"Bearer {api_key}"),)
@pytest.mark.parametrize(
("caller", "model"),
(("wildcard_bare_key", BARE_WILDCARD_MODEL), ("wildcard_prefixed_key", WILDCARD_MODEL)),
ids=("bare_model", "prefixed_model"),
)
def test_key_restricted_to_a_model_under_the_team_wildcard_uses_the_team_deployment(
rag_rig: RagRig, caller: Literal["wildcard_bare_key", "wildcard_prefixed_key"], model: str
) -> None:
key: Final = getattr(rag_rig.tenants, caller)
store_id: Final = _completed_store(rag_rig, _ingest(rag_rig.gateway, key))
assert rag_rig.signatures() == _ingest_signatures("wildcard", WILDCARD_API_KEY, store_id)
rows: Final = _store_rows(store_id)
assert [row["team_id"] for row in rows] == [rag_rig.tenants.team_w], rows
assert WILDCARD_API_KEY not in json.dumps(rows), rows
response: Final = _query(rag_rig.gateway, key, model, {**OPENAI, "vector_store_id": store_id})
assert response.status_code == 200, response.text
assert rag_rig.signatures() == (
_search_signature("wildcard", WILDCARD_API_KEY, store_id),
_chat_signature("wildcard", WILDCARD_API_KEY),
)
def test_key_restricted_to_a_global_model_under_the_team_wildcard_splits_rag_and_chat(rag_rig: RagRig) -> None:
tenants: Final = rag_rig.tenants
store_id: Final = _completed_store(rag_rig, _ingest(rag_rig.gateway, tenants.wildcard_global_key))
assert rag_rig.signatures() == _ingest_signatures("wildcard", WILDCARD_API_KEY, store_id)
response: Final = _query(
rag_rig.gateway, tenants.wildcard_global_key, tenants.global_model, {**OPENAI, "vector_store_id": store_id}
)
assert response.status_code == 200, response.text
assert rag_rig.signatures() == (
_search_signature("wildcard", WILDCARD_API_KEY, store_id),
_chat_signature("shared", SHARED_API_KEY),
)
def test_files_upload_by_a_key_restricted_under_the_team_wildcard_uses_the_team_deployment(rag_rig: RagRig) -> None:
response: Final = rag_rig.gateway.request_multipart(
"/v1/files", {"purpose": "assistants"}, {"file": DOCUMENT}, key=rag_rig.tenants.wildcard_bare_key
)
assert response.status_code == 200, response.text
assert rag_rig.signatures() == (("wildcard", "POST", "/v1/files", f"Bearer {WILDCARD_API_KEY}"),)
@pytest.mark.parametrize("batch_options", ({}, {"litellm_credential_name": "no-such-credential"}), ids=("plain", "credential_name"))
@pytest.mark.parametrize(
("caller", "prefix", "api_key", "reads_input"),
(("team_a_key", "team", TEAM_API_KEY, False), ("wildcard_bare_key", "wildcard", WILDCARD_API_KEY, True)),
ids=("team_key", "restricted_wildcard_key"),
)
def test_batch_create_reads_the_input_and_creates_the_batch_through_the_team_deployment(
rag_rig: RagRig,
caller: Literal["team_a_key", "wildcard_bare_key"],
prefix: str,
api_key: str,
reads_input: bool,
batch_options: Mapping[str, JsonValue],
) -> None:
key: Final = getattr(rag_rig.tenants, caller)
uploaded: Final = rag_rig.gateway.request_multipart(
"/v1/files", {"purpose": "batch"}, {"file": BATCH_INPUT}, key=key
)
assert uploaded.status_code == 200, uploaded.text
file_id: Final = string_value(object_value(uploaded.json())["id"])
created: Final = rag_rig.gateway.request(
"POST",
"/v1/batches",
{"input_file_id": file_id, "endpoint": "/v1/chat/completions", "completion_window": "24h", **batch_options},
key=key,
)
assert created.status_code == 200, created.text
assert object_value(created.json())["input_file_id"] == file_id, created.text
bearer: Final = f"Bearer {api_key}"
input_read: Final = ((prefix, "GET", f"/v1/files/{file_id}/content", bearer),) if reads_input else ()
assert rag_rig.signatures() == (
(prefix, "POST", "/v1/files", bearer),
*input_read,
(prefix, "POST", "/v1/batches", bearer),
)
def test_gemini_ingest_never_receives_the_team_openai_key(rag_rig: RagRig) -> None:
store_id: Final = _completed_store(
rag_rig, _ingest(rag_rig.gateway, rag_rig.tenants.team_a_key, {"custom_llm_provider": "gemini"})
)
store_name: Final = store_id.removeprefix("fileSearchStores/")
assert tuple(
(call.prefix, call.method, call.route, call.google_key, call.authorization) for call in rag_rig.calls()
) == (
("gemini", "POST", "/v1beta/fileSearchStores", GEMINI_API_KEY, None),
(
"gemini",
"POST",
f"/upload/v1beta/fileSearchStores/{store_name}:uploadToFileSearchStore",
GEMINI_API_KEY,
None,
),
("gemini", "PUT", f"/upload-session/{store_name}", None, None),
)
@pytest.mark.parametrize("provider", (7, ["openai"], "", "x" * 5120), ids=("int", "list", "empty", "five_kb"))
def test_malformed_ingest_provider_is_rejected_before_any_upstream_call(rag_rig: RagRig, provider: JsonValue) -> None:
response: Final = _ingest(rag_rig.gateway, rag_rig.tenants.team_a_key, {"custom_llm_provider": provider})
assert response.status_code == 400, response.text
assert rag_rig.calls() == ()
@pytest.mark.parametrize("stream", (False, True), ids=("json", "stream"))
def test_unmanaged_store_query_searches_with_the_team_deployment(rag_rig: RagRig, stream: bool) -> None:
tenants: Final = rag_rig.tenants
store_id: Final = f"vs_unmanaged_{uuid.uuid4().hex}"
response: Final = _query(
rag_rig.gateway,
tenants.team_a_key,
tenants.team_a_model,
{**OPENAI, "vector_store_id": store_id},
stream=stream,
)
assert response.status_code == 200, response.text
assert "integration response" in response.text, response.text
assert rag_rig.signatures() == (
_search_signature("team", TEAM_API_KEY, store_id),
_chat_signature("team", TEAM_API_KEY),
)
@pytest.mark.parametrize("case", ("api_key", "api_base", "null_api_key", "empty_api_key"))
def test_caller_credentials_in_retrieval_config_never_reach_an_unmanaged_search(
rag_rig: RagRig, case: Literal["api_key", "api_base", "null_api_key", "empty_api_key"]
) -> None:
tenants: Final = rag_rig.tenants
store_id: Final = f"vs_unmanaged_{uuid.uuid4().hex}"
caller_fields: Final[Mapping[str, JsonValue]] = MappingProxyType(
{
"api_key": {"api_key": CALLER_API_KEY},
"api_base": {"api_base": f"{rag_rig.upstream.url}/caller/v1"},
"null_api_key": {"api_key": None},
"empty_api_key": {"api_key": ""},
}[case]
)
response: Final = _query(
rag_rig.gateway,
tenants.team_a_key,
tenants.team_a_model,
{**OPENAI, **caller_fields, "vector_store_id": store_id},
)
assert response.status_code == 200, response.text
assert rag_rig.signatures() == (
_search_signature("team", TEAM_API_KEY, store_id),
_chat_signature("team", TEAM_API_KEY),
)
def _managed_store(rig: RagRig, scenario: Scenario, litellm_params: Mapping[str, JsonValue]) -> str:
store_id: Final = f"vs_managed_{uuid.uuid4().hex}"
rig.gateway.post(
"/vector_store/new",
{"vector_store_id": store_id, "custom_llm_provider": "openai", "litellm_params": dict(litellm_params)},
)
scenario.cleanups.callback(rig.gateway.post, "/vector_store/delete", {"vector_store_id": store_id})
return store_id
@pytest.mark.parametrize(
("stored", "prefix", "api_key"),
(("own_key", "caller", CALLER_API_KEY), ("null_key", "team", TEAM_API_KEY), ("empty_key", "team", TEAM_API_KEY)),
ids=("own_key", "null_key", "empty_key"),
)
def test_managed_store_credentials_decide_the_search_key(
rag_rig: RagRig, stored: Literal["own_key", "null_key", "empty_key"], prefix: str, api_key: str
) -> None:
tenants: Final = rag_rig.tenants
litellm_params: Final[Mapping[str, JsonValue]] = MappingProxyType(
{
"own_key": {"api_key": CALLER_API_KEY, "api_base": f"{rag_rig.upstream.url}/caller/v1"},
"null_key": {"api_key": None},
"empty_key": {"api_key": ""},
}[stored]
)
with rag_rig.gateway.scenario() as scenario:
store_id: Final = _managed_store(rag_rig, scenario, litellm_params)
response: Final = _query(
rag_rig.gateway, tenants.team_a_key, tenants.team_a_model, {**OPENAI, "vector_store_id": store_id}
)
assert response.status_code == 200, response.text
assert rag_rig.signatures() == (
_search_signature(prefix, api_key, store_id),
_chat_signature("team", TEAM_API_KEY),
)
def test_credential_name_in_retrieval_config_is_rejected_before_search(rag_rig: RagRig) -> None:
tenants: Final = rag_rig.tenants
response: Final = _query(
rag_rig.gateway,
tenants.team_a_key,
tenants.team_a_model,
{
**OPENAI,
"vector_store_id": f"vs_unmanaged_{uuid.uuid4().hex}",
"litellm_credential_name": tenants.credential_name,
},
)
assert response.status_code == 400, response.text
assert rag_rig.calls() == ()
def test_azure_query_never_receives_the_team_openai_key(rag_rig: RagRig) -> None:
tenants: Final = rag_rig.tenants
store_id: Final = f"vs_unmanaged_{uuid.uuid4().hex}"
response: Final = _query(
rag_rig.gateway,
tenants.team_a_key,
tenants.team_a_model,
{"custom_llm_provider": "azure", "vector_store_id": store_id},
)
assert response.status_code == 200, response.text
assert tuple((call.prefix, call.route, call.azure_key, call.authorization) for call in rag_rig.calls()) == (
("azure", f"/openai/vector_stores/{store_id}/search", AZURE_API_KEY, None),
("team", "/v1/chat/completions", None, f"Bearer {TEAM_API_KEY}"),
)
def test_another_team_cannot_query_a_team_store(rag_rig: RagRig) -> None:
tenants: Final = rag_rig.tenants
store_id: Final = _completed_store(rag_rig, _ingest(rag_rig.gateway, tenants.team_a_key))
assert rag_rig.signatures() == _ingest_signatures("team", TEAM_API_KEY, store_id)
response: Final = _query(
rag_rig.gateway, tenants.team_w_key, WILDCARD_MODEL, {**OPENAI, "vector_store_id": store_id}
)
assert response.status_code == 403, response.text
assert rag_rig.calls() == ()
@pytest.mark.parametrize("vector_store_id", (7, ["vs_a"], None), ids=("int", "list", "null"))
def test_non_string_vector_store_id_is_rejected_before_search(rag_rig: RagRig, vector_store_id: JsonValue) -> None:
tenants: Final = rag_rig.tenants
response: Final = _query(
rag_rig.gateway, tenants.team_a_key, tenants.team_a_model, {**OPENAI, "vector_store_id": vector_store_id}
)
assert response.status_code == 400, response.text
assert "vector_store_id must be a non-empty string" in response.text, response.text
assert rag_rig.calls() == ()
def test_empty_vector_store_id_is_rejected_before_search(rag_rig: RagRig) -> None:
tenants: Final = rag_rig.tenants
response: Final = _query(
rag_rig.gateway, tenants.team_a_key, tenants.team_a_model, {**OPENAI, "vector_store_id": ""}
)
assert (response.status_code, rag_rig.signatures()) == (400, ()), response.text
def test_five_kb_vector_store_id_searches_with_the_team_deployment(rag_rig: RagRig) -> None:
tenants: Final = rag_rig.tenants
store_id: Final = f"vs_{'v' * 5120}"
response: Final = _query(
rag_rig.gateway, tenants.team_a_key, tenants.team_a_model, {**OPENAI, "vector_store_id": store_id}
)
assert response.status_code == 200, response.text
assert rag_rig.signatures() == (
_search_signature("team", TEAM_API_KEY, store_id),
_chat_signature("team", TEAM_API_KEY),
)
@pytest.mark.parametrize("provider", (7, ["openai"], "", "x" * 5120), ids=("int", "list", "empty", "five_kb"))
def test_malformed_query_provider_fails_without_any_upstream_call(rag_rig: RagRig, provider: JsonValue) -> None:
tenants: Final = rag_rig.tenants
response: Final = _query(
rag_rig.gateway,
tenants.team_a_key,
tenants.team_a_model,
{"custom_llm_provider": provider, "vector_store_id": f"vs_unmanaged_{uuid.uuid4().hex}"},
)
assert (response.status_code, rag_rig.signatures()) == (500, ()), response.text
def test_jwt_route_outside_the_team_allowed_routes_names_the_route(rag_rig: RagRig) -> None:
tenants: Final = rag_rig.tenants
response: Final = rag_rig.gateway.request(
"POST",
"/v1beta/interactions",
{"model": tenants.team_a_model, "input": "hello"},
key=rag_rig.jwt(tenants.jwt_member, tenants.team_a),
)
assert response.status_code == 403, response.text
assert (
f"Team {tenants.team_a} can access model {tenants.team_a_model} but route /v1beta/interactions "
"is not in litellm_jwtauth.team_allowed_routes"
) in response.text, response.text
assert rag_rig.calls() == ()
def _llm_body(route: str, model: str) -> dict[str, JsonValue]:
prompt: Final = f"hello {uuid.uuid4().hex}"
if route == "/v1/responses":
return {"model": model, "input": prompt}
if route == "/v1/messages":
return {"model": model, "max_tokens": 16, "messages": [{"role": "user", "content": prompt}]}
return {"model": model, "messages": [{"role": "user", "content": prompt}]}
@pytest.mark.parametrize("route", ("/v1/chat/completions", "/v1/messages", "/v1/responses"))
def test_jwt_team_member_keeps_its_llm_routes(rag_rig: RagRig, route: str) -> None:
tenants: Final = rag_rig.tenants
response: Final = rag_rig.gateway.request(
"POST",
route,
_llm_body(route, tenants.team_a_model),
key=rag_rig.jwt(tenants.jwt_member, tenants.team_a),
headers={"anthropic-version": "2023-06-01"},
)
assert response.status_code == 200, response.text
calls: Final = rag_rig.calls()
assert [(call.prefix, call.authorization) for call in calls] == [("team", f"Bearer {TEAM_API_KEY}")], calls
async def _ingest_burst(rig: RagRig, names: tuple[str, ...]) -> tuple[httpx.Response, ...]:
async with httpx.AsyncClient(base_url=str(rig.gateway.client.base_url), timeout=60, trust_env=False) as client:
return tuple(
await asyncio.gather(
*(
client.post(
"/v1/rag/ingest",
data=_ingest_request(name, OPENAI),
files={"file": DOCUMENT},
headers={"Authorization": f"Bearer {rig.tenants.team_a_key}"},
)
for name in names
)
)
)
def _burst_names() -> tuple[str, ...]:
return tuple(f"rag-burst-{uuid.uuid4().hex}" for _ in range(10))
def _all_ingest_signatures(store_ids: Iterable[str]) -> list[Signature]:
return sorted(chain.from_iterable(_ingest_signatures("team", TEAM_API_KEY, store_id) for store_id in store_ids))
async def test_ingest_burst_during_a_provider_outage_fails_each_request_then_recovers(rag_rig: RagRig) -> None:
before: Final = _store_rows("")
rag_rig.outage.set()
try:
failed: Final = await _ingest_burst(rag_rig, _burst_names())
finally:
rag_rig.outage.clear()
assert [response.status_code for response in failed] == [503] * 10, [response.text for response in failed]
outage_calls: Final = rag_rig.signatures()
assert outage_calls == (("team", "POST", "/v1/vector_stores", f"Bearer {TEAM_API_KEY}"),) * 10, outage_calls
assert _store_rows("") == before
recovered: Final = await _ingest_burst(rag_rig, _burst_names())
store_ids: Final = tuple(_completed_store(rag_rig, response) for response in recovered)
assert len(frozenset(store_ids)) == 10, store_ids
assert sorted(rag_rig.signatures()) == _all_ingest_signatures(store_ids)
assert [[row["team_id"] for row in _store_rows(store_id)] for store_id in store_ids] == [
[rag_rig.tenants.team_a]
] * 10
def _open_upstream_connections(pid: int, upstream: str) -> int:
port: Final = urlsplit(upstream).port
return sum(
1
for connection in psutil.Process(pid).net_connections(kind="tcp")
if connection.status == psutil.CONN_ESTABLISHED and connection.raddr and connection.raddr.port == port
)
async def _query_burst(gateway: Gateway, key: str, model: str, count: int) -> tuple[httpx.Response, ...]:
async with httpx.AsyncClient(base_url=str(gateway.client.base_url), timeout=90, trust_env=False) as client:
results: Final = await asyncio.gather(
*(
client.post(
"/v1/rag/query",
json={
"model": model,
"messages": [{"role": "user", "content": f"held query {uuid.uuid4().hex}"}],
"retrieval_config": {**OPENAI, "vector_store_id": f"vs_burst_{uuid.uuid4().hex}", "top_k": 1},
},
headers={"Authorization": f"Bearer {key}"},
)
for _ in range(count)
),
return_exceptions=True,
)
for result in results:
assert not isinstance(result, BaseException) or isinstance(result, httpx.TransportError), repr(result)
return tuple(result for result in results if isinstance(result, httpx.Response))
async def test_worker_sigkill_mid_query_burst_leaves_the_sibling_on_team_credentials(tmp_path: Path) -> None:
release: Final = threading.Event()
held_searches: Final[SimpleQueue[str]] = SimpleQueue()
def respond(request: Request) -> Reply:
matched: Final = PREFIXED.match(request.target)
if matched is None:
return _error_reply(404)
if STORE_SEARCH.match(matched[2]) is not None:
held_searches.put(request.target)
assert release.wait(timeout=60), "The burst was never released"
return _search_reply(request)
return _openai_reply(request, matched[1], matched[2], threading.Event())
with (
gateway_from_environment() as admin_gateway,
admin_gateway.scenario() as scenario,
wire_server(respond) as upstream,
):
model: Final = f"integration-rag-kill-{uuid.uuid4().hex}"
team_id: Final = scenario.team(models=[model])
_deployment(
admin_gateway,
scenario,
name=model,
model="openai/gpt-4o-mini",
api_key=TEAM_API_KEY,
api_base=f"{upstream.url}/team/v1",
team_id=team_id,
)
key: Final = scenario.key(team_id=team_id)
with owned_proxy_process(
admin_gateway,
tmp_path,
_provider_environment(upstream.url),
remove_environment=("GOOGLE_API_KEY",),
workers=2,
) as owned:
workers: Final = eventually(
lambda: tuple(int(pid) for pid in STARTED_WORKER.findall(owned.log.read_text())),
lambda pids: len(pids) == 2,
seconds=30,
)
burst: Final = asyncio.create_task(_query_burst(owned.gateway, key, model, 20))
await asyncio.to_thread(eventually, held_searches.qsize, lambda size: size == 20, 60)
held_by: Final = MappingProxyType({pid: _open_upstream_connections(pid, upstream.url) for pid in workers})
assert sum(held_by.values()) == 20, held_by
victim_pid, survivor_pid = sorted(workers, key=held_by.__getitem__)
with owned.replacing_worker(victim_pid) as kill:
kill.freeze()
await asyncio.to_thread(kill)
release.set()
served: Final = await burst
assert held_by[survivor_pid] >= 10, held_by
assert [response.status_code for response in served] == [200] * held_by[survivor_pid], held_by
(follow_up,) = await _query_burst(owned.gateway, key, model, 1)
assert follow_up.status_code == 200, follow_up.text
calls: Final = tuple(_call(request) for request in upstream.drain())
assert {(call.prefix, call.authorization) for call in calls} == {("team", f"Bearer {TEAM_API_KEY}")}, calls
assert sum(1 for call in calls if call.route == "/v1/chat/completions") == held_by[survivor_pid] + 1, calls
def _ingest_and_drain(
gateway: Gateway, scenario: Scenario, key: str, upstream: Wire
) -> tuple[str, tuple[Signature, ...]]:
upstream.drain()
response: Final = _ingest(gateway, key)
assert response.status_code == 200, response.text
store_id: Final = string_value(object_value(response.json())["vector_store_id"])
scenario.cleanups.callback(gateway.post, "/vector_store/delete", {"vector_store_id": store_id})
calls: Final = tuple(_call(request) for request in upstream.drain())
return store_id, tuple(call.signature for call in calls if call.route != "/v1/models")
def _ingests_after_editing(
gateway: Gateway, scenario: Scenario, key: str, upstream: Wire, model_id: str
) -> tuple[tuple[str, tuple[Signature, ...]], ...]:
edited: Final = gateway.request("PATCH", f"/model/{model_id}/update", {"litellm_params": {"timeout": 30}})
assert edited.status_code == 200, edited.text
return tuple(_ingest_and_drain(gateway, scenario, key, upstream) for _ in range(4))
def test_a_team_with_two_openai_deployments_keeps_ingesting_through_the_older_one_after_either_is_edited(
tmp_path: Path,
) -> None:
with (
gateway_from_environment() as admin_gateway,
admin_gateway.scenario() as scenario,
wire_server(_responder(threading.Event())) as upstream,
):
team_model: Final = f"integration-rag-team-{uuid.uuid4().hex}"
wildcard_model: Final = f"integration-rag-wildcard-{uuid.uuid4().hex}"
team_id: Final = scenario.team(models=[team_model, wildcard_model])
team_deployment: Final = (
_deployment(
admin_gateway,
scenario,
name=team_model,
model="openai/gpt-4o-mini",
api_key=TEAM_API_KEY,
api_base=f"{upstream.url}/team/v1",
team_id=team_id,
model_id=f"z-{uuid.uuid4().hex}",
),
"team",
TEAM_API_KEY,
)
wildcard_deployment: Final = (
_deployment(
admin_gateway,
scenario,
name=wildcard_model,
model="openai/gpt-4o-mini",
api_key=WILDCARD_API_KEY,
api_base=f"{upstream.url}/wildcard/v1",
team_id=team_id,
model_id=f"a-{uuid.uuid4().hex}",
),
"wildcard",
WILDCARD_API_KEY,
)
(picked_id, picked_prefix, picked_key), (other_id, _, _) = team_deployment, wildcard_deployment
key: Final = scenario.key(team_id=team_id)
with (
owned_proxy(
admin_gateway,
tmp_path,
_provider_environment(upstream.url),
remove_environment=("GOOGLE_API_KEY",),
workers=2,
) as gateway,
gateway.scenario() as owned_scenario,
):
ingests: Final = (
_ingest_and_drain(gateway, owned_scenario, key, upstream),
*_ingests_after_editing(gateway, owned_scenario, key, upstream, picked_id),
*_ingests_after_editing(gateway, owned_scenario, key, upstream, other_id),
)
assert [calls for _, calls in ingests] == [
_ingest_signatures(picked_prefix, picked_key, store_id) for store_id, _ in ingests
]
@pytest.mark.parametrize(
("stored", "prefix", "api_key"),
(("own_key", "caller", CALLER_API_KEY), ("null_key", "team", TEAM_API_KEY)),
ids=("own_key", "null_key"),
)
def test_ingest_into_a_managed_store_uses_its_own_key_else_the_team_key(
rag_rig: RagRig, stored: Literal["own_key", "null_key"], prefix: str, api_key: str
) -> None:
litellm_params: Final[Mapping[str, JsonValue]] = MappingProxyType(
{
"own_key": {"api_key": CALLER_API_KEY, "api_base": f"{rag_rig.upstream.url}/caller/v1"},
"null_key": {"api_key": None},
}[stored]
)
with rag_rig.gateway.scenario() as scenario:
store_id: Final = _managed_store(rag_rig, scenario, litellm_params)
response: Final = _ingest(rag_rig.gateway, rag_rig.tenants.team_a_key, {**OPENAI, "vector_store_id": store_id})
assert response.status_code == 200, response.text
assert object_value(response.json())["vector_store_id"] == store_id, response.text
bearer: Final = f"Bearer {api_key}"
assert rag_rig.signatures() == (
(prefix, "POST", "/v1/files", bearer),
(prefix, "POST", f"/v1/vector_stores/{store_id}/files", bearer),
)
def _file_id_ingest(gateway: Gateway, key: str, vector_store: Mapping[str, JsonValue]) -> httpx.Response:
return gateway.request(
"POST",
"/v1/rag/ingest",
{
"file_id": "file-team-existing",
"ingest_options": {"name": "litellm-rag-ingest", "vector_store": dict(vector_store)},
},
key=key,
)
@pytest.mark.parametrize(
("vector_store", "ingestion"),
(
({"custom_llm_provider": "gemini"}, "GeminiRAGIngestion"),
({"custom_llm_provider": "bedrock", "vector_store_id": BUCKETLESS_KNOWLEDGE_BASE}, "BedrockRAGIngestion"),
({"custom_llm_provider": "s3_vectors", "vector_bucket_name": "integration-bucket"}, "S3VectorsRAGIngestion"),
({"custom_llm_provider": "vertex_ai"}, "VertexAIRAGIngestion"),
),
ids=("gemini", "bedrock", "s3_vectors", "vertex_ai"),
)
def test_existing_file_id_on_a_store_that_cannot_take_one_is_rejected_before_any_upstream_call(
rag_rig: RagRig, vector_store: Mapping[str, JsonValue], ingestion: str
) -> None:
before: Final = _store_rows("")
response: Final = _file_id_ingest(rag_rig.gateway, rag_rig.tenants.team_a_key, vector_store)
assert response.status_code == 400, response.text
assert f"{ingestion} does not support ingesting an existing file_id" in response.text, response.text
assert rag_rig.calls() == ()
assert _store_rows("") == before
def test_openai_file_id_without_a_vector_store_id_is_rejected_before_any_upstream_call(rag_rig: RagRig) -> None:
before: Final = _store_rows("")
response: Final = _file_id_ingest(rag_rig.gateway, rag_rig.tenants.team_a_key, OPENAI)
assert response.status_code == 400, response.text
assert "vector_store_id is required when ingesting an existing file_id" in response.text, response.text
assert rag_rig.calls() == ()
assert _store_rows("") == before
def test_openai_file_id_attaches_to_an_existing_store_with_the_team_key(rag_rig: RagRig) -> None:
store_id: Final = f"vs_existing_{uuid.uuid4().hex}"
response: Final = _file_id_ingest(
rag_rig.gateway, rag_rig.tenants.team_a_key, {**OPENAI, "vector_store_id": store_id}
)
assert _completed_store(rag_rig, response) == store_id
assert rag_rig.signatures() == (("team", "POST", f"/v1/vector_stores/{store_id}/files", f"Bearer {TEAM_API_KEY}"),)
@pytest.mark.parametrize(
("knowledge_base", "error", "routes"),
(
(
EMPTY_KNOWLEDGE_BASE,
f"No data sources found for Knowledge Base {EMPTY_KNOWLEDGE_BASE}",
(("POST", f"/knowledgebases/{EMPTY_KNOWLEDGE_BASE}/datasources/"),),
),
(
BUCKETLESS_KNOWLEDGE_BASE,
f"Could not auto-detect S3 bucket for data source {WEB_DATA_SOURCE}",
(
("POST", f"/knowledgebases/{BUCKETLESS_KNOWLEDGE_BASE}/datasources/"),
("GET", f"/knowledgebases/{BUCKETLESS_KNOWLEDGE_BASE}/datasources/{WEB_DATA_SOURCE}"),
),
),
),
ids=("no_data_source", "no_s3_bucket"),
)
def test_bedrock_knowledge_base_the_caller_named_without_a_usable_data_source_fails_ingest_with_400(
rag_rig: RagRig, knowledge_base: str, error: str, routes: tuple[tuple[str, str], ...]
) -> None:
before: Final = _store_rows("")
response: Final = _ingest(
rag_rig.gateway,
rag_rig.tenants.team_a_key,
{"custom_llm_provider": "bedrock", "vector_store_id": knowledge_base},
)
assert response.status_code == 400, response.text
assert error in response.text, response.text
assert tuple((call.prefix, call.method, call.route) for call in rag_rig.calls()) == tuple(
("bedrock", method, route) for method, route in routes
)
assert _store_rows("") == before
def test_s3_vectors_bucket_name_the_caller_sent_is_rejected_with_400_after_embedding(rag_rig: RagRig) -> None:
tenants: Final = rag_rig.tenants
before: Final = _store_rows("")
request: Final = {
"request": json.dumps(
{
"ingest_options": {
"name": "litellm-rag-ingest",
"embedding": {"model": tenants.shared_model},
"vector_store": {"custom_llm_provider": "s3_vectors", "vector_bucket_name": "ab", "dimension": 3},
}
}
)
}
response: Final = rag_rig.gateway.request_multipart(
"/v1/rag/ingest", request, {"file": DOCUMENT}, key=tenants.team_a_key
)
assert response.status_code == 400, response.text
assert "Invalid vector_bucket_name 'ab'" in response.text, response.text
assert rag_rig.signatures() == (("shared", "POST", "/v1/embeddings", f"Bearer {SHARED_API_KEY}"),)
assert _store_rows("") == before
@dataclass(frozen=True, slots=True)
class KeylessRig:
gateway: Gateway
upstream: Wire
key: str
@pytest.fixture(scope="module")
def keyless_rig(tmp_path_factory: pytest.TempPathFactory) -> Iterator[KeylessRig]:
directory: Final = tmp_path_factory.mktemp("rag_keyless_providers")
with (
gateway_from_environment() as admin_gateway,
admin_gateway.scenario() as scenario,
wire_server(_responder(threading.Event())) as upstream,
):
key: Final = scenario.key()
with owned_proxy(
admin_gateway,
directory,
{"OPENAI_API_KEY": ENVIRONMENT_API_KEY, "OPENAI_API_BASE": f"{upstream.url}/environment/v1"},
remove_environment=("GEMINI_API_KEY", "GOOGLE_API_KEY", "VERTEXAI_PROJECT"),
workers=2,
) as gateway:
yield KeylessRig(gateway, upstream, key)
@pytest.mark.parametrize(
("provider", "status", "error"),
(
("gemini", 500, "GEMINI_API_KEY or GOOGLE_API_KEY is required for Gemini File Search"),
("vertex_ai", 400, "vertex_project is required for Vertex AI RAG ingestion"),
),
ids=("gemini_without_a_proxy_key", "vertex_ai_without_a_project"),
)
def test_ingest_status_says_whether_the_caller_or_the_proxy_must_fix_the_missing_setting(
keyless_rig: KeylessRig, provider: str, status: int, error: str
) -> None:
before: Final = _store_rows("")
response: Final = _ingest(keyless_rig.gateway, keyless_rig.key, {"custom_llm_provider": provider})
assert response.status_code == status, response.text
assert error in response.text, response.text
assert tuple(keyless_rig.upstream.drain()) == ()
assert _store_rows("") == before
async def _jwt_chat_burst(rig: RagRig, token: str, model: str, count: int) -> tuple[httpx.Response, ...]:
async with httpx.AsyncClient(base_url=str(rig.gateway.client.base_url), timeout=60, trust_env=False) as client:
return tuple(
await asyncio.gather(
*(
client.post(
"/v1/chat/completions",
json=_llm_body("/v1/chat/completions", model),
headers={"Authorization": f"Bearer {token}"},
)
for _ in range(count)
)
)
)
async def test_jwt_in_ten_teams_that_hold_the_model_always_resolves_the_lowest_team_id(rag_rig: RagRig) -> None:
tenants: Final = rag_rig.tenants
token: Final = rag_rig.jwt(tenants.jwt_multi_member, *reversed(tenants.jwt_teams))
responses: Final = await _jwt_chat_burst(rag_rig, token, tenants.jwt_team_model, 16)
assert [response.status_code for response in responses] == [200] * 16, [response.text for response in responses]
assert (
rag_rig.signatures()
== (("team", "POST", "/v1/chat/completions", f"Bearer {_jwt_team_key(min(tenants.jwt_teams))}"),) * 16
)
def _prompt() -> str:
return f"hello {uuid.uuid4().hex}"
def _sync_sdk_text(route: str, base_url: str, token: str, model: str, stream: bool) -> str:
if route == "/v1/messages":
messages_client: Final = anthropic.Anthropic(base_url=base_url, api_key=token, max_retries=0)
request: Final = {"model": model, "max_tokens": 16, "messages": [{"role": "user", "content": _prompt()}]}
if stream:
return "".join(
event.delta.text
for event in messages_client.messages.create(**request, stream=True)
if event.type == "content_block_delta" and event.delta.type == "text_delta"
)
message: Final = messages_client.messages.create(**request)
return "".join(block.text for block in message.content if block.type == "text")
client: Final = openai.OpenAI(base_url=f"{base_url}/v1", api_key=token, max_retries=0)
if route == "/v1/responses":
if stream:
return "".join(
event.delta
for event in client.responses.create(model=model, input=_prompt(), stream=True)
if event.type == "response.output_text.delta"
)
return client.responses.create(model=model, input=_prompt()).output_text
chat_messages: Final = [{"role": "user", "content": _prompt()}]
if stream:
return "".join(
chunk.choices[0].delta.content or ""
for chunk in client.chat.completions.create(model=model, messages=chat_messages, stream=True)
if chunk.choices
)
return client.chat.completions.create(model=model, messages=chat_messages).choices[0].message.content or ""
async def _async_sdk_text(route: str, base_url: str, token: str, model: str, stream: bool) -> str:
if route == "/v1/messages":
messages_client: Final = anthropic.AsyncAnthropic(base_url=base_url, api_key=token, max_retries=0)
request: Final = {"model": model, "max_tokens": 16, "messages": [{"role": "user", "content": _prompt()}]}
if stream:
return "".join(
[
event.delta.text
async for event in await messages_client.messages.create(**request, stream=True)
if event.type == "content_block_delta" and event.delta.type == "text_delta"
]
)
message: Final = await messages_client.messages.create(**request)
return "".join(block.text for block in message.content if block.type == "text")
client: Final = openai.AsyncOpenAI(base_url=f"{base_url}/v1", api_key=token, max_retries=0)
if route == "/v1/responses":
if stream:
return "".join(
[
event.delta
async for event in await client.responses.create(model=model, input=_prompt(), stream=True)
if event.type == "response.output_text.delta"
]
)
return (await client.responses.create(model=model, input=_prompt())).output_text
chat_messages: Final = [{"role": "user", "content": _prompt()}]
if stream:
return "".join(
[
chunk.choices[0].delta.content or ""
async for chunk in await client.chat.completions.create(
model=model, messages=chat_messages, stream=True
)
if chunk.choices
]
)
completion: Final = await client.chat.completions.create(model=model, messages=chat_messages)
return completion.choices[0].message.content or ""
@pytest.mark.parametrize("stream", (False, True), ids=("json", "stream"))
@pytest.mark.parametrize("client", ("sync", "async"))
@pytest.mark.parametrize("route", ("/v1/chat/completions", "/v1/messages", "/v1/responses"))
async def test_jwt_team_member_reaches_its_llm_routes_through_the_sdks(
rag_rig: RagRig, route: str, client: Literal["sync", "async"], stream: bool
) -> None:
tenants: Final = rag_rig.tenants
base_url: Final = str(rag_rig.gateway.client.base_url).rstrip("/")
token: Final = rag_rig.jwt(tenants.jwt_member, tenants.team_a)
text: Final = (
await asyncio.to_thread(_sync_sdk_text, route, base_url, token, tenants.team_a_model, stream)
if client == "sync"
else await _async_sdk_text(route, base_url, token, tenants.team_a_model, stream)
)
assert text == "integration response", text
calls: Final = rag_rig.calls()
assert [(call.prefix, call.authorization) for call in calls] == [("team", f"Bearer {TEAM_API_KEY}")], calls
def _batch_for(rig: RagRig, headers: Mapping[str, str]) -> tuple[str, str]:
uploaded: Final = rig.gateway.client.post(
"/v1/files", data={"purpose": "batch"}, files={"file": BATCH_INPUT}, headers=dict(headers)
)
assert uploaded.status_code == 200, uploaded.text
file_id: Final = string_value(object_value(uploaded.json())["id"])
created: Final = rig.gateway.request(
"POST",
"/v1/batches",
{"input_file_id": file_id, "endpoint": "/v1/chat/completions", "completion_window": "24h"},
headers=headers,
)
assert created.status_code == 200, created.text
rig.upstream.drain()
return file_id, string_value(object_value(created.json())["id"])
FOLLOW_UP_CALLS: Final = (
("GET", "/v1/files/{file_id}", "/v1/files/{file_id}"),
("GET", "/v1/files/{file_id}/content", "/v1/files/{file_id}/content"),
("DELETE", "/v1/files/{file_id}", "/v1/files/{file_id}"),
("GET", "/v1/files?purpose=batch&custom_llm_provider=openai", "/v1/files"),
("GET", "/v1/batches/{batch_id}", "/v1/batches/{batch_id}"),
("POST", "/v1/batches/{batch_id}/cancel", "/v1/batches/{batch_id}/cancel"),
)
FOLLOW_UP_CALL_IDS: Final = (
"file_retrieve",
"file_content",
"file_delete",
"file_list",
"batch_retrieve",
"batch_cancel",
)
def _bearer(key: str) -> Mapping[str, str]:
return MappingProxyType({"Authorization": f"Bearer {key}"})
@pytest.mark.parametrize(("method", "path", "route"), FOLLOW_UP_CALLS, ids=FOLLOW_UP_CALL_IDS)
def test_files_and_batches_calls_by_a_key_restricted_under_the_team_wildcard_use_the_team_deployment(
rag_rig: RagRig, method: str, path: str, route: str
) -> None:
key: Final = rag_rig.tenants.wildcard_bare_key
file_id, batch_id = _batch_for(rag_rig, _bearer(key))
response: Final = rag_rig.gateway.request(method, path.format(file_id=file_id, batch_id=batch_id), key=key)
assert response.status_code == 200, response.text
assert rag_rig.signatures() == (
("wildcard", method, route.format(file_id=file_id, batch_id=batch_id), f"Bearer {WILDCARD_API_KEY}"),
)
def test_batch_list_by_a_key_restricted_under_the_team_wildcard_is_answered_by_the_proxy_without_an_upstream_call(
rag_rig: RagRig,
) -> None:
key: Final = rag_rig.tenants.wildcard_bare_key
_batch_for(rag_rig, _bearer(key))
response: Final = rag_rig.gateway.request("GET", "/v1/batches?custom_llm_provider=openai", key=key)
assert response.status_code == 200, response.text
assert object_value(response.json())["object"] == "list", response.text
assert rag_rig.signatures() == ()
def _upload_batch_input(rig: RagRig, key: str) -> str:
uploaded: Final = rig.gateway.request_multipart("/v1/files", {"purpose": "batch"}, {"file": BATCH_INPUT}, key=key)
assert uploaded.status_code == 200, uploaded.text
return string_value(object_value(uploaded.json())["id"])
def _create_batch(rig: RagRig, key: str, file_id: str) -> httpx.Response:
return rig.gateway.request(
"POST",
"/v1/batches",
{"input_file_id": file_id, "endpoint": "/v1/chat/completions", "completion_window": "24h"},
key=key,
)
@pytest.mark.parametrize(
("team", "prefix", "api_key"),
(("team_a", "team", TEAM_API_KEY), ("team_s", "shared", SHARED_API_KEY)),
ids=("team_deployment", "accessible_deployment"),
)
def test_batch_over_the_key_tpm_limit_is_rejected_after_reading_its_input_where_it_was_uploaded(
rag_rig: RagRig, team: Literal["team_a", "team_s"], prefix: str, api_key: str
) -> None:
key: Final = rag_rig.scenario.key(team_id=getattr(rag_rig.tenants, team), tpm_limit=1)
file_id: Final = _upload_batch_input(rag_rig, key)
response: Final = _create_batch(rag_rig, key, file_id)
assert response.status_code == 429, response.text
bearer: Final = f"Bearer {api_key}"
assert rag_rig.signatures() == (
(prefix, "POST", "/v1/files", bearer),
(prefix, "GET", f"/v1/files/{file_id}/content", bearer),
)
def test_batch_whose_input_names_a_model_outside_the_key_allowlist_is_rejected_before_creation(
rag_rig: RagRig,
) -> None:
key: Final = rag_rig.tenants.restricted_key
file_id: Final = _upload_batch_input(rag_rig, key)
response: Final = _create_batch(rag_rig, key, file_id)
assert response.status_code == 403, response.text
assert BARE_WILDCARD_MODEL in response.text, response.text
bearer: Final = f"Bearer {SHARED_API_KEY}"
assert rag_rig.signatures() == (
("shared", "POST", "/v1/files", bearer),
("shared", "GET", f"/v1/files/{file_id}/content", bearer),
)
def test_azure_files_upload_by_a_key_restricted_under_the_team_azure_wildcard_uses_the_team_deployment(
rag_rig: RagRig,
) -> None:
response: Final = rag_rig.gateway.request_multipart(
"/v1/files",
{"purpose": "assistants", "custom_llm_provider": "azure"},
{"file": DOCUMENT},
key=rag_rig.tenants.azure_restricted_key,
)
assert response.status_code == 200, response.text
assert tuple((call.prefix, call.method, call.route, call.azure_key) for call in rag_rig.calls()) == (
("azure", "POST", "/openai/files", WILDCARD_API_KEY),
)
def test_query_without_a_vector_store_id_is_rejected_before_search(rag_rig: RagRig) -> None:
tenants: Final = rag_rig.tenants
response: Final = _query(rag_rig.gateway, tenants.team_a_key, tenants.team_a_model, OPENAI)
assert response.status_code == 400, response.text
assert "retrieval_config must contain 'vector_store_id'" in response.text, response.text
assert rag_rig.calls() == ()
def _forwarding(key: str) -> Mapping[str, str]:
return MappingProxyType({"Authorization": f"Bearer {key}", "x-api-key": FORWARDED_API_KEY})
@pytest.mark.parametrize(
("stored", "stream", "search"),
(
(None, False, ("team", FORWARDED_API_KEY)),
(None, True, ("team", FORWARDED_API_KEY)),
("own_key", False, ("caller", CALLER_API_KEY)),
("null_key", False, ("team", FORWARDED_API_KEY)),
),
ids=("unmanaged_json", "unmanaged_stream", "managed_own_key", "managed_null_key"),
)
def test_forwarded_provider_key_on_query_wins_over_team_provider_credentials(
rag_rig: RagRig, stored: Literal["own_key", "null_key"] | None, stream: bool, search: tuple[str, str]
) -> None:
tenants: Final = rag_rig.tenants
with rag_rig.gateway.scenario() as scenario:
store_id: Final = (
f"vs_unmanaged_{uuid.uuid4().hex}"
if stored is None
else _managed_store(
rag_rig,
scenario,
{"api_key": CALLER_API_KEY, "api_base": f"{rag_rig.upstream.url}/caller/v1"}
if stored == "own_key"
else {"api_key": None},
)
)
response: Final = rag_rig.gateway.request(
"POST",
"/v1/rag/query",
{
"model": tenants.team_a_model,
"messages": [{"role": "user", "content": f"query forwarded key {uuid.uuid4().hex}"}],
"retrieval_config": {**OPENAI, "top_k": 1, "vector_store_id": store_id},
**({"stream": True} if stream else {}),
},
headers=_forwarding(tenants.team_a_key),
)
assert response.status_code == 200, response.text
assert "integration response" in response.text, response.text
assert rag_rig.signatures() == (
_search_signature(search[0], search[1], store_id),
_chat_signature("team", FORWARDED_API_KEY),
)
def test_forwarded_provider_key_on_ingest_wins_over_team_provider_credentials(rag_rig: RagRig) -> None:
tenants: Final = rag_rig.tenants
response: Final = rag_rig.gateway.client.post(
"/v1/rag/ingest",
data=dict(_ingest_request("litellm-rag-forwarded-key", OPENAI)),
files={"file": DOCUMENT},
headers=dict(_forwarding(tenants.team_a_key)),
)
store_id: Final = _completed_store(rag_rig, response)
assert rag_rig.signatures() == _ingest_signatures("team", FORWARDED_API_KEY, store_id)
@pytest.mark.parametrize(
("caller", "prefix"),
(("team_a_key", "team"), ("wildcard_bare_key", "wildcard")),
ids=("team_key", "restricted_wildcard_key"),
)
def test_forwarded_provider_key_on_files_and_batches_wins_over_team_provider_credentials(
rag_rig: RagRig, caller: Literal["team_a_key", "wildcard_bare_key"], prefix: str
) -> None:
headers: Final = _forwarding(getattr(rag_rig.tenants, caller))
uploaded: Final = rag_rig.gateway.client.post(
"/v1/files", data={"purpose": "batch"}, files={"file": BATCH_INPUT}, headers=dict(headers)
)
assert uploaded.status_code == 200, uploaded.text
file_id: Final = string_value(object_value(uploaded.json())["id"])
created: Final = rag_rig.gateway.request(
"POST",
"/v1/batches",
{"input_file_id": file_id, "endpoint": "/v1/chat/completions", "completion_window": "24h"},
headers=headers,
)
assert created.status_code == 200, created.text
calls: Final = rag_rig.calls()
assert [(call.prefix, call.authorization) for call in calls] == [
(prefix, f"Bearer {FORWARDED_API_KEY}")
] * len(calls), calls
assert {(call.method, call.route) for call in calls} >= {("POST", "/v1/files"), ("POST", "/v1/batches")}, calls
@pytest.mark.parametrize("stray_claim", (7, None), ids=("int", "null"))
@pytest.mark.parametrize("route", ("/v1/chat/completions", "/v1/rag/query"))
def test_jwt_team_claim_with_a_non_string_entry_still_resolves_the_string_team(
rag_rig: RagRig, route: str, stray_claim: JsonValue
) -> None:
tenants: Final = rag_rig.tenants
token: Final = rag_rig.jwt(tenants.jwt_member, tenants.team_a, stray_claim)
store_id: Final = f"vs_unmanaged_{uuid.uuid4().hex}"
response: Final = (
_query(rag_rig.gateway, token, tenants.team_a_model, {**OPENAI, "vector_store_id": store_id})
if route == "/v1/rag/query"
else rag_rig.gateway.request("POST", route, _llm_body(route, tenants.team_a_model), key=token)
)
assert response.status_code == 200, response.text
search: Final = (_search_signature("team", TEAM_API_KEY, store_id),) if route == "/v1/rag/query" else ()
assert rag_rig.signatures() == (*search, _chat_signature("team", TEAM_API_KEY))
@pytest.mark.parametrize("stray_claim", (7, None), ids=("int", "null"))
def test_jwt_header_team_outside_a_claim_with_a_non_string_entry_is_denied_with_403(
rag_rig: RagRig, stray_claim: JsonValue
) -> None:
tenants: Final = rag_rig.tenants
header_team: Final = f"integration-rag-unknown-team-{uuid.uuid4().hex}"
response: Final = rag_rig.gateway.request(
"POST",
"/v1/chat/completions",
_llm_body("/v1/chat/completions", tenants.team_a_model),
key=rag_rig.jwt(tenants.jwt_member, tenants.team_a, stray_claim),
headers={"x-litellm-team-id": header_team},
)
assert response.status_code == 403, response.text
assert f"x-litellm-team-id '{header_team}' does not resolve to a team id or a unique team alias" in response.text, (
response.text
)
assert f"'{tenants.team_a}'" in response.text, response.text
assert rag_rig.calls() == ()
@pytest.mark.parametrize(("method", "path", "route"), FOLLOW_UP_CALLS, ids=FOLLOW_UP_CALL_IDS)
@pytest.mark.parametrize(
("caller", "prefix"),
(("team_a_key", "team"), ("wildcard_bare_key", "wildcard")),
ids=("team_key", "restricted_wildcard_key"),
)
def test_forwarded_provider_key_on_files_and_batches_follow_up_calls_replaces_the_team_deployment_key(
rag_rig: RagRig,
caller: Literal["team_a_key", "wildcard_bare_key"],
prefix: str,
method: str,
path: str,
route: str,
) -> None:
headers: Final = _forwarding(getattr(rag_rig.tenants, caller))
file_id, batch_id = _batch_for(rag_rig, headers)
response: Final = rag_rig.gateway.request(method, path.format(file_id=file_id, batch_id=batch_id), headers=headers)
assert response.status_code == 200, response.text
assert rag_rig.signatures() == (
(prefix, method, route.format(file_id=file_id, batch_id=batch_id), f"Bearer {FORWARDED_API_KEY}"),
)
@pytest.mark.parametrize(
("x_api_key", "prefix", "api_key"),
(
(("",), "wildcard", WILDCARD_API_KEY),
(("f" * 5120,), "wildcard", "f" * 5120),
((FORWARDED_API_KEY, FORWARDED_API_KEY), "wildcard", FORWARDED_API_KEY),
),
ids=("empty", "five_kb", "repeated"),
)
def test_blank_oversized_or_repeated_forwarded_provider_key_on_files_and_batches(
rag_rig: RagRig, x_api_key: tuple[str, ...], prefix: str, api_key: str
) -> None:
headers: Final = (
("Authorization", f"Bearer {rag_rig.tenants.wildcard_bare_key}"),
*(("x-api-key", value) for value in x_api_key),
)
uploaded: Final = rag_rig.gateway.client.post(
"/v1/files", data={"purpose": "batch"}, files={"file": BATCH_INPUT}, headers=headers
)
assert uploaded.status_code == 200, uploaded.text
file_id: Final = string_value(JSON_OBJECT.validate_json(uploaded.content)["id"])
created: Final = rag_rig.gateway.client.post(
"/v1/batches",
json={"input_file_id": file_id, "endpoint": "/v1/chat/completions", "completion_window": "24h"},
headers=headers,
)
assert created.status_code == 200, created.text
calls: Final = rag_rig.calls()
assert [(call.prefix, call.authorization) for call in calls] == [(prefix, f"Bearer {api_key}")] * len(calls), calls
assert {(call.method, call.route) for call in calls} == {
("POST", "/v1/files"),
("GET", f"/v1/files/{file_id}/content"),
("POST", "/v1/batches"),
}, calls
def _forwarded_ingest(rig: RagRig, vector_store: Mapping[str, JsonValue]) -> httpx.Response:
return rig.gateway.client.post(
"/v1/rag/ingest",
data=dict(_ingest_request("litellm-rag-forwarded-key", vector_store)),
files={"file": DOCUMENT},
headers=dict(_forwarding(rig.tenants.team_a_key)),
)
def test_forwarded_provider_key_on_ingest_never_overrides_a_named_credential(rag_rig: RagRig) -> None:
store_id: Final = _completed_store(
rag_rig, _forwarded_ingest(rag_rig, {**OPENAI, "litellm_credential_name": rag_rig.tenants.credential_name})
)
assert rag_rig.signatures() == _ingest_signatures("credential", CREDENTIAL_API_KEY, store_id)
@pytest.mark.parametrize(
("stored", "prefix", "api_key"),
(("own_key", "caller", CALLER_API_KEY), ("null_key", "team", FORWARDED_API_KEY)),
ids=("own_key", "null_key"),
)
def test_forwarded_provider_key_on_ingest_into_a_managed_store_yields_only_to_the_store_key(
rag_rig: RagRig, stored: Literal["own_key", "null_key"], prefix: str, api_key: str
) -> None:
with rag_rig.gateway.scenario() as scenario:
store_id: Final = _managed_store(
rag_rig,
scenario,
{"api_key": CALLER_API_KEY, "api_base": f"{rag_rig.upstream.url}/caller/v1"}
if stored == "own_key"
else {"api_key": None},
)
response: Final = _forwarded_ingest(rag_rig, {**OPENAI, "vector_store_id": store_id})
assert response.status_code == 200, response.text
assert JSON_OBJECT.validate_json(response.content)["vector_store_id"] == store_id, response.text
bearer: Final = f"Bearer {api_key}"
assert rag_rig.signatures() == (
(prefix, "POST", "/v1/files", bearer),
(prefix, "POST", f"/v1/vector_stores/{store_id}/files", bearer),
)
def test_forwarded_provider_key_never_reaches_a_gemini_ingest(rag_rig: RagRig) -> None:
store_id: Final = _completed_store(rag_rig, _forwarded_ingest(rag_rig, {"custom_llm_provider": "gemini"}))
requests: Final = rag_rig.upstream.drain()
store_name: Final = store_id.removeprefix("fileSearchStores/")
assert tuple(
(call.prefix, call.method, call.route, call.google_key, call.authorization)
for call in (_call(request) for request in requests)
) == (
("gemini", "POST", "/v1beta/fileSearchStores", GEMINI_API_KEY, None),
(
"gemini",
"POST",
f"/upload/v1beta/fileSearchStores/{store_name}:uploadToFileSearchStore",
GEMINI_API_KEY,
None,
),
("gemini", "PUT", f"/upload-session/{store_name}", None, None),
)
assert [request.headers.get("x-api-key") for request in requests] == [None] * len(requests), requests