The fastest, litest AI Gateway. Rust core with Python SDK. Call 100+ LLM APIs in OpenAI (or native) format with cost tracking, guardrails, load balancing, and logging [Bedrock, Azure, OpenAI, Anthropic, OpenAI, VertexAI, vLLM, Nvidia NIM]
Python ai-gatewayanthropicazure-openaibedrockgatewaylangchainlitellmllmllm-gatewayllmopsmcp-gatewayopenaiopenai-proxyrustrust-aivertex-ai
import asyncio
import logging
import re
import threading
from unittest.mock import MagicMock, mock_open, patch
import pytest
import yaml
from litellm.proxy.common_utils.load_config_utils import (
gcs_config_bucket,
get_config_from_bucket,
get_file_contents_from_s3,
resolve_bucket_includes,
)
from tests._master_key import MASTER_KEY
class TestGetFileContentsFromS3:
"""Test suite for S3 config loading functionality."""
@patch("boto3.client")
@patch("litellm.main.bedrock_converse_chat_completion")
@patch("yaml.safe_load")
def test_get_file_contents_from_s3_no_temp_file_creation(
self, mock_yaml_load, mock_bedrock, mock_boto3_client
):
"""
Test that get_file_contents_from_s3 doesn't create temporary files
and uses yaml.safe_load directly on the S3 response content.
Note: It's critical that yaml.safe_load is used
Relevant issue/PR: https://github.com/BerriAI/litellm/pull/12078
"""
# Mock credentials
mock_credentials = MagicMock()
mock_credentials.access_key = "test_access_key"
mock_credentials.secret_key = "test_secret_key"
mock_credentials.token = "test_token"
mock_bedrock.get_credentials.return_value = mock_credentials
# Mock S3 client and response
mock_s3_client = MagicMock()
mock_boto3_client.return_value = mock_s3_client
# Mock S3 response with YAML content
yaml_content = """
model_list:
- model_name: gpt-3.5-turbo
litellm_params:
model: gpt-3.5-turbo
"""
mock_response_body = MagicMock()
mock_response_body.read.return_value = yaml_content.encode("utf-8")
mock_s3_response = {"Body": mock_response_body}
mock_s3_client.get_object.return_value = mock_s3_response
# Mock yaml.safe_load to return parsed config
expected_config = {
"model_list": [
{
"model_name": "gpt-3.5-turbo",
"litellm_params": {"model": "gpt-3.5-turbo"},
}
]
}
mock_yaml_load.return_value = expected_config
# Call the function
bucket_name = "test-bucket"
object_key = "config.yaml"
result = get_file_contents_from_s3(bucket_name, object_key)
# Assertions
assert result == expected_config
# Verify S3 client was created with correct credentials
mock_boto3_client.assert_called_once_with(
"s3",
aws_access_key_id="test_access_key",
aws_secret_access_key="test_secret_key",
aws_session_token="test_token",
)
# Verify S3 get_object was called with correct parameters
mock_s3_client.get_object.assert_called_once_with(
Bucket=bucket_name, Key=object_key
)
# Verify the response body was read and decoded
mock_response_body.read.assert_called_once()
# Verify yaml.safe_load was called with the decoded content
mock_yaml_load.assert_called_once_with(yaml_content)
class TestBucketConfigIncludes:
@staticmethod
def _bucket(objects):
async def fetch(object_key):
return objects.get(object_key)
return fetch
@pytest.mark.asyncio
async def test_include_resolves_against_the_config_objects_prefix(self):
merged = await resolve_bucket_includes(
config={"include": ["model_config.yaml"], "general_settings": {"master_key": MASTER_KEY}},
object_key="configs/prod/config.yaml",
fetch=self._bucket(
{"configs/prod/model_config.yaml": {"model_list": [{"model_name": "gpt-4o-mini"}]}}
),
)
assert merged == {
"general_settings": {"master_key": MASTER_KEY},
"model_list": [{"model_name": "gpt-4o-mini"}],
}
@pytest.mark.asyncio
async def test_include_with_a_leading_slash_reads_from_the_bucket_root(self):
merged = await resolve_bucket_includes(
config={"include": ["/shared/models.yaml"]},
object_key="configs/prod/config.yaml",
fetch=self._bucket({"shared/models.yaml": {"model_list": [{"model_name": "shared"}]}}),
)
assert merged == {"model_list": [{"model_name": "shared"}]}
@pytest.mark.asyncio
async def test_include_walks_out_of_the_prefix_with_dot_dot(self):
merged = await resolve_bucket_includes(
config={"include": ["../shared/models.yaml"]},
object_key="configs/prod/config.yaml",
fetch=self._bucket({"configs/shared/models.yaml": {"model_list": [{"model_name": "shared"}]}}),
)
assert merged == {"model_list": [{"model_name": "shared"}]}
@pytest.mark.asyncio
async def test_included_configs_may_declare_further_includes(self):
merged = await resolve_bucket_includes(
config={"include": ["models.yaml"]},
object_key="configs/config.yaml",
fetch=self._bucket(
{
"configs/models.yaml": {
"include": ["extra/more_models.yaml"],
"model_list": [{"model_name": "first"}],
},
"configs/extra/more_models.yaml": {"model_list": [{"model_name": "second"}]},
}
),
)
assert merged == {"model_list": [{"model_name": "first"}, {"model_name": "second"}]}
@pytest.mark.asyncio
async def test_a_nested_include_resolves_against_the_object_that_declares_it(self):
merged = await resolve_bucket_includes(
config={"include": ["shared/models.yaml"]},
object_key="configs/config.yaml",
fetch=self._bucket(
{
"configs/shared/models.yaml": {
"include": ["more_models.yaml"],
"model_list": [{"model_name": "first"}],
},
"configs/shared/more_models.yaml": {"model_list": [{"model_name": "second"}]},
"configs/more_models.yaml": {"model_list": [{"model_name": "wrong-prefix"}]},
}
),
)
assert merged == {"model_list": [{"model_name": "first"}, {"model_name": "second"}]}
@pytest.mark.asyncio
async def test_an_object_pulled_in_twice_is_merged_once(self):
merged = await resolve_bucket_includes(
config={"include": ["a.yaml", "b.yaml"]},
object_key="configs/config.yaml",
fetch=self._bucket(
{
"configs/a.yaml": {"include": ["shared.yaml"]},
"configs/b.yaml": {"include": ["./shared.yaml"]},
"configs/shared.yaml": {"model_list": [{"model_name": "shared"}]},
}
),
)
assert merged == {"model_list": [{"model_name": "shared"}]}
@pytest.mark.asyncio
async def test_a_cycle_between_included_objects_terminates(self):
merged = await asyncio.wait_for(
resolve_bucket_includes(
config={"include": ["a.yaml"]},
object_key="configs/config.yaml",
fetch=self._bucket(
{
"configs/a.yaml": {"include": ["b.yaml"], "model_list": [{"model_name": "from-a"}]},
"configs/b.yaml": {"include": ["a.yaml"], "model_list": [{"model_name": "from-b"}]},
}
),
),
timeout=10,
)
assert merged == {"model_list": [{"model_name": "from-a"}, {"model_name": "from-b"}]}
@pytest.mark.asyncio
async def test_list_values_are_extended_and_other_values_are_overridden(self):
merged = await resolve_bucket_includes(
config={
"include": ["models.yaml"],
"model_list": [{"model_name": "from-root"}],
"litellm_settings": {"drop_params": True},
},
object_key="config.yaml",
fetch=self._bucket(
{
"models.yaml": {
"model_list": [{"model_name": "from-include"}],
"litellm_settings": {"num_retries": 3},
}
}
),
)
assert merged == {
"model_list": [{"model_name": "from-root"}, {"model_name": "from-include"}],
"litellm_settings": {"num_retries": 3},
}
@pytest.mark.asyncio
async def test_a_missing_included_object_fails_loudly_with_its_key(self):
with pytest.raises(FileNotFoundError, match=re.escape("configs/prod/model_config.yaml")):
await resolve_bucket_includes(
config={"include": ["model_config.yaml"]},
object_key="configs/prod/config.yaml",
fetch=self._bucket({}),
)
@pytest.mark.asyncio
async def test_a_non_list_include_fails_loudly(self):
with pytest.raises(ValueError, match="'include' must be a list of file paths"):
await resolve_bucket_includes(
config={"include": "model_config.yaml"},
object_key="config.yaml",
fetch=self._bucket({}),
)
@pytest.mark.asyncio
async def test_get_config_from_bucket_merges_includes_over_s3(self, monkeypatch):
objects = {
"lit6982/config.yaml": {
"include": ["model_config.yaml"],
"general_settings": {"master_key": MASTER_KEY},
},
"lit6982/model_config.yaml": {"model_list": [{"model_name": "included-model"}]},
}
monkeypatch.setattr(
"litellm.proxy.common_utils.load_config_utils.s3_object_reader",
lambda bucket_name: objects.get,
)
config = await get_config_from_bucket(
bucket_type="s3", bucket_name="litellm-configs", object_key="lit6982/config.yaml"
)
assert config == {
"general_settings": {"master_key": MASTER_KEY},
"model_list": [{"model_name": "included-model"}],
}
@pytest.mark.asyncio
async def test_the_blocking_s3_work_runs_off_the_event_loop_thread(self, monkeypatch):
loop_thread = threading.current_thread()
threads = []
def build_reader(bucket_name):
threads.append(threading.current_thread())
def read(object_key):
threads.append(threading.current_thread())
return {"model_list": [{"model_name": "a-model"}]}
return read
monkeypatch.setattr("litellm.proxy.common_utils.load_config_utils.s3_object_reader", build_reader)
await get_config_from_bucket(bucket_type="s3", bucket_name="litellm-configs", object_key="config.yaml")
assert len(threads) == 2 and loop_thread not in threads
@pytest.mark.asyncio
async def test_one_s3_client_serves_the_whole_include_tree(self, monkeypatch):
objects = {
"lit6982/config.yaml": {"include": ["model_config.yaml"]},
"lit6982/model_config.yaml": {"model_list": [{"model_name": "included-model"}]},
}
readers = []
def build_reader(bucket_name):
requested = []
readers.append(requested)
def read(object_key):
requested.append(object_key)
return objects.get(object_key)
return read
monkeypatch.setattr("litellm.proxy.common_utils.load_config_utils.s3_object_reader", build_reader)
await get_config_from_bucket(
bucket_type="s3", bucket_name="litellm-configs", object_key="lit6982/config.yaml"
)
assert readers == [["lit6982/config.yaml", "lit6982/model_config.yaml"]]
@pytest.mark.asyncio
async def test_an_empty_included_object_merges_as_an_empty_config(self, monkeypatch):
objects = {
"lit6982/config.yaml": "include:\n - empty.yaml\nmodel_list:\n - model_name: only-model\n",
"lit6982/empty.yaml": "",
}
class FakeGCSBucket:
async def download_gcs_object(self, object_key):
return objects[object_key].encode("utf-8")
monkeypatch.setattr(
"litellm.proxy.common_utils.load_config_utils.gcs_config_bucket",
lambda bucket_name: FakeGCSBucket(),
)
config = await get_config_from_bucket(
bucket_type="gcs", bucket_name="litellm-configs", object_key="lit6982/config.yaml"
)
assert config == {"model_list": [{"model_name": "only-model"}]}
@pytest.mark.asyncio
async def test_get_config_from_bucket_merges_includes_over_gcs(self, monkeypatch):
objects = {
"lit6982/config.yaml": {
"include": ["model_config.yaml"],
"general_settings": {"master_key": MASTER_KEY},
},
"lit6982/model_config.yaml": {"model_list": [{"model_name": "included-model"}]},
}
buckets = []
class FakeGCSBucket:
def __init__(self):
self.requested = []
buckets.append(self)
async def download_gcs_object(self, object_key):
self.requested.append(object_key)
return yaml.dump(objects[object_key]).encode("utf-8")
monkeypatch.setattr(
"litellm.proxy.common_utils.load_config_utils.gcs_config_bucket",
lambda bucket_name: FakeGCSBucket(),
)
config = await get_config_from_bucket(
bucket_type="gcs", bucket_name="litellm-configs", object_key="lit6982/config.yaml"
)
assert config == {
"general_settings": {"master_key": MASTER_KEY},
"model_list": [{"model_name": "included-model"}],
}
assert [bucket.requested for bucket in buckets] == [
["lit6982/config.yaml", "lit6982/model_config.yaml"]
]
@pytest.mark.asyncio
async def test_get_config_from_bucket_returns_none_when_the_root_object_is_missing(self, monkeypatch):
monkeypatch.setattr(
"litellm.proxy.common_utils.load_config_utils.s3_object_reader",
lambda bucket_name: (lambda object_key: None),
)
assert (
await get_config_from_bucket(
bucket_type="s3", bucket_name="litellm-configs", object_key="missing.yaml"
)
is None
)
@pytest.mark.asyncio
async def test_an_object_pulled_in_twice_is_read_once(self):
objects = {
"configs/a.yaml": {"include": ["shared.yaml"]},
"configs/b.yaml": {"include": ["./shared.yaml"]},
"configs/shared.yaml": {"model_list": [{"model_name": "shared"}]},
}
requested = []
async def fetch(object_key):
requested.append(object_key)
return objects.get(object_key)
await resolve_bucket_includes(
config={"include": ["a.yaml", "b.yaml"]},
object_key="configs/config.yaml",
fetch=fetch,
)
assert requested == ["configs/a.yaml", "configs/b.yaml", "configs/shared.yaml"]
@pytest.mark.asyncio
async def test_an_empty_root_object_does_not_boot_an_empty_proxy(self, monkeypatch):
class FakeGCSBucket:
async def download_gcs_object(self, object_key):
return b""
monkeypatch.setattr(
"litellm.proxy.common_utils.load_config_utils.gcs_config_bucket",
lambda bucket_name: FakeGCSBucket(),
)
config = await get_config_from_bucket(
bucket_type="gcs", bucket_name="litellm-configs", object_key="lit6982/config.yaml"
)
assert config is None
@pytest.mark.asyncio
async def test_an_object_that_is_not_valid_yaml_is_reported_as_a_yaml_error(self, monkeypatch, caplog):
class FakeGCSBucket:
async def download_gcs_object(self, object_key):
return b"model_list: [\n"
monkeypatch.setattr(
"litellm.proxy.common_utils.load_config_utils.gcs_config_bucket",
lambda bucket_name: FakeGCSBucket(),
)
with caplog.at_level(logging.ERROR, logger="LiteLLM Proxy"):
config = await get_config_from_bucket(
bucket_type="gcs", bucket_name="litellm-configs", object_key="lit6982/config.yaml"
)
assert config is None
assert [
record
for record in caplog.records
if "not valid YAML" in record.getMessage() and "lit6982/config.yaml" in record.getMessage()
]
class TestGCSConfigBucketClient:
@pytest.mark.asyncio
async def test_reading_a_config_from_gcs_does_not_need_an_enterprise_license(self, monkeypatch):
monkeypatch.setattr("litellm.proxy.proxy_server.premium_user", False)
bucket = gcs_config_bucket("litellm-configs")
assert bucket is not None
assert bucket.BUCKET_NAME == "litellm-configs"
@pytest.mark.asyncio
async def test_reading_a_config_from_gcs_starts_no_background_task(self, monkeypatch):
monkeypatch.setattr("litellm.proxy.proxy_server.premium_user", True)
running_before = asyncio.all_tasks()
gcs_config_bucket("litellm-configs")
assert asyncio.all_tasks() - running_before == set()