The fastest, litest AI Gateway. Rust core with Python SDK. Call 100+ LLM APIs in OpenAI (or native) format with cost tracking, guardrails, load balancing, and logging [Bedrock, Azure, OpenAI, Anthropic, OpenAI, VertexAI, vLLM, Nvidia NIM]
Python ai-gatewayanthropicazure-openaibedrockgatewaylangchainlitellmllmllm-gatewayllmopsmcp-gatewayopenaiopenai-proxyrustrust-aivertex-ai
import json
from typing import Final
from unittest.mock import AsyncMock, Mock, patch
import httpx
import pytest
from pydantic import ValidationError
import litellm
from litellm.llms.replicate.chat.handler import async_completion
from litellm.llms.replicate.chat.handler import completion as replicate_completion
from litellm.llms.replicate.chat.transformation import ReplicateConfig
from litellm.llms.replicate.common_utils import ReplicateError
from litellm.types.utils import ModelResponse
def _transform(raw_response: httpx.Response) -> ModelResponse:
return ReplicateConfig().transform_response(
model="acme/echo-model",
raw_response=raw_response,
model_response=ModelResponse(),
logging_obj=Mock(),
request_data={"input": {"prompt": "Hello"}},
messages=[{"role": "user", "content": "Hello"}],
optional_params={},
litellm_params={},
encoding=None,
)
@pytest.mark.parametrize(
("output", "content"),
[
(["Hello", ", ", "world"], "Hello, world"),
("Hello", "Hello"),
([], " "),
("", " "),
([""], " "),
],
)
def test_transform_response_joins_the_prediction_output_into_the_message_content(output: object, content: str):
response = _transform(httpx.Response(200, json={"status": "succeeded", "output": output}))
assert response.choices[0].message.content == content
assert response.model == "replicate/acme/echo-model"
assert response.usage.total_tokens == response.usage.prompt_tokens + response.usage.completion_tokens
def test_transform_response_uses_a_blank_message_when_the_prediction_has_no_output():
response = _transform(httpx.Response(200, json={"status": "succeeded"}))
assert response.choices[0].message.content == " "
@pytest.mark.parametrize("output", [None, 7, True, ["secret-output", 7], ["secret-output", None], [["secret-output"]]])
def test_transform_response_rejects_an_output_that_is_not_made_of_strings_without_echoing_it(output: object):
with pytest.raises(ValidationError) as exc_info:
_transform(httpx.Response(200, json={"status": "succeeded", "output": output}))
assert "secret-output" not in str(exc_info.value)
def test_transform_response_reads_a_succeeded_prediction_that_carries_unmodelled_fields() -> None:
response = _transform(
httpx.Response(
200,
json={
"id": "p1",
"status": "succeeded",
"output": ["Hello", " there"],
"metrics": {"predict_time": 1.5, "stages": [1, {"name": None}]},
"urls": {"get": "https://api.replicate.com/v1/predictions/p1"},
"error": None,
},
)
)
assert response.choices[0].message.content == "Hello there"
assert response.model == "replicate/acme/echo-model"
@pytest.mark.parametrize(
("body", "reported"),
[
pytest.param(
{"status": "failed", "error": "boom", "output": None},
"{'status': 'failed', 'error': 'boom', 'output': None}",
id="failed",
),
pytest.param(
{"status": "processing", "output": ["partial"]},
"{'status': 'processing', 'output': ['partial']}",
id="still-processing",
),
pytest.param(
{"detail": {"b": [1, 2.5, True], "a": "z"}},
"{'detail': {'b': [1, 2.5, True], 'a': 'z'}}",
id="no-status",
),
pytest.param({}, "{}", id="empty-object"),
],
)
def test_transform_response_reports_an_unsuccessful_prediction_body_as_received(
body: dict[str, object], reported: str
) -> None:
with pytest.raises(ReplicateError) as exc_info:
_transform(httpx.Response(500, json=body, headers={"retry-after": "3"}))
assert exc_info.value.status_code == 422
assert exc_info.value.message == f"LiteLLM Error - prediction not succeeded - {reported}"
assert exc_info.value.headers["retry-after"] == "3"
@pytest.mark.parametrize(
("body", "echo"),
[
pytest.param(["secret-output"], "secret-output", id="list"),
pytest.param("secret-output", "secret-output", id="string"),
pytest.param(4815162342, "4815162342", id="number"),
pytest.param(None, "NoneType", id="null"),
],
)
def test_transform_response_rejects_a_body_that_is_not_an_object_without_echoing_it(body: object, echo: str) -> None:
with pytest.raises(ValidationError) as exc_info:
_transform(httpx.Response(500, content=json.dumps(body).encode()))
assert echo not in str(exc_info.value)
@pytest.mark.parametrize(
("supplied", "stored"),
[
pytest.param({}, {}, id="nothing-supplied"),
pytest.param(
{"system_prompt": "", "max_new_tokens": 0, "temperature": 1, "debug": False},
{"system_prompt": "", "max_new_tokens": 0, "temperature": 1, "debug": False},
id="falsy-values-are-kept",
),
pytest.param({"top_k": 40, "seed": None, "stop_sequences": None}, {"top_k": 40}, id="none-is-skipped"),
],
)
def test_init_stores_each_supplied_value_except_none_as_class_level_config(
supplied: dict[str, object], stored: dict[str, object]
) -> None:
class ScopedReplicateConfig(ReplicateConfig):
pass
ScopedReplicateConfig(**supplied)
assert ScopedReplicateConfig.get_config() == stored
def test_init_called_again_with_none_keeps_the_value_stored_earlier() -> None:
class ScopedReplicateConfig(ReplicateConfig):
pass
ScopedReplicateConfig(top_k=40)
ScopedReplicateConfig(top_k=None, seed=5)
assert ScopedReplicateConfig.get_config() == {"top_k": 40, "seed": 5}
@pytest.mark.asyncio
@patch("litellm.llms.replicate.chat.handler.asyncio.sleep", new_callable=AsyncMock)
@patch("litellm.llms.replicate.chat.handler.get_async_httpx_client")
async def test_async_completion_handles_starting_status(mock_get_client, mock_sleep):
"""Test that async completion polls correctly when status is 'starting'"""
mock_client = AsyncMock()
mock_get_client.return_value = mock_client
post_response = Mock()
post_response.json.return_value = {
"id": "test-prediction-id",
"urls": {
"get": "https://api.replicate.com/v1/predictions/test-id",
"cancel": "https://api.replicate.com/v1/predictions/test-id/cancel",
},
}
mock_client.post = AsyncMock(return_value=post_response)
get_response_starting = Mock()
get_response_starting.status_code = 200
get_response_starting.json.return_value = {"id": "test-prediction-id", "status": "starting", "output": None}
get_response_processing = Mock()
get_response_processing.status_code = 200
get_response_processing.json.return_value = {"id": "test-prediction-id", "status": "processing", "output": None}
get_response_succeeded = Mock()
get_response_succeeded.status_code = 200
get_response_succeeded.json.return_value = {
"id": "test-prediction-id",
"status": "succeeded",
"output": ["Hello", " from", " DeepSeek!"],
}
get_response_succeeded.text = json.dumps(get_response_succeeded.json.return_value)
get_response_succeeded.headers = {}
mock_client.get = AsyncMock(side_effect=[get_response_starting, get_response_processing, get_response_succeeded])
model_response = litellm.ModelResponse()
model_response.choices = [litellm.Choices()]
model_response.choices[0].message = litellm.Message(content="")
mock_logging = Mock()
mock_logging.post_call = Mock()
result = await async_completion(
model_response=model_response,
model="deepseek-ai/deepseek-v3",
messages=[{"role": "user", "content": "Hi"}],
encoding=None,
optional_params={},
litellm_params={},
version_id="deepseek-ai/deepseek-v3",
input_data={"input": {"prompt": "test"}},
api_key="test-key",
api_base="https://api.replicate.com",
logging_obj=mock_logging,
print_verbose=print,
headers={"Authorization": "Token test-key"},
)
assert result is not None
assert result.choices[0].message.content == "Hello from DeepSeek!"
assert mock_client.get.call_count == 3
class TestReplicateOutputFormats:
@pytest.mark.usefixtures("fake_provider_credentials")
def test_transform_response_list_output(self):
"""Test standard list output format"""
from litellm.llms.replicate.chat.transformation import ReplicateConfig
config = ReplicateConfig()
# Mock response with list output
mock_response = Mock()
mock_response.status_code = 200
mock_response.json.return_value = {
"status": "succeeded",
"output": ["Hello", " ", "world"],
}
mock_response.text = json.dumps(mock_response.json.return_value)
mock_response.headers = {}
model_response = litellm.ModelResponse()
model_response.choices = [litellm.Choices()]
model_response.choices[0].message = litellm.Message(content="")
mock_logging = Mock()
mock_logging.post_call = Mock()
result = config.transform_response(
model="meta/llama-2-70b-chat",
raw_response=mock_response,
model_response=model_response,
logging_obj=mock_logging,
request_data={"input": {"prompt": "test"}},
messages=[{"role": "user", "content": "Hi"}],
optional_params={},
litellm_params={},
encoding=None,
api_key="test-key",
)
assert result.choices[0].message.content == "Hello world"
def test_transform_response_string_output():
"""Test string output format (as used by some DeepSeek models)"""
from litellm.llms.replicate.chat.transformation import ReplicateConfig
config = ReplicateConfig()
mock_response = Mock()
mock_response.status_code = 200
mock_response.json.return_value = {"status": "succeeded", "output": "Hello from DeepSeek"}
mock_response.text = json.dumps(mock_response.json.return_value)
mock_response.headers = {}
model_response = litellm.ModelResponse()
model_response.choices = [litellm.Choices()]
model_response.choices[0].message = litellm.Message(content="")
mock_logging = Mock()
mock_logging.post_call = Mock()
result = config.transform_response(
model="deepseek-ai/deepseek-v3",
raw_response=mock_response,
model_response=model_response,
logging_obj=mock_logging,
request_data={"input": {"prompt": "test"}},
messages=[{"role": "user", "content": "Hi"}],
optional_params={},
litellm_params={},
encoding=None,
api_key="test-key",
)
assert result.choices[0].message.content == "Hello from DeepSeek"
DEEPSEEK_V3_TOKENIZER_CONFIG = {
"add_bos_token": True,
"add_eos_token": False,
"bos_token": {"__type": "AddedToken", "content": "<|begin▁of▁sentence|>", "lstrip": False, "rstrip": False},
"eos_token": {"__type": "AddedToken", "content": "<|end▁of▁sentence|>", "lstrip": False, "rstrip": False},
"model_max_length": 131072,
"tokenizer_class": "LlamaTokenizerFast",
"chat_template": (
"{{ bos_token }}{% for message in messages %}"
"{% if message['role'] == 'system' %}{{ message['content'] }}"
"{% elif message['role'] == 'user' %}{{ '<|User|>' + message['content'] }}"
"{% elif message['role'] == 'assistant' %}{{ '<|Assistant|>' + message['content'] + eos_token }}"
"{% endif %}{% endfor %}{% if add_generation_prompt %}{{ '<|Assistant|>' }}{% endif %}"
),
}
@pytest.mark.respx(assert_all_called=False)
@patch("litellm.llms.replicate.chat.handler.get_httpx_client")
def test_sync_completion_handles_starting_status(mock_get_client, monkeypatch, respx_mock):
monkeypatch.setattr(litellm, "known_tokenizer_config", dict(litellm.known_tokenizer_config))
respx_mock.get("https://huggingface.co/deepseek-ai/deepseek-v3/raw/main/tokenizer_config.json").respond(
200, json=DEEPSEEK_V3_TOKENIZER_CONFIG
)
mock_client = Mock()
mock_get_client.return_value = mock_client
post_response = Mock()
post_response.json.return_value = {
"id": "test-prediction-id",
"urls": {
"get": "https://api.replicate.com/v1/predictions/test-id",
"cancel": "https://api.replicate.com/v1/predictions/test-id/cancel",
},
}
mock_client.post.return_value = post_response
get_response_starting = Mock()
get_response_starting.status_code = 200
get_response_starting.json.return_value = {
"id": "test-prediction-id",
"status": "starting",
"output": None,
}
get_response_succeeded = Mock()
get_response_succeeded.status_code = 200
get_response_succeeded.json.return_value = {
"id": "test-prediction-id",
"status": "succeeded",
"output": ["Hello", " DeepSeek!"],
}
get_response_succeeded.text = json.dumps(get_response_succeeded.json.return_value)
get_response_succeeded.headers = {}
mock_client.get.side_effect = [get_response_starting, get_response_succeeded]
model_response = litellm.ModelResponse()
model_response.choices = [litellm.Choices()]
model_response.choices[0].message = litellm.Message(content="")
mock_logging = Mock()
mock_logging.post_call = Mock()
with patch("time.sleep"):
result = replicate_completion(
model="deepseek-ai/deepseek-v3",
messages=[{"role": "user", "content": "Hi"}],
api_base="https://api.replicate.com",
model_response=model_response,
print_verbose=print,
optional_params={},
litellm_params={},
logging_obj=mock_logging,
api_key="test-key",
encoding=None,
headers={},
)
assert result is not None
assert result.choices[0].message.content == "Hello DeepSeek!"
assert mock_client.get.call_count >= 1
assert json.loads(mock_client.post.call_args.kwargs["data"])["input"]["prompt"] == (
"<|begin▁of▁sentence|><|User|>Hi<|Assistant|>"
)
@pytest.mark.parametrize(
("model", "max_tokens_key"),
[
("replicate/vicuna-13b", "max_length"),
("replicate/flan-t5-xl", "max_length"),
("meta/codellama-13b", "max_tokens"),
("meta/llama-2-7b", "max_new_tokens"),
],
)
def test_map_openai_params_renames_openai_params_to_replicate_inputs(model: str, max_tokens_key: str):
mapped: Final = ReplicateConfig().map_openai_params(
non_default_params={"stream": True, "max_tokens": 64, "temperature": 0.2, "top_p": 0.9, "stop": ["\n"]},
optional_params={},
model=model,
drop_params=False,
)
assert mapped == {
"stream": True,
max_tokens_key: 64,
"temperature": 0.2,
"top_p": 0.9,
"stop_sequences": ["\n"],
}