The fastest, litest AI Gateway. Rust core with Python SDK. Call 100+ LLM APIs in OpenAI (or native) format with cost tracking, guardrails, load balancing, and logging [Bedrock, Azure, OpenAI, Anthropic, OpenAI, VertexAI, vLLM, Nvidia NIM]
Python ai-gatewayanthropicazure-openaibedrockgatewaylangchainlitellmllmllm-gatewayllmopsmcp-gatewayopenaiopenai-proxyrustrust-aivertex-ai
import datetime
import json
from typing import Final
import httpx
import pytest
import respx
import litellm
from litellm.litellm_core_utils.litellm_logging import Logging
from litellm.llms.azure_ai.agents.handler import AzureAIAgentsHandler
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
from litellm.types.utils import ModelResponse
API_BASE: Final = "https://agents-test.services.ai.azure.com/api/projects/litellm-test"
API_VERSION: Final = "2025-05-01"
AGENT_ID: Final = "asst_hbnoK9BOCcHhC3lC4MDroVGG"
@pytest.fixture(autouse=True)
def _httpx_only_transport(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True")
def _assistants_api(thread_id: str, answer: str, project_path: str = "/api/projects/litellm-test") -> respx.MockRouter:
thread_path: Final = f"{project_path}/threads/{thread_id}"
router: Final = respx.mock(assert_all_called=False)
router.post(path=f"{project_path}/threads").respond(
200, json={"id": thread_id, "object": "thread", "created_at": 1767225600, "metadata": {}}
)
router.post(path=f"{thread_path}/messages").respond(
200, json={"id": "msg_user", "object": "thread.message", "thread_id": thread_id, "role": "user", "content": []}
)
router.post(path=f"{thread_path}/runs").respond(
200,
json={
"id": "run_1",
"object": "thread.run",
"thread_id": thread_id,
"assistant_id": AGENT_ID,
"status": "queued",
},
)
router.get(path=f"{thread_path}/runs/run_1").respond(
200, json={"id": "run_1", "object": "thread.run", "thread_id": thread_id, "status": "completed"}
)
router.get(path=f"{thread_path}/messages").respond(
200,
json={
"object": "list",
"data": [
{
"id": "msg_assistant",
"object": "thread.message",
"thread_id": thread_id,
"role": "assistant",
"content": [{"type": "text", "text": {"value": answer, "annotations": []}}],
},
{
"id": "msg_user",
"object": "thread.message",
"thread_id": thread_id,
"role": "user",
"content": [{"type": "text", "text": {"value": "earlier", "annotations": []}}],
},
],
"first_id": "msg_assistant",
"last_id": "msg_user",
"has_more": False,
},
)
return router
def _recorded(router: respx.MockRouter) -> list[tuple[str, str, str, object]]:
return [
(
call.request.method,
f"{call.request.url.path}?{call.request.url.query.decode()}",
call.request.headers["authorization"],
json.loads(call.request.content) if call.request.content else None,
)
for call in router.calls
]
@pytest.mark.asyncio
async def test_acompletion_creates_thread_posts_message_runs_and_returns_the_reply() -> None:
router: Final = _assistants_api("thread_new", "25 * 4 = 100")
with router:
response: Final = await litellm.acompletion(
model=f"azure_ai/agents/{AGENT_ID}",
messages=[{"role": "user", "content": "Hi Agent, what is 25 * 4?"}],
api_base=API_BASE,
api_key="azure-ad-token",
stream=False,
)
recorded: Final = _recorded(router)
query: Final = f"api-version={API_VERSION}"
auth: Final = "Bearer azure-ad-token"
assert recorded == [
("POST", f"/api/projects/litellm-test/threads?{query}", auth, None),
(
"POST",
f"/api/projects/litellm-test/threads/thread_new/messages?{query}",
auth,
{"role": "user", "content": "Hi Agent, what is 25 * 4?"},
),
("POST", f"/api/projects/litellm-test/threads/thread_new/runs?{query}", auth, {"assistant_id": AGENT_ID}),
("GET", f"/api/projects/litellm-test/threads/thread_new/runs/run_1?{query}", auth, None),
("GET", f"/api/projects/litellm-test/threads/thread_new/messages?{query}", auth, None),
]
assert response.choices[0].message.content == "25 * 4 = 100"
assert response._hidden_params["thread_id"] == "thread_new"
@pytest.mark.asyncio
async def test_acompletion_creates_the_thread_under_a_project_named_threads() -> None:
router: Final = _assistants_api("thread_new", "hello", project_path="/api/projects/threads")
with router:
await litellm.acompletion(
model=f"azure_ai/agents/{AGENT_ID}",
messages=[{"role": "user", "content": "hi"}],
api_base="https://agents-test.services.ai.azure.com/api/projects/threads",
api_key="azure-ad-token",
stream=False,
)
recorded: Final = _recorded(router)
assert recorded[0][:2] == ("POST", f"/api/projects/threads/threads?api-version={API_VERSION}")
@pytest.mark.asyncio
async def test_handler_acompletion_with_thread_id_reuses_that_thread() -> None:
router: Final = _assistants_api("thread_existing", "Your name is Alice.")
messages: Final[list[dict[str, object]]] = [{"role": "user", "content": "What is my name?"}]
logging_obj: Final = Logging(
model=f"azure_ai/agents/{AGENT_ID}",
messages=messages,
stream=False,
call_type="acompletion",
start_time=datetime.datetime(2026, 1, 1),
litellm_call_id="azure-agents-thread-reuse",
function_id="azure-agents-thread-reuse",
)
with router:
response: Final = await AzureAIAgentsHandler().acompletion(
model=f"agents/{AGENT_ID}",
messages=messages,
api_base=API_BASE,
api_key="azure-ad-token",
model_response=ModelResponse(),
logging_obj=logging_obj,
optional_params={"thread_id": "thread_existing"},
litellm_params={},
timeout=30.0,
client=AsyncHTTPHandler(),
)
recorded: Final = _recorded(router)
assert [(method, url.split("?")[0], body) for method, url, _, body in recorded] == [
("POST", "/api/projects/litellm-test/threads/thread_existing/messages", messages[0]),
("POST", "/api/projects/litellm-test/threads/thread_existing/runs", {"assistant_id": AGENT_ID}),
("GET", "/api/projects/litellm-test/threads/thread_existing/runs/run_1", None),
("GET", "/api/projects/litellm-test/threads/thread_existing/messages", None),
]
assert response.choices[0].message.content == "Your name is Alice."
assert response._hidden_params["thread_id"] == "thread_existing"
def _sse(event: str, data: dict[str, object] | str) -> str:
return f"event: {event}\ndata: {data if isinstance(data, str) else json.dumps(data)}\n\n"
def _message_delta(text: str) -> dict[str, object]:
return {
"id": "msg_stream",
"object": "thread.message.delta",
"delta": {"content": [{"index": 0, "type": "text", "text": {"value": text}}]},
}
@pytest.mark.asyncio
async def test_acompletion_stream_yields_agent_message_deltas_then_stop(respx_mock: respx.MockRouter) -> None:
thread: Final = {"id": "thread_stream", "object": "thread", "created_at": 1767225600, "metadata": {}}
run: Final = {"id": "run_stream", "object": "thread.run", "thread_id": "thread_stream", "assistant_id": AGENT_ID}
message: Final = {"id": "msg_stream", "object": "thread.message", "thread_id": "thread_stream", "role": "assistant"}
completed_content: Final = [{"type": "text", "text": {"value": "10 + 5 = 15", "annotations": []}}]
stream_body: Final = "".join(
(
_sse("thread.created", thread),
_sse("thread.run.created", {**run, "status": "queued"}),
_sse("thread.message.created", {**message, "status": "in_progress", "content": []}),
_sse("thread.message.delta", _message_delta("10 + 5")),
_sse("thread.message.delta", _message_delta(" = 15")),
_sse("thread.message.completed", {**message, "status": "completed", "content": completed_content}),
_sse("thread.run.completed", {**run, "status": "completed"}),
_sse("done", "[DONE]"),
)
)
route: Final = respx_mock.post(f"{API_BASE}/threads/runs").respond(
200, text=stream_body, headers={"content-type": "text/event-stream"}
)
response: Final = await litellm.acompletion(
model=f"azure_ai/agents/{AGENT_ID}",
messages=[{"role": "user", "content": "Hi Agent, what is 10 + 5?"}],
api_base=API_BASE,
api_key="azure-ad-token",
stream=True,
)
chunks: Final = [(chunk.choices[0].delta.content, chunk.choices[0].finish_reason) async for chunk in response]
assert route.calls.last.request.url.params["api-version"] == API_VERSION
assert json.loads(route.calls.last.request.content) == {
"assistant_id": AGENT_ID,
"stream": True,
"thread": {"messages": [{"role": "user", "content": "Hi Agent, what is 10 + 5?"}]},
}
assert chunks == [("10 + 5", None), (" = 15", None), (None, "stop")]