Platform for stateful agents: AI with advanced memory that can learn and self-improve over time.
feat: add new azure api maintaining backward compat (#9387)
Ari Webb5fd5a6dd07a3
diff --git a/letta/llm_api/azure_client.py b/letta/llm_api/azure_client.pyindex 3700c2a0f..80926aec7 100644--- a/letta/llm_api/azure_client.py+++ b/letta/llm_api/azure_client.py@@ -1,19 +1,31 @@+import jsonimport osfrom typing import List, Optional, Tuple-from openai import AsyncAzureOpenAI, AzureOpenAI+from openai import AsyncAzureOpenAI, AsyncOpenAI, AsyncStream, AzureOpenAI, OpenAIfrom openai.types.chat.chat_completion import ChatCompletion+from openai.types.chat.chat_completion_chunk import ChatCompletionChunk+from openai.types.responses.response_stream_event import ResponseStreamEventfrom letta.helpers.json_helpers import sanitize_unicode_surrogatesfrom letta.llm_api.openai_client import OpenAIClient+from letta.log import get_loggerfrom letta.otel.tracing import trace_methodfrom letta.schemas.embedding_config import EmbeddingConfigfrom letta.schemas.enums import ProviderCategoryfrom letta.schemas.llm_config import LLMConfigfrom letta.settings import model_settings+logger = get_logger(__name__)+class AzureClient(OpenAIClient):+ @staticmethod+ def _is_v1_endpoint(base_url: str) -> bool:+ if not base_url:+ return False+ return base_url.rstrip("/").endswith("/openai/v1")+def get_byok_overrides(self, llm_config: LLMConfig) -> Tuple[Optional[str], Optional[str], Optional[str]]:if llm_config.provider_category == ProviderCategory.byok:from letta.services.provider_manager import ProviderManager@@ -30,20 +42,36 @@ class AzureClient(OpenAIClient):return None, None, None+ def _resolve_credentials(self, api_key, base_url, api_version):+ """Resolve credentials, falling back to env vars. For v1 endpoints, api_version is not required."""+ if not api_key:+ api_key = model_settings.azure_api_key or os.environ.get("AZURE_API_KEY")+ if not base_url:+ base_url = model_settings.azure_base_url or os.environ.get("AZURE_BASE_URL")+ if not api_version and not self._is_v1_endpoint(base_url):+ api_version = model_settings.azure_api_version or os.environ.get("AZURE_API_VERSION")+ return api_key, base_url, api_version+@trace_methoddef request(self, request_data: dict, llm_config: LLMConfig) -> dict:"""Performs underlying synchronous request to OpenAI API and returns raw response dict."""api_key, base_url, api_version = self.get_byok_overrides(llm_config)- if not api_key or not base_url or not api_version:- api_key = model_settings.azure_api_key or os.environ.get("AZURE_API_KEY")- base_url = model_settings.azure_base_url or os.environ.get("AZURE_BASE_URL")- api_version = model_settings.azure_api_version or os.environ.get("AZURE_API_VERSION")+ api_key, base_url, api_version = self._resolve_credentials(api_key, base_url, api_version)++ if self._is_v1_endpoint(base_url):+ client = OpenAI(api_key=api_key, base_url=base_url)+ else:+ client = AzureOpenAI(api_key=api_key, azure_endpoint=base_url, api_version=api_version)- client = AzureOpenAI(api_key=api_key, azure_endpoint=base_url, api_version=api_version)- response: ChatCompletion = client.chat.completions.create(**request_data)- return response.model_dump()+ # Route based on payload shape: Responses uses 'input', Chat Completions uses 'messages'+ if "input" in request_data and "messages" not in request_data:+ resp = client.responses.create(**request_data)+ return resp.model_dump()+ else:+ response: ChatCompletion = client.chat.completions.create(**request_data)+ return response.model_dump()@trace_methodasync def request_async(self, request_data: dict, llm_config: LLMConfig) -> dict:@@ -53,17 +81,60 @@ class AzureClient(OpenAIClient):request_data = sanitize_unicode_surrogates(request_data)api_key, base_url, api_version = await self.get_byok_overrides_async(llm_config)- if not api_key or not base_url or not api_version:- api_key = model_settings.azure_api_key or os.environ.get("AZURE_API_KEY")- base_url = model_settings.azure_base_url or os.environ.get("AZURE_BASE_URL")- api_version = model_settings.azure_api_version or os.environ.get("AZURE_API_VERSION")+ api_key, base_url, api_version = self._resolve_credentials(api_key, base_url, api_version)+try:- client = AsyncAzureOpenAI(api_key=api_key, azure_endpoint=base_url, api_version=api_version)- response: ChatCompletion = await client.chat.completions.create(**request_data)+ if self._is_v1_endpoint(base_url):+ client = AsyncOpenAI(api_key=api_key, base_url=base_url)+ else:+ client = AsyncAzureOpenAI(api_key=api_key, azure_endpoint=base_url, api_version=api_version)++ # Route based on payload shape: Responses uses 'input', Chat Completions uses 'messages'+ if "input" in request_data and "messages" not in request_data:+ resp = await client.responses.create(**request_data)+ return resp.model_dump()+ else:+ response: ChatCompletion = await client.chat.completions.create(**request_data)+ return response.model_dump()except Exception as e:raise self.handle_llm_error(e)- return response.model_dump()+ @trace_method+ async def stream_async(self, request_data: dict, llm_config: LLMConfig) -> AsyncStream[ChatCompletionChunk | ResponseStreamEvent]:+ """+ Performs underlying asynchronous streaming request to Azure/OpenAI and returns the async stream iterator.+ """+ request_data = sanitize_unicode_surrogates(request_data)++ api_key, base_url, api_version = await self.get_byok_overrides_async(llm_config)+ api_key, base_url, api_version = self._resolve_credentials(api_key, base_url, api_version)++ if self._is_v1_endpoint(base_url):+ client = AsyncOpenAI(api_key=api_key, base_url=base_url)+ else:+ client = AsyncAzureOpenAI(api_key=api_key, azure_endpoint=base_url, api_version=api_version)++ # Route based on payload shape: Responses uses 'input', Chat Completions uses 'messages'+ if "input" in request_data and "messages" not in request_data:+ try:+ response_stream: AsyncStream[ResponseStreamEvent] = await client.responses.create(+ **request_data,+ stream=True,+ )+ except Exception as e:+ logger.error(f"Error streaming Azure Responses request: {e} with request data: {json.dumps(request_data)}")+ raise e+ else:+ try:+ response_stream: AsyncStream[ChatCompletionChunk] = await client.chat.completions.create(+ **request_data,+ stream=True,+ stream_options={"include_usage": True},+ )+ except Exception as e:+ logger.error(f"Error streaming Azure Chat Completions request: {e} with request data: {json.dumps(request_data)}")+ raise e+ return response_stream@trace_methodasync def request_embeddings(self, inputs: List[str], embedding_config: EmbeddingConfig) -> List[List[float]]:@@ -71,7 +142,12 @@ class AzureClient(OpenAIClient):api_key = model_settings.azure_api_key or os.environ.get("AZURE_API_KEY")base_url = model_settings.azure_base_url or os.environ.get("AZURE_BASE_URL")api_version = model_settings.azure_api_version or os.environ.get("AZURE_API_VERSION")- client = AsyncAzureOpenAI(api_key=api_key, api_version=api_version, azure_endpoint=base_url)++ if self._is_v1_endpoint(base_url):+ client = AsyncOpenAI(api_key=api_key, base_url=base_url)+ else:+ client = AsyncAzureOpenAI(api_key=api_key, api_version=api_version, azure_endpoint=base_url)+response = await client.embeddings.create(model=embedding_config.embedding_model, input=inputs)# TODO: add total usagediff --git a/letta/schemas/providers/azure.py b/letta/schemas/providers/azure.pyindex cc552dea2..11fa24524 100644--- a/letta/schemas/providers/azure.py+++ b/letta/schemas/providers/azure.py@@ -46,6 +46,12 @@ class AzureProvider(Provider):def replace_none_with_default(cls, v):return v if v is not None else cls.LATEST_API_VERSION+ @staticmethod+ def _is_v1_endpoint(base_url: str) -> bool:+ if not base_url:+ return False+ return base_url.rstrip("/").endswith("/openai/v1")+def get_azure_chat_completions_endpoint(self, model: str):return f"{self.base_url}/openai/deployments/{model}/chat/completions?api-version={self.api_version}"@@ -60,10 +66,50 @@ class AzureProvider(Provider):# That's the only api version that works with this deployments endpointreturn f"{self.base_url}/openai/deployments?api-version=2023-03-15-preview"+ def _get_resource_base_url(self) -> str:+ """Derive the Azure resource base URL (e.g. https://project.openai.azure.com) from any endpoint format."""+ url = self.base_url.rstrip("/")+ if url.endswith("/openai/v1"):+ return url[: -len("/openai/v1")]+ return url++ async def _get_deployments(self, api_key: str | None) -> list[dict]:+ """Fetch deployments using the legacy 2023-03-15-preview endpoint.++ Works for both v1 and legacy endpoints since it hits the resource base URL.+ Returns the raw deployment dicts (each has 'id' = deployment name).+ """+ resource_base = self._get_resource_base_url()+ url = f"{resource_base}/openai/deployments?api-version=2023-03-15-preview"++ headers = {"Content-Type": "application/json"}+ if api_key is not None:+ headers["api-key"] = f"{api_key}"++ try:+ timeout = httpx.Timeout(15.0, connect=10.0)+ async with httpx.AsyncClient(timeout=timeout) as http_client:+ response = await http_client.get(url, headers=headers)+ response.raise_for_status()+ except httpx.TimeoutException as e:+ raise RuntimeError(f"Azure API timeout after 15s: {e}")+ except httpx.HTTPStatusError as e:+ raise RuntimeError(f"Failed to retrieve deployment list: {e}")++ return response.json().get("data", [])+async def azure_openai_get_deployed_model_list(self) -> list:"""https://learn.microsoft.com/en-us/rest/api/azureopenai/models/list?view=rest-azureopenai-2023-05-15&tabs=HTTP"""api_key = await self.api_key_enc.get_plaintext_async() if self.api_key_enc else None++ if self._is_v1_endpoint(self.base_url):+ # The v1 /models endpoint returns base model names (e.g. "gpt-5.2-chat-2025-12-11")+ # but inference calls require deployment names (e.g. "gpt-5.2-chat").+ # Query the legacy deployments endpoint to get actual deployment names.+ return await self._get_deployments(api_key)++ # Legacy path: use Azure SDK + deployments endpointclient = AsyncAzureOpenAI(api_key=api_key, api_version=self.api_version, azure_endpoint=self.base_url)try:@@ -122,6 +168,37 @@ class AzureProvider(Provider):async def list_llm_models_async(self) -> list[LLMConfig]:model_list = await self.azure_openai_get_deployed_model_list()++ if self._is_v1_endpoint(self.base_url):+ # v1 path: follow OpenAIProvider pattern with litellm context window lookup+ configs = []+ for model in model_list:+ model_name = model.get("id")+ if not model_name:+ continue++ # Use capabilities if present, otherwise accept all (Azure deployments are user-curated)+ capabilities = model.get("capabilities")+ if capabilities and capabilities.get("chat_completion") is not None:+ if not capabilities.get("chat_completion"):+ continue++ context_window_size = await self.get_model_context_window_async(model_name)+ configs.append(+ LLMConfig(+ model=model_name,+ model_endpoint_type="azure",+ model_endpoint=self.base_url,+ context_window=context_window_size,+ handle=self.get_handle(model_name),+ max_tokens=self.get_default_max_output_tokens(model_name),+ provider_name=self.name,+ provider_category=self.provider_category,+ )+ )+ return configs++ # Legacy path# Extract models that support text generationmodel_options = [m for m in model_list if m.get("capabilities").get("chat_completion") == True]@@ -145,6 +222,38 @@ class AzureProvider(Provider):return configsasync def list_embedding_models_async(self) -> list[EmbeddingConfig]:+ model_list = await self.azure_openai_get_deployed_model_list()++ if self._is_v1_endpoint(self.base_url):+ # v1 path: use base URL as endpoint, filter by capabilities or name+ configs = []+ for model in model_list:+ model_name = model.get("id")+ if not model_name:+ continue++ # Use capabilities if present, otherwise filter by name+ capabilities = model.get("capabilities")+ if capabilities and capabilities.get("embeddings") is not None:+ if not capabilities.get("embeddings"):+ continue+ elif "embedding" not in model_name:+ continue++ configs.append(+ EmbeddingConfig(+ embedding_model=model_name,+ embedding_endpoint_type="azure",+ embedding_endpoint=self.base_url,+ embedding_dim=768,+ embedding_chunk_size=DEFAULT_EMBEDDING_CHUNK_SIZE,+ handle=self.get_handle(model_name, is_embedding=True),+ batch_size=1024,+ )+ )+ return configs++ # Legacy pathdef valid_embedding_model(m: dict, require_embedding_in_name: bool = True):valid_name = Trueif require_embedding_in_name:@@ -152,9 +261,7 @@ class AzureProvider(Provider):return m.get("capabilities").get("embeddings") == True and valid_name- model_list = await self.azure_openai_get_deployed_model_list()# Extract models that support embeddings-model_options = [m for m in model_list if valid_embedding_model(m)]configs = []@@ -179,6 +286,23 @@ class AzureProvider(Provider):llm_default = LLM_MAX_CONTEXT_WINDOW.get(model_name, 4096)return AZURE_MODEL_TO_CONTEXT_LENGTH.get(model_name, llm_default)+ async def get_model_context_window_async(self, model_name: str) -> int | None:+ """Get context window size, using litellm specs for v1 endpoints or hardcoded map for legacy."""+ if self._is_v1_endpoint(self.base_url):+ from letta.model_specs.litellm_model_specs import get_context_window++ # Litellm keys Azure models with an "azure/" prefix+ context_window = await get_context_window(f"azure/{model_name}")+ if context_window is not None:+ return context_window+ # Try without prefix as fallback+ context_window = await get_context_window(model_name)+ if context_window is not None:+ return context_window+ # Fall back to hardcoded map, then default+ return self.get_model_context_window(model_name)+ return self.get_model_context_window(model_name)+async def check_api_key(self):api_key = await self.api_key_enc.get_plaintext_async() if self.api_key_enc else Noneif not api_key: