Platform for stateful agents: AI with advanced memory that can learn and self-improve over time.
feat: add azure byok (#3884)
cthomas9f7c533765b7
diff --git a/alembic/versions/ffb17eb241fc_add_api_version_to_byok_providers.py b/alembic/versions/ffb17eb241fc_add_api_version_to_byok_providers.pynew file mode 100644index 000000000..28c2a288a--- /dev/null+++ b/alembic/versions/ffb17eb241fc_add_api_version_to_byok_providers.py@@ -0,0 +1,31 @@+"""add api version to byok providers++Revision ID: ffb17eb241fc+Revises: 5fb8bba2c373+Create Date: 2025-08-12 14:35:26.375985++"""++from typing import Sequence, Union++import sqlalchemy as sa++from alembic import op++# revision identifiers, used by Alembic.+revision: str = "ffb17eb241fc"+down_revision: Union[str, None] = "5fb8bba2c373"+branch_labels: Union[str, Sequence[str], None] = None+depends_on: Union[str, Sequence[str], None] = None+++def upgrade() -> None:+ # ### commands auto generated by Alembic - please adjust! ###+ op.add_column("providers", sa.Column("api_version", sa.String(), nullable=True))+ # ### end Alembic commands ###+++def downgrade() -> None:+ # ### commands auto generated by Alembic - please adjust! ###+ op.drop_column("providers", "api_version")+ # ### end Alembic commands ###diff --git a/letta/llm_api/azure_client.py b/letta/llm_api/azure_client.pyindex 954688968..423be5315 100644--- a/letta/llm_api/azure_client.py+++ b/letta/llm_api/azure_client.py@@ -7,22 +7,41 @@ from openai.types.chat.chat_completion import ChatCompletionfrom letta.llm_api.openai_client import OpenAIClientfrom letta.otel.tracing import trace_methodfrom letta.schemas.embedding_config import EmbeddingConfig+from letta.schemas.enums import ProviderCategoryfrom letta.schemas.llm_config import LLMConfigfrom letta.settings import model_settingsclass AzureClient(OpenAIClient):+ def get_byok_overrides(self, llm_config: LLMConfig) -> Tuple[Optional[str], Optional[str], Optional[str]]:+ if llm_config.provider_category == ProviderCategory.byok:+ from letta.services.provider_manager import ProviderManager++ return ProviderManager().get_azure_credentials(llm_config.provider_name, actor=self.actor)++ return None, None, None++ async def get_byok_overrides_async(self, llm_config: LLMConfig) -> Tuple[Optional[str], Optional[str], Optional[str]]:+ if llm_config.provider_category == ProviderCategory.byok:+ from letta.services.provider_manager import ProviderManager++ return await ProviderManager().get_azure_credentials_async(llm_config.provider_name, actor=self.actor)++ return None, None, None+@trace_methoddef request(self, request_data: dict, llm_config: LLMConfig) -> dict:"""Performs underlying synchronous request to OpenAI API and returns raw response dict."""- api_key = model_settings.azure_api_key or os.environ.get("AZURE_API_KEY")- base_url = model_settings.azure_base_url or os.environ.get("AZURE_BASE_URL")- api_version = model_settings.azure_api_version or os.environ.get("AZURE_API_VERSION")- client = AzureOpenAI(api_key=api_key, azure_endpoint=base_url, api_version=api_version)+ api_key, base_url, api_version = self.get_byok_overrides(llm_config)+ if not api_key or not base_url or not api_version:+ api_key = model_settings.azure_api_key or os.environ.get("AZURE_API_KEY")+ base_url = model_settings.azure_base_url or os.environ.get("AZURE_BASE_URL")+ api_version = model_settings.azure_api_version or os.environ.get("AZURE_API_VERSION")+ client = AzureOpenAI(api_key=api_key, azure_endpoint=base_url, api_version=api_version)response: ChatCompletion = client.chat.completions.create(**request_data)return response.model_dump()@@ -31,11 +50,13 @@ class AzureClient(OpenAIClient):"""Performs underlying asynchronous request to OpenAI API and returns raw response dict."""- api_key = model_settings.azure_api_key or os.environ.get("AZURE_API_KEY")- base_url = model_settings.azure_base_url or os.environ.get("AZURE_BASE_URL")- api_version = model_settings.azure_api_version or os.environ.get("AZURE_API_VERSION")- client = AsyncAzureOpenAI(api_key=api_key, azure_endpoint=base_url, api_version=api_version)+ api_key, base_url, api_version = await self.get_byok_overrides_async(llm_config)+ if not api_key or not base_url or not api_version:+ api_key = model_settings.azure_api_key or os.environ.get("AZURE_API_KEY")+ base_url = model_settings.azure_base_url or os.environ.get("AZURE_BASE_URL")+ api_version = model_settings.azure_api_version or os.environ.get("AZURE_API_VERSION")+ client = AsyncAzureOpenAI(api_key=api_key, azure_endpoint=base_url, api_version=api_version)response: ChatCompletion = await client.chat.completions.create(**request_data)return response.model_dump()diff --git a/letta/orm/provider.py b/letta/orm/provider.pyindex 1237ea9d8..b46a95b85 100644--- a/letta/orm/provider.py+++ b/letta/orm/provider.py@@ -31,6 +31,7 @@ class Provider(SqlalchemyBase, OrganizationMixin):base_url: Mapped[str] = mapped_column(nullable=True, doc="Base URL for the provider.")access_key: Mapped[str] = mapped_column(nullable=True, doc="Access key used for requests to the provider.")region: Mapped[str] = mapped_column(nullable=True, doc="Region used for requests to the provider.")+ api_version: Mapped[str] = mapped_column(nullable=True, doc="API version used for requests to the provider.")# relationshipsorganization: Mapped["Organization"] = relationship("Organization", back_populates="providers")diff --git a/letta/schemas/providers/azure.py b/letta/schemas/providers/azure.pyindex e51c1775d..21b9e1d49 100644--- a/letta/schemas/providers/azure.py+++ b/letta/schemas/providers/azure.py@@ -78,3 +78,12 @@ class AzureProvider(Provider):# Hard coded as there are no API endpoints for thisllm_default = LLM_MAX_TOKENS.get(model_name, 4096)return AZURE_MODEL_TO_CONTEXT_LENGTH.get(model_name, llm_default)++ async def check_api_key(self):+ if not self.api_key:+ raise ValueError("No API key provided")++ try:+ await self.list_llm_models_async()+ except Exception as e:+ raise LLMAuthenticationError(message=f"Failed to authenticate with Azure: {e}", code=ErrorCode.UNAUTHENTICATED)diff --git a/letta/schemas/providers/base.py b/letta/schemas/providers/base.pyindex eef2cb396..2ecbe2b3a 100644--- a/letta/schemas/providers/base.py+++ b/letta/schemas/providers/base.py@@ -24,6 +24,7 @@ class Provider(ProviderBase):base_url: str | None = Field(None, description="Base URL for the provider.")access_key: str | None = Field(None, description="Access key used for requests to the provider.")region: str | None = Field(None, description="Region used for requests to the provider.")+ api_version: str | None = Field(None, description="API version used for requests to the provider.")organization_id: str | None = Field(None, description="The organization id of the user")updated_at: datetime | None = Field(None, description="The last update timestamp of the provider.")@@ -186,12 +187,16 @@ class ProviderCreate(ProviderBase):api_key: str = Field(..., description="API key or secret key used for requests to the provider.")access_key: str | None = Field(None, description="Access key used for requests to the provider.")region: str | None = Field(None, description="Region used for requests to the provider.")+ base_url: str | None = Field(None, description="Base URL used for requests to the provider.")+ api_version: str | None = Field(None, description="API version used for requests to the provider.")class ProviderUpdate(ProviderBase):api_key: str = Field(..., description="API key or secret key used for requests to the provider.")access_key: str | None = Field(None, description="Access key used for requests to the provider.")region: str | None = Field(None, description="Region used for requests to the provider.")+ base_url: str | None = Field(None, description="Base URL used for requests to the provider.")+ api_version: str | None = Field(None, description="API version used for requests to the provider.")class ProviderCheck(BaseModel):@@ -199,3 +204,5 @@ class ProviderCheck(BaseModel):api_key: str = Field(..., description="API key or secret key used for requests to the provider.")access_key: str | None = Field(None, description="Access key used for requests to the provider.")region: str | None = Field(None, description="Region used for requests to the provider.")+ base_url: str | None = Field(None, description="Base URL used for requests to the provider.")+ api_version: str | None = Field(None, description="API version used for requests to the provider.")diff --git a/letta/services/provider_manager.py b/letta/services/provider_manager.pyindex 46fe615e3..cfb32a82c 100644--- a/letta/services/provider_manager.py+++ b/letta/services/provider_manager.py@@ -205,6 +205,28 @@ class ProviderManager:region = providers[0].region if providers else Nonereturn access_key, secret_key, region+ @enforce_types+ @trace_method+ def get_azure_credentials(+ self, provider_name: Union[str, None], actor: PydanticUser+ ) -> Tuple[Optional[str], Optional[str], Optional[str]]:+ providers = self.list_providers(name=provider_name, actor=actor)+ api_key = providers[0].api_key if providers else None+ base_url = providers[0].base_url if providers else None+ api_version = providers[0].api_version if providers else None+ return api_key, base_url, api_version++ @enforce_types+ @trace_method+ async def get_azure_credentials_async(+ self, provider_name: Union[str, None], actor: PydanticUser+ ) -> Tuple[Optional[str], Optional[str], Optional[str]]:+ providers = await self.list_providers_async(name=provider_name, actor=actor)+ api_key = providers[0].api_key if providers else None+ base_url = providers[0].base_url if providers else None+ api_version = providers[0].api_version if providers else None+ return api_key, base_url, api_version+@enforce_types@trace_methodasync def check_provider_api_key(self, provider_check: ProviderCheck) -> None:@@ -215,6 +237,8 @@ class ProviderManager:provider_category=ProviderCategory.byok,access_key=provider_check.access_key, # This contains the access key ID for Bedrockregion=provider_check.region,+ base_url=provider_check.base_url,+ api_version=provider_check.api_version,).cast_to_subtype()# TODO: add more string sanity checks here before we hit actual endpoints