Platform for stateful agents: AI with advanced memory that can learn and self-improve over time.
feat: add endpoint to test connection to llm provider (#2032)
jnjpng05e1623389fb
diff --git a/letta/llm_api/anthropic.py b/letta/llm_api/anthropic.pyindex aada2259b..88cf0e79f 100644--- a/letta/llm_api/anthropic.py+++ b/letta/llm_api/anthropic.py@@ -19,7 +19,7 @@ from anthropic.types.beta import (BetaToolUseBlock,)-from letta.errors import BedrockError, BedrockPermissionError+from letta.errors import BedrockError, BedrockPermissionError, ErrorCode, LLMAuthenticationError, LLMErrorfrom letta.helpers.datetime_helpers import get_utc_time_int, timestamp_to_datetimefrom letta.llm_api.aws_bedrock import get_bedrock_clientfrom letta.llm_api.helpers import add_inner_thoughts_to_functions@@ -119,6 +119,20 @@ DUMMY_FIRST_USER_MESSAGE = "User initializing bootup sequence."VALID_EVENT_TYPES = {"content_block_stop", "message_stop"}+def anthropic_check_valid_api_key(api_key: Union[str, None]) -> None:+ if api_key:+ anthropic_client = anthropic.Anthropic(api_key=api_key)+ try:+ # just use a cheap model to count some tokens - as of 5/7/2025 this is faster than fetching the list of models+ anthropic_client.messages.count_tokens(model=MODEL_LIST[-1]["name"], messages=[{"role": "user", "content": "a"}])+ except anthropic.AuthenticationError as e:+ raise LLMAuthenticationError(message=f"Failed to authenticate with Anthropic: {e}", code=ErrorCode.UNAUTHENTICATED)+ except Exception as e:+ raise LLMError(message=f"{e}", code=ErrorCode.INTERNAL_SERVER_ERROR)+ else:+ raise ValueError("No API key provided")++def antropic_get_model_context_window(url: str, api_key: Union[str, None], model: str) -> int:for model_dict in anthropic_get_model_list(url=url, api_key=api_key):if model_dict["name"] == model:diff --git a/letta/llm_api/google_ai_client.py b/letta/llm_api/google_ai_client.pyindex ad650c5f1..f056a64bd 100644--- a/letta/llm_api/google_ai_client.py+++ b/letta/llm_api/google_ai_client.py@@ -3,11 +3,14 @@ import uuidfrom typing import List, Optional, Tupleimport requests+from google import genaifrom google.genai.types import FunctionCallingConfig, FunctionCallingConfigMode, ToolConfigfrom letta.constants import NON_USER_MSG_PREFIX+from letta.errors import ErrorCode, LLMAuthenticationError, LLMErrorfrom letta.helpers.datetime_helpers import get_utc_time_intfrom letta.helpers.json_helpers import json_dumps+from letta.llm_api.google_constants import GOOGLE_MODEL_FOR_API_KEY_CHECKfrom letta.llm_api.helpers import make_post_requestfrom letta.llm_api.llm_client_base import LLMClientBasefrom letta.local_llm.json_parser import clean_json_string_extra_backslash@@ -443,6 +446,23 @@ def get_gemini_endpoint_and_headers(return url, headers+def google_ai_check_valid_api_key(api_key: str):+ client = genai.Client(api_key=api_key)+ # use the count token endpoint for a cheap model - as of 5/7/2025 this is slightly faster than fetching the list of models+ try:+ client.models.count_tokens(+ model=GOOGLE_MODEL_FOR_API_KEY_CHECK,+ contents="",+ )+ except genai.errors.ClientError as e:+ # google api returns 400 invalid argument for invalid api key+ if e.code == 400:+ raise LLMAuthenticationError(message=f"Failed to authenticate with Google AI: {e}", code=ErrorCode.UNAUTHENTICATED)+ raise e+ except Exception as e:+ raise LLMError(message=f"{e}", code=ErrorCode.INTERNAL_SERVER_ERROR)++def google_ai_get_model_list(base_url: str, api_key: str, key_in_header: bool = True) -> List[dict]:from letta.utils import printddiff --git a/letta/llm_api/google_constants.py b/letta/llm_api/google_constants.pyindex c720a33ac..1c30d6151 100644--- a/letta/llm_api/google_constants.py+++ b/letta/llm_api/google_constants.py@@ -14,3 +14,5 @@ GOOGLE_MODEL_TO_CONTEXT_LENGTH = {GOOGLE_MODEL_TO_OUTPUT_LENGTH = {"gemini-2.0-flash-001": 8192, "gemini-2.5-pro-exp-03-25": 65536}GOOGLE_EMBEDING_MODEL_TO_DIM = {"text-embedding-005": 768, "text-multilingual-embedding-002": 768}++GOOGLE_MODEL_FOR_API_KEY_CHECK = "gemini-2.0-flash-lite"diff --git a/letta/llm_api/openai.py b/letta/llm_api/openai.pyindex e35429bc5..2fe8ade33 100644--- a/letta/llm_api/openai.py+++ b/letta/llm_api/openai.py@@ -5,6 +5,7 @@ import requestsfrom openai import OpenAIfrom letta.constants import LETTA_MODEL_ENDPOINT+from letta.errors import ErrorCode, LLMAuthenticationError, LLMErrorfrom letta.helpers.datetime_helpers import timestamp_to_datetimefrom letta.llm_api.helpers import add_inner_thoughts_to_functions, convert_to_structured_output, make_post_requestfrom letta.llm_api.openai_client import accepts_developer_role, supports_parallel_tool_calling, supports_temperature_param@@ -34,6 +35,21 @@ from letta.utils import get_tool_call_id, smart_urljoinlogger = get_logger(__name__)+def openai_check_valid_api_key(base_url: str, api_key: Union[str, None]) -> None:+ if api_key:+ try:+ # just get model list to check if the api key is valid until we find a cheaper / quicker endpoint+ openai_get_model_list(url=base_url, api_key=api_key)+ except requests.HTTPError as e:+ if e.response.status_code == 401:+ raise LLMAuthenticationError(message=f"Failed to authenticate with OpenAI: {e}", code=ErrorCode.UNAUTHENTICATED)+ raise e+ except Exception as e:+ raise LLMError(message=f"{e}", code=ErrorCode.INTERNAL_SERVER_ERROR)+ else:+ raise ValueError("No API key provided")++def openai_get_model_list(url: str, api_key: Optional[str] = None, fix_url: Optional[bool] = False, extra_params: Optional[dict] = None) -> dict:diff --git a/letta/schemas/enums.py b/letta/schemas/enums.pyindex 2a3de4092..555ffaddd 100644--- a/letta/schemas/enums.py+++ b/letta/schemas/enums.py@@ -3,6 +3,7 @@ from enum import Enumclass ProviderType(str, Enum):anthropic = "anthropic"+ anthropic_bedrock = "bedrock"google_ai = "google_ai"google_vertex = "google_vertex"openai = "openai"diff --git a/letta/schemas/providers.py b/letta/schemas/providers.pyindex 291271e39..f1e9edd63 100644--- a/letta/schemas/providers.py+++ b/letta/schemas/providers.py@@ -2,7 +2,7 @@ import warningsfrom datetime import datetimefrom typing import List, Literal, Optional-from pydantic import Field, model_validator+from pydantic import BaseModel, Field, model_validatorfrom letta.constants import LETTA_MODEL_ENDPOINT, LLM_MAX_TOKENS, MIN_CONTEXT_WINDOWfrom letta.llm_api.azure_openai import get_azure_chat_completions_endpoint, get_azure_embeddings_endpoint@@ -40,6 +40,10 @@ class Provider(ProviderBase):if not self.id:self.id = ProviderBase.generate_id(prefix=ProviderBase.__id_prefix__)+ def check_api_key(self):+ """Check if the API key is valid for the provider"""+ raise NotImplementedError+def list_llm_models(self) -> List[LLMConfig]:return []@@ -112,6 +116,11 @@ class ProviderUpdate(ProviderBase):api_key: str = Field(..., description="API key used for requests to the provider.")+class ProviderCheck(BaseModel):+ provider_type: ProviderType = Field(..., description="The type of the provider.")+ api_key: str = Field(..., description="API key used for requests to the provider.")++class LettaProvider(Provider):provider_type: Literal[ProviderType.letta] = Field(ProviderType.letta, description="The type of the provider.")provider_category: ProviderCategory = Field(ProviderCategory.base, description="The category of the provider (base or byok)")@@ -148,6 +157,11 @@ class OpenAIProvider(Provider):api_key: str = Field(..., description="API key for the OpenAI API.")base_url: str = Field(..., description="Base URL for the OpenAI API.")+ def check_api_key(self):+ from letta.llm_api.openai import openai_check_valid_api_key++ openai_check_valid_api_key(self.base_url, self.api_key)+def list_llm_models(self) -> List[LLMConfig]:from letta.llm_api.openai import openai_get_model_list@@ -549,6 +563,11 @@ class AnthropicProvider(Provider):api_key: str = Field(..., description="API key for the Anthropic API.")base_url: str = "https://api.anthropic.com/v1"+ def check_api_key(self):+ from letta.llm_api.anthropic import anthropic_check_valid_api_key++ anthropic_check_valid_api_key(self.api_key)+def list_llm_models(self) -> List[LLMConfig]:from letta.llm_api.anthropic import MODEL_LIST, anthropic_get_model_list@@ -951,6 +970,11 @@ class GoogleAIProvider(Provider):api_key: str = Field(..., description="API key for the Google AI API.")base_url: str = "https://generativelanguage.googleapis.com"+ def check_api_key(self):+ from letta.llm_api.google_ai_client import google_ai_check_valid_api_key++ google_ai_check_valid_api_key(self.api_key)+def list_llm_models(self):from letta.llm_api.google_ai_client import google_ai_get_model_listdiff --git a/letta/server/rest_api/routers/v1/providers.py b/letta/server/rest_api/routers/v1/providers.pyindex 0cf29114e..8111ffbaf 100644--- a/letta/server/rest_api/routers/v1/providers.py+++ b/letta/server/rest_api/routers/v1/providers.py@@ -3,9 +3,10 @@ from typing import TYPE_CHECKING, List, Optionalfrom fastapi import APIRouter, Body, Depends, Header, HTTPException, Query, statusfrom fastapi.responses import JSONResponse+from letta.errors import LLMAuthenticationErrorfrom letta.orm.errors import NoResultFoundfrom letta.schemas.enums import ProviderType-from letta.schemas.providers import Provider, ProviderCreate, ProviderUpdate+from letta.schemas.providers import Provider, ProviderCheck, ProviderCreate, ProviderUpdatefrom letta.server.rest_api.utils import get_letta_serverif TYPE_CHECKING:@@ -67,6 +68,22 @@ def modify_provider(return server.provider_manager.update_provider(provider_id=provider_id, request=request, actor=actor)[email protected]("/check", response_model=None, operation_id="check_provider")+def check_provider(+ provider_type: ProviderType = Query(...),+ api_key: str = Header(..., alias="x-api-key"),+ server: "SyncServer" = Depends(get_letta_server),+):+ try:+ provider_check = ProviderCheck(provider_type=provider_type, api_key=api_key)+ server.provider_manager.check_provider_api_key(provider_check=provider_check)+ return JSONResponse(status_code=status.HTTP_200_OK, content={"message": f"Valid api key for provider_type={provider_type.value}"})+ except LLMAuthenticationError as e:+ raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail=f"{e.message}")+ except Exception as e:+ raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=f"{e}")++@router.delete("/{provider_id}", response_model=None, operation_id="delete_provider")def delete_provider(provider_id: str,diff --git a/letta/services/provider_manager.py b/letta/services/provider_manager.pyindex 49ec99f46..e77a3f2fa 100644--- a/letta/services/provider_manager.py+++ b/letta/services/provider_manager.py@@ -3,7 +3,7 @@ from typing import List, Optional, Unionfrom letta.orm.provider import Provider as ProviderModelfrom letta.schemas.enums import ProviderCategory, ProviderTypefrom letta.schemas.providers import Provider as PydanticProvider-from letta.schemas.providers import ProviderCreate, ProviderUpdate+from letta.schemas.providers import ProviderCheck, ProviderCreate, ProviderUpdatefrom letta.schemas.user import User as PydanticUserfrom letta.utils import enforce_types@@ -99,3 +99,18 @@ class ProviderManager:def get_override_key(self, provider_name: Union[str, None], actor: PydanticUser) -> Optional[str]:providers = self.list_providers(name=provider_name, actor=actor)return providers[0].api_key if providers else None++ @enforce_types+ def check_provider_api_key(self, provider_check: ProviderCheck) -> None:+ provider = PydanticProvider(+ name=provider_check.provider_type.value,+ provider_type=provider_check.provider_type,+ api_key=provider_check.api_key,+ provider_category=ProviderCategory.byok,+ ).cast_to_subtype()++ # TODO: add more string sanity checks here before we hit actual endpoints+ if not provider.api_key:+ raise ValueError("API key is required")++ provider.check_api_key()