Skip to content

armgit / letta

Public Imported from GitHub
0

Platform for stateful agents: AI with advanced memory that can learn and self-improve over time.

feat: Asyncify anthropic model listing (#2283)

Matthew Zhouedc07175791c
diff --git a/letta/llm_api/anthropic.py b/letta/llm_api/anthropic.py
index 89329d01d..fadc652d6 100644
--- a/letta/llm_api/anthropic.py
+++ b/letta/llm_api/anthropic.py
@@ -134,13 +134,13 @@ def anthropic_check_valid_api_key(api_key: Union[str, None]) -> None:
def antropic_get_model_context_window(url: str, api_key: Union[str, None], model: str) -> int:
- for model_dict in anthropic_get_model_list(url=url, api_key=api_key):
+ for model_dict in anthropic_get_model_list(api_key=api_key):
if model_dict["name"] == model:
return model_dict["context_window"]
raise ValueError(f"Can't find model '{model}' in Anthropic model list")
-def anthropic_get_model_list(url: str, api_key: Union[str, None]) -> dict:
+def anthropic_get_model_list(api_key: Optional[str]) -> dict:
"""https://docs.anthropic.com/claude/docs/models-overview"""
# NOTE: currently there is no GET /models, so we need to hardcode
@@ -159,6 +159,25 @@ def anthropic_get_model_list(url: str, api_key: Union[str, None]) -> dict:
return models_json["data"]
+async def anthropic_get_model_list_async(api_key: Optional[str]) -> dict:
+ """https://docs.anthropic.com/claude/docs/models-overview"""
+
+ # NOTE: currently there is no GET /models, so we need to hardcode
+ # return MODEL_LIST
+
+ if api_key:
+ anthropic_client = anthropic.AsyncAnthropic(api_key=api_key)
+ elif model_settings.anthropic_api_key:
+ anthropic_client = anthropic.AsyncAnthropic()
+ else:
+ raise ValueError("No API key provided")
+
+ models = await anthropic_client.models.list()
+ models_json = models.model_dump()
+ assert "data" in models_json, f"Anthropic model query response missing 'data' field: {models_json}"
+ return models_json["data"]
+
+
def convert_tools_to_anthropic_format(tools: List[Tool]) -> List[dict]:
"""See: https://docs.anthropic.com/claude/docs/tool-use
diff --git a/letta/schemas/providers.py b/letta/schemas/providers.py
index 1c0072b44..fb8e5e05c 100644
--- a/letta/schemas/providers.py
+++ b/letta/schemas/providers.py
@@ -733,26 +733,19 @@ class AnthropicProvider(Provider):
anthropic_check_valid_api_key(self.api_key)
def list_llm_models(self) -> List[LLMConfig]:
- from letta.llm_api.anthropic import MODEL_LIST, anthropic_get_model_list
+ from letta.llm_api.anthropic import anthropic_get_model_list
- models = anthropic_get_model_list(self.base_url, api_key=self.api_key)
+ models = anthropic_get_model_list(api_key=self.api_key)
+ return self._list_llm_models(models)
- """
- Example response:
- {
- "data": [
- {
- "type": "model",
- "id": "claude-3-5-sonnet-20241022",
- "display_name": "Claude 3.5 Sonnet (New)",
- "created_at": "2024-10-22T00:00:00Z"
- }
- ],
- "has_more": true,
- "first_id": "<string>",
- "last_id": "<string>"
- }
- """
+ async def list_llm_models_async(self) -> List[LLMConfig]:
+ from letta.llm_api.anthropic import anthropic_get_model_list_async
+
+ models = await anthropic_get_model_list_async(api_key=self.api_key)
+ return self._list_llm_models(models)
+
+ def _list_llm_models(self, models) -> List[LLMConfig]:
+ from letta.llm_api.anthropic import MODEL_LIST
configs = []
for model in models:
@@ -810,9 +803,6 @@ class AnthropicProvider(Provider):
)
return configs
- def list_embedding_models(self) -> List[EmbeddingConfig]:
- return []
-
class MistralProvider(Provider):
provider_type: Literal[ProviderType.mistral] = Field(ProviderType.mistral, description="The type of the provider.")
diff --git a/tests/test_providers.py b/tests/test_providers.py
index 67ae5a831..ea79126d7 100644
--- a/tests/test_providers.py
+++ b/tests/test_providers.py
@@ -61,6 +61,17 @@ def test_anthropic():
assert models[0].handle == f"{provider.name}/{models[0].model}"
+async def test_anthropic_async():
+ provider = AnthropicProvider(
+ name="anthropic",
+ api_key=model_settings.anthropic_api_key,
+ )
+ models = await provider.list_llm_models_async()
+ assert len(models) > 0
+ assert models[0].handle == f"{provider.name}/{models[0].model}"
+
+
def test_groq():
provider = GroqProvider(
name="groq",