Skip to content

armgit / letta

Public Imported from GitHub
0

Platform for stateful agents: AI with advanced memory that can learn and self-improve over time.

feat: handle overloaded errors [LET-5969] (#5856)

Sarah Woodersd109bab5876a
diff --git a/letta/errors.py b/letta/errors.py
index 34a82ccb8..e2c1074ba 100644
--- a/letta/errors.py
+++ b/letta/errors.py
@@ -202,6 +202,10 @@ class LLMTimeoutError(LLMError):
"""Error when LLM request times out"""
+class LLMProviderOverloaded(LLMError):
+ """Error when LLM provider is overloaded"""
+
+
class BedrockPermissionError(LettaError):
"""Exception raised for errors in the Bedrock permission process."""
diff --git a/letta/llm_api/anthropic_client.py b/letta/llm_api/anthropic_client.py
index 322c6a5ad..92246c1b6 100644
--- a/letta/llm_api/anthropic_client.py
+++ b/letta/llm_api/anthropic_client.py
@@ -19,6 +19,7 @@ from letta.errors import (
LLMConnectionError,
LLMNotFoundError,
LLMPermissionDeniedError,
+ LLMProviderOverloaded,
LLMRateLimitError,
LLMServerError,
LLMTimeoutError,
@@ -513,6 +514,11 @@ class AnthropicClient(LLMClientBase):
if isinstance(e, anthropic.APIStatusError):
logger.warning(f"[Anthropic] API status error: {str(e)}")
+ if "overloaded" in str(e).lower():
+ return LLMProviderOverloaded(
+ message=f"Anthropic API is overloaded: {str(e)}",
+ code=ErrorCode.INTERNAL_SERVER_ERROR,
+ )
return LLMServerError(
message=f"Anthropic API error: {str(e)}",
code=ErrorCode.INTERNAL_SERVER_ERROR,
diff --git a/letta/server/rest_api/app.py b/letta/server/rest_api/app.py
index a3584f0c5..055b119d7 100644
--- a/letta/server/rest_api/app.py
+++ b/letta/server/rest_api/app.py
@@ -40,6 +40,7 @@ from letta.errors import (
LettaUserNotFoundError,
LLMAuthenticationError,
LLMError,
+ LLMProviderOverloaded,
LLMRateLimitError,
LLMTimeoutError,
PendingApprovalError,
@@ -398,6 +399,7 @@ def create_application() -> "FastAPI":
# 503 Service Unavailable errors
app.add_exception_handler(OperationalError, _error_handler_503)
app.add_exception_handler(LettaServiceUnavailableError, _error_handler_503)
+ app.add_exception_handler(LLMProviderOverloaded, _error_handler_503)
@app.exception_handler(IncompatibleAgentType)
async def handle_incompatible_agent_type(request: Request, exc: IncompatibleAgentType):