Skip to content

armgit / letta

Public Imported from GitHub
0

Platform for stateful agents: AI with advanced memory that can learn and self-improve over time.

feat(core): switch auto mode primary to baseten serverless (#10051)

Ari Webbe32eb57f084f
diff --git a/letta/interfaces/openai_streaming_interface.py b/letta/interfaces/openai_streaming_interface.py
index a973c1fcf..317083276 100644
--- a/letta/interfaces/openai_streaming_interface.py
+++ b/letta/interfaces/openai_streaming_interface.py
@@ -875,21 +875,6 @@ class SimpleOpenAIStreamingInterface:
choice = chunk.choices[0]
message_delta = choice.delta
- if message_delta.content is not None and message_delta.content != "":
- if prev_message_type and prev_message_type != "assistant_message":
- message_index += 1
- assistant_msg = AssistantMessage(
- id=self.letta_message_id,
- content=message_delta.content,
- date=datetime.now(timezone.utc).isoformat(),
- otid=Message.generate_otid_from_id(self.letta_message_id, message_index),
- run_id=self.run_id,
- step_id=self.step_id,
- )
- self.content_messages.append(assistant_msg)
- prev_message_type = assistant_msg.message_type
- yield assistant_msg
-
if hasattr(chunk, "choices") and len(chunk.choices) > 0 and hasattr(chunk.choices[0], "delta"):
delta = chunk.choices[0].delta
# Check for reasoning_content (standard) or reasoning (OpenRouter)
@@ -911,6 +896,21 @@ class SimpleOpenAIStreamingInterface:
prev_message_type = reasoning_msg.message_type
yield reasoning_msg
+ if message_delta.content is not None and message_delta.content != "":
+ if prev_message_type and prev_message_type != "assistant_message":
+ message_index += 1
+ assistant_msg = AssistantMessage(
+ id=self.letta_message_id,
+ content=message_delta.content,
+ date=datetime.now(timezone.utc).isoformat(),
+ otid=Message.generate_otid_from_id(self.letta_message_id, message_index),
+ run_id=self.run_id,
+ step_id=self.step_id,
+ )
+ self.content_messages.append(assistant_msg)
+ prev_message_type = assistant_msg.message_type
+ yield assistant_msg
+
if message_delta.tool_calls is not None and len(message_delta.tool_calls) > 0:
# Accumulate per-index tool call fragments and emit deltas
for tool_call in message_delta.tool_calls:
diff --git a/letta/llm_api/baseten_client.py b/letta/llm_api/baseten_client.py
index 069873911..4212fdf01 100644
--- a/letta/llm_api/baseten_client.py
+++ b/letta/llm_api/baseten_client.py
@@ -1,3 +1,4 @@
+import logging
from typing import List, Optional
from openai import AsyncOpenAI, AsyncStream, OpenAI
@@ -12,6 +13,8 @@ from letta.schemas.llm_config import LLMConfig
from letta.schemas.message import Message as PydanticMessage
from letta.settings import model_settings
+logger = logging.getLogger(__name__)
+
def _build_base_url(model_endpoint: str) -> str:
"""Build the Baseten endpoint URL.
@@ -97,6 +100,39 @@ class BasetenClient(OpenAIClient):
return {"api_key": api_key, "base_url": base_url}
return {"api_key": "unused", "base_url": base_url, "default_headers": {"Authorization": f"Api-Key {api_key}"}}
+ @trace_method
+ def handle_llm_error(self, e: Exception, llm_config: Optional[LLMConfig] = None) -> Exception:
+ """Log Baseten provider errors with searchable tag, then delegate to parent."""
+ status_code = getattr(e, "status_code", None)
+ body = getattr(e, "body", None)
+
+ # Extract error details from the response body
+ error_code = None
+ error_message = None
+ if isinstance(body, dict):
+ error_data = body.get("error", {})
+ if isinstance(error_data, dict):
+ error_code = error_data.get("code") or error_data.get("type")
+ error_message = error_data.get("message")
+ elif isinstance(error_data, str):
+ error_message = error_data
+ if error_message is None:
+ error_message = str(e)[:500]
+
+ is_serverless = llm_config and llm_config.model_endpoint and llm_config.model_endpoint.startswith("https://")
+
+ logger.warning(
+ f"[BASETEN_PROVIDER_ERROR] handle={llm_config.handle if llm_config else None} "
+ f"model={llm_config.model if llm_config else None} "
+ f"serverless={is_serverless} "
+ f"error_type={type(e).__name__} "
+ f"status={status_code} "
+ f"error_code={error_code} "
+ f"message={error_message}"
+ )
+
+ return super().handle_llm_error(e, llm_config=llm_config)
+
@trace_method
def request(self, request_data: dict, llm_config: LLMConfig) -> dict:
client = OpenAI(**self._build_client_kwargs(llm_config))