Skip to content

armgit / letta

Public Imported from GitHub
0

Platform for stateful agents: AI with advanced memory that can learn and self-improve over time.

fix: baseten (#9849)

Ari Webb6fcdf9cea78a
diff --git a/letta/llm_api/baseten_client.py b/letta/llm_api/baseten_client.py
index 59c8542a3..204fe5f61 100644
--- a/letta/llm_api/baseten_client.py
+++ b/letta/llm_api/baseten_client.py
@@ -43,8 +43,7 @@ class BasetenClient(OpenAIClient):
) -> dict:
data = super().build_request_data(agent_type, messages, llm_config, tools, force_tool_call, requires_subsequent_tool_call)
- # Baseten expects a generic model name
- data["model"] = "baseten"
+ # model field is passed through from llm_config.model (e.g. "zai-org/GLM-5")
# Baseten uses max_tokens, not max_completion_tokens
if "max_completion_tokens" in data:
@@ -84,7 +83,7 @@ class BasetenClient(OpenAIClient):
@trace_method
def request(self, request_data: dict, llm_config: LLMConfig) -> dict:
api_key = model_settings.baseten_api_key
- base_url = _build_base_url(llm_config.model)
+ base_url = _build_base_url(llm_config.model_endpoint)
client = OpenAI(api_key="unused", base_url=base_url, default_headers={"Authorization": f"Api-Key {api_key}"})
response: ChatCompletion = client.chat.completions.create(**request_data)
return response.model_dump()
@@ -93,7 +92,7 @@ class BasetenClient(OpenAIClient):
async def request_async(self, request_data: dict, llm_config: LLMConfig) -> dict:
request_data = sanitize_unicode_surrogates(request_data)
api_key = model_settings.baseten_api_key
- base_url = _build_base_url(llm_config.model)
+ base_url = _build_base_url(llm_config.model_endpoint)
client = AsyncOpenAI(api_key="unused", base_url=base_url, default_headers={"Authorization": f"Api-Key {api_key}"})
response: ChatCompletion = await client.chat.completions.create(**request_data)
return response.model_dump()
@@ -102,7 +101,7 @@ class BasetenClient(OpenAIClient):
async def stream_async(self, request_data: dict, llm_config: LLMConfig) -> AsyncStream[ChatCompletionChunk]:
request_data = sanitize_unicode_surrogates(request_data)
api_key = model_settings.baseten_api_key
- base_url = _build_base_url(llm_config.model)
+ base_url = _build_base_url(llm_config.model_endpoint)
client = AsyncOpenAI(api_key="unused", base_url=base_url, default_headers={"Authorization": f"Api-Key {api_key}"})
response_stream: AsyncStream[ChatCompletionChunk] = await client.chat.completions.create(
**request_data, stream=True, stream_options={"include_usage": True}
diff --git a/letta/services/llm_router/client.py b/letta/services/llm_router/client.py
index cb4ed9d17..392b3fccd 100644
--- a/letta/services/llm_router/client.py
+++ b/letta/services/llm_router/client.py
@@ -42,8 +42,9 @@ def _build_baseten_config() -> "LLMConfig":
from letta.schemas.llm_config import LLMConfig
return LLMConfig(
- model="3m5zok9w",
+ model="zai-org/GLM-5",
model_endpoint_type="baseten",
+ model_endpoint="3m5zok9w",
context_window=180000,
max_tokens=16384,
handle="baseten/glm-5",