Platform for stateful agents: AI with advanced memory that can learn and self-improve over time.
fix: baseten (#9849)
Ari Webb6fcdf9cea78a
diff --git a/letta/llm_api/baseten_client.py b/letta/llm_api/baseten_client.pyindex 59c8542a3..204fe5f61 100644--- a/letta/llm_api/baseten_client.py+++ b/letta/llm_api/baseten_client.py@@ -43,8 +43,7 @@ class BasetenClient(OpenAIClient):) -> dict:data = super().build_request_data(agent_type, messages, llm_config, tools, force_tool_call, requires_subsequent_tool_call)- # Baseten expects a generic model name- data["model"] = "baseten"+ # model field is passed through from llm_config.model (e.g. "zai-org/GLM-5")# Baseten uses max_tokens, not max_completion_tokensif "max_completion_tokens" in data:@@ -84,7 +83,7 @@ class BasetenClient(OpenAIClient):@trace_methoddef request(self, request_data: dict, llm_config: LLMConfig) -> dict:api_key = model_settings.baseten_api_key- base_url = _build_base_url(llm_config.model)+ base_url = _build_base_url(llm_config.model_endpoint)client = OpenAI(api_key="unused", base_url=base_url, default_headers={"Authorization": f"Api-Key {api_key}"})response: ChatCompletion = client.chat.completions.create(**request_data)return response.model_dump()@@ -93,7 +92,7 @@ class BasetenClient(OpenAIClient):async def request_async(self, request_data: dict, llm_config: LLMConfig) -> dict:request_data = sanitize_unicode_surrogates(request_data)api_key = model_settings.baseten_api_key- base_url = _build_base_url(llm_config.model)+ base_url = _build_base_url(llm_config.model_endpoint)client = AsyncOpenAI(api_key="unused", base_url=base_url, default_headers={"Authorization": f"Api-Key {api_key}"})response: ChatCompletion = await client.chat.completions.create(**request_data)return response.model_dump()@@ -102,7 +101,7 @@ class BasetenClient(OpenAIClient):async def stream_async(self, request_data: dict, llm_config: LLMConfig) -> AsyncStream[ChatCompletionChunk]:request_data = sanitize_unicode_surrogates(request_data)api_key = model_settings.baseten_api_key- base_url = _build_base_url(llm_config.model)+ base_url = _build_base_url(llm_config.model_endpoint)client = AsyncOpenAI(api_key="unused", base_url=base_url, default_headers={"Authorization": f"Api-Key {api_key}"})response_stream: AsyncStream[ChatCompletionChunk] = await client.chat.completions.create(**request_data, stream=True, stream_options={"include_usage": True}diff --git a/letta/services/llm_router/client.py b/letta/services/llm_router/client.pyindex cb4ed9d17..392b3fccd 100644--- a/letta/services/llm_router/client.py+++ b/letta/services/llm_router/client.py@@ -42,8 +42,9 @@ def _build_baseten_config() -> "LLMConfig":from letta.schemas.llm_config import LLMConfigreturn LLMConfig(- model="3m5zok9w",+ model="zai-org/GLM-5",model_endpoint_type="baseten",+ model_endpoint="3m5zok9w",context_window=180000,max_tokens=16384,handle="baseten/glm-5",