Platform for stateful agents: AI with advanced memory that can learn and self-improve over time.
fix: make togetherai nebius xai etc usable via the openaiprovider (#1981)
Charles Packerfce28c73e381
diff --git a/letta/llm_api/llm_api_tools.py b/letta/llm_api/llm_api_tools.pyindex 7a778cdad..d86abc9b3 100644--- a/letta/llm_api/llm_api_tools.py+++ b/letta/llm_api/llm_api_tools.py@@ -215,6 +215,9 @@ def create(chat_completion_request=data,stream_interface=stream_interface,name=name,+ # NOTE: needs to be true for OpenAI proxies that use the `reasoning_content` field+ # For example, DeepSeek, or LM Studio+ expect_reasoning_content=False,)else: # Client did not request token streaming (expect a blocking backend response)data.stream = False@@ -272,6 +275,9 @@ def create(chat_completion_request=data,stream_interface=stream_interface,name=name,+ # TODO turn on to support reasoning content from xAI reasoners:+ # https://docs.x.ai/docs/guides/reasoning#reasoning+ expect_reasoning_content=False,)else: # Client did not request token streaming (expect a blocking backend response)data.stream = False@@ -486,7 +492,10 @@ def create(if stream:raise NotImplementedError(f"Streaming not yet implemented for TogetherAI (via the /completions endpoint).")- if model_settings.together_api_key is None and llm_config.model_endpoint == "https://api.together.ai/v1/completions":+ if model_settings.together_api_key is None and (+ llm_config.model_endpoint == "https://api.together.ai/v1/completions"+ or llm_config.model_endpoint == "https://api.together.xyz/v1/completions"+ ):raise LettaConfigurationError(message="TogetherAI key is missing from letta config file", missing_fields=["together_api_key"])return get_chat_completion(@@ -560,6 +569,8 @@ def create(chat_completion_request=data,stream_interface=stream_interface,name=name,+ # TODO should we toggle for R1 vs V3?+ expect_reasoning_content=True,)else: # Client did not request token streaming (expect a blocking backend response)data.stream = Falsediff --git a/letta/llm_api/openai.py b/letta/llm_api/openai.pyindex 2fe8ade33..6a0f182b0 100644--- a/letta/llm_api/openai.py+++ b/letta/llm_api/openai.py@@ -8,7 +8,13 @@ from letta.constants import LETTA_MODEL_ENDPOINTfrom letta.errors import ErrorCode, LLMAuthenticationError, LLMErrorfrom letta.helpers.datetime_helpers import timestamp_to_datetimefrom letta.llm_api.helpers import add_inner_thoughts_to_functions, convert_to_structured_output, make_post_request-from letta.llm_api.openai_client import accepts_developer_role, supports_parallel_tool_calling, supports_temperature_param+from letta.llm_api.openai_client import (+ accepts_developer_role,+ requires_auto_tool_choice,+ supports_parallel_tool_calling,+ supports_structured_output,+ supports_temperature_param,+)from letta.local_llm.constants import INNER_THOUGHTS_KWARG, INNER_THOUGHTS_KWARG_DESCRIPTION, INNER_THOUGHTS_KWARG_DESCRIPTION_GO_FIRSTfrom letta.local_llm.utils import num_tokens_from_functions, num_tokens_from_messagesfrom letta.log import get_logger@@ -50,9 +56,7 @@ def openai_check_valid_api_key(base_url: str, api_key: Union[str, None]) -> Noneraise ValueError("No API key provided")-def openai_get_model_list(- url: str, api_key: Optional[str] = None, fix_url: Optional[bool] = False, extra_params: Optional[dict] = None-) -> dict:+def openai_get_model_list(url: str, api_key: Optional[str] = None, fix_url: bool = False, extra_params: Optional[dict] = None) -> dict:"""https://platform.openai.com/docs/api-reference/models/list"""from letta.utils import printd@@ -154,7 +158,10 @@ def build_openai_chat_completions_request(elif function_call not in ["none", "auto", "required"]:tool_choice = ToolFunctionChoice(type="function", function=ToolFunctionChoiceFunctionCall(name=function_call))else:- tool_choice = function_call+ if requires_auto_tool_choice(llm_config):+ tool_choice = "auto"+ else:+ tool_choice = function_calldata = ChatCompletionRequest(model=model,messages=openai_message_list,@@ -197,12 +204,13 @@ def build_openai_chat_completions_request(if use_structured_output and data.tools is not None and len(data.tools) > 0:# Convert to structured output style (which has 'strict' and no optionals)for tool in data.tools:- try:- # tool["function"] = convert_to_structured_output(tool["function"])- structured_output_version = convert_to_structured_output(tool.function.model_dump())- tool.function = FunctionSchema(**structured_output_version)- except ValueError as e:- warnings.warn(f"Failed to convert tool function to structured output, tool={tool}, error={e}")+ if supports_structured_output(llm_config):+ try:+ # tool["function"] = convert_to_structured_output(tool["function"])+ structured_output_version = convert_to_structured_output(tool.function.model_dump())+ tool.function = FunctionSchema(**structured_output_version)+ except ValueError as e:+ warnings.warn(f"Failed to convert tool function to structured output, tool={tool}, error={e}")return data@@ -221,7 +229,7 @@ def openai_chat_completions_process_stream(expect_reasoning_content: bool = True,name: Optional[str] = None,) -> ChatCompletionResponse:- """Process a streaming completion response, and return a ChatCompletionRequest at the end.+ """Process a streaming completion response, and return a ChatCompletionResponse at the end.To "stream" the response in Letta, we want to call a streaming-compatible interface functionon the chunks received from the OpenAI-compatible server POST SSE response.@@ -293,6 +301,9 @@ def openai_chat_completions_process_stream(url=url, api_key=api_key, chat_completion_request=chat_completion_request):assert isinstance(chat_completion_chunk, ChatCompletionChunkResponse), type(chat_completion_chunk)+ if chat_completion_chunk.choices is None or len(chat_completion_chunk.choices) == 0:+ warnings.warn(f"No choices in chunk: {chat_completion_chunk}")+ continue# NOTE: this assumes that the tool call ID will only appear in one of the chunks during the streamif override_tool_call_id:@@ -429,6 +440,9 @@ def openai_chat_completions_process_stream(except Exception as e:if stream_interface:stream_interface.stream_end()+ import traceback++ traceback.print_exc()logger.error(f"Parsing ChatCompletion stream failed with error:\n{str(e)}")raise efinally:@@ -463,14 +477,27 @@ def openai_chat_completions_request_stream(url: str,api_key: str,chat_completion_request: ChatCompletionRequest,+ fix_url: bool = False,) -> Generator[ChatCompletionChunkResponse, None, None]:++ # In some cases we may want to double-check the URL and do basic correction, eg:+ # In Letta config the address for vLLM is w/o a /v1 suffix for simplicity+ # However if we're treating the server as an OpenAI proxy we want the /v1 suffix on our model hit+ if fix_url:+ if not url.endswith("/v1"):+ url = smart_urljoin(url, "v1")+data = prepare_openai_payload(chat_completion_request)data["stream"] = Trueclient = OpenAI(api_key=api_key, base_url=url, max_retries=0)- stream = client.chat.completions.create(**data)- for chunk in stream:- # TODO: Use the native OpenAI objects here?- yield ChatCompletionChunkResponse(**chunk.model_dump(exclude_none=True))+ try:+ stream = client.chat.completions.create(**data)+ for chunk in stream:+ # TODO: Use the native OpenAI objects here?+ yield ChatCompletionChunkResponse(**chunk.model_dump(exclude_none=True))+ except Exception as e:+ print(f"Error request stream from /v1/chat/completions, url={url}, data={data}:\n{e}")+ raise edef openai_chat_completions_request(diff --git a/letta/llm_api/openai_client.py b/letta/llm_api/openai_client.pyindex c641f5e1c..dc6804d12 100644--- a/letta/llm_api/openai_client.py+++ b/letta/llm_api/openai_client.py@@ -75,6 +75,35 @@ def supports_parallel_tool_calling(model: str) -> bool:return True+# TODO move into LLMConfig as a field?+def supports_structured_output(llm_config: LLMConfig) -> bool:+ """Certain providers don't support structured output."""++ # FIXME pretty hacky - turn off for providers we know users will use,+ # but also don't support structured output+ if "nebius.com" in llm_config.model_endpoint:+ return False+ else:+ return True+++# TODO move into LLMConfig as a field?+def requires_auto_tool_choice(llm_config: LLMConfig) -> bool:+ """Certain providers require the tool choice to be set to 'auto'."""++ if "nebius.com" in llm_config.model_endpoint:+ return True+ # proxy also has this issue (FIXME check)+ elif llm_config.model_endpoint == LETTA_MODEL_ENDPOINT:+ return True+ # same with vLLM (FIXME check)+ elif llm_config.handle and "vllm" in llm_config.handle:+ return True+ else:+ # will use "required" instead of "auto"+ return False++class OpenAIClient(LLMClientBase):def _prepare_client_kwargs(self, llm_config: LLMConfig) -> dict:api_key = None@@ -136,7 +165,7 @@ class OpenAIClient(LLMClientBase):# TODO(matt) move into LLMConfig# TODO: This vllm checking is very brittle and is a patch at mosttool_choice = None- if llm_config.model_endpoint == LETTA_MODEL_ENDPOINT or (llm_config.handle and "vllm" in llm_config.handle):+ if requires_auto_tool_choice(llm_config):tool_choice = "auto" # TODO change to "required" once proxy supports itelif tools:# only set if tools is non-Null@@ -171,11 +200,12 @@ class OpenAIClient(LLMClientBase):if data.tools is not None and len(data.tools) > 0:# Convert to structured output style (which has 'strict' and no optionals)for tool in data.tools:- try:- structured_output_version = convert_to_structured_output(tool.function.model_dump())- tool.function = FunctionSchema(**structured_output_version)- except ValueError as e:- logger.warning(f"Failed to convert tool function to structured output, tool={tool}, error={e}")+ if supports_structured_output(llm_config):+ try:+ structured_output_version = convert_to_structured_output(tool.function.model_dump())+ tool.function = FunctionSchema(**structured_output_version)+ except ValueError as e:+ logger.warning(f"Failed to convert tool function to structured output, tool={tool}, error={e}")return data.model_dump(exclude_unset=True)diff --git a/letta/schemas/llm_config.py b/letta/schemas/llm_config.pyindex 903d9a7ee..ab0247088 100644--- a/letta/schemas/llm_config.py+++ b/letta/schemas/llm_config.py@@ -24,7 +24,6 @@ class LLMConfig(BaseModel):max_tokens (int): The maximum number of tokens to generate."""- # TODO: 🤮 don't default to a vendor! bug city!model: str = Field(..., description="LLM model name. ")model_endpoint_type: Literal["openai",diff --git a/letta/schemas/openai/chat_completion_response.py b/letta/schemas/openai/chat_completion_response.pyindex d4332b22d..44ef5cff1 100644--- a/letta/schemas/openai/chat_completion_response.py+++ b/letta/schemas/openai/chat_completion_response.py@@ -1,5 +1,5 @@import datetime-from typing import Dict, List, Literal, Optional, Union+from typing import List, Literal, Optional, Unionfrom pydantic import BaseModel@@ -27,6 +27,7 @@ class LogProbToken(BaseModel):bytes: Optional[List[int]]+# Legacy?class MessageContentLogProb(BaseModel):token: strlogprob: float@@ -34,6 +35,25 @@ class MessageContentLogProb(BaseModel):top_logprobs: Optional[List[LogProbToken]]+class TopLogprob(BaseModel):+ token: str+ bytes: Optional[List[int]] = None+ logprob: float+++class ChatCompletionTokenLogprob(BaseModel):+ token: str+ bytes: Optional[List[int]] = None+ logprob: float+ top_logprobs: List[TopLogprob]+++class ChoiceLogprobs(BaseModel):+ content: Optional[List[ChatCompletionTokenLogprob]] = None++ refusal: Optional[List[ChatCompletionTokenLogprob]] = None++class Message(BaseModel):content: Optional[str] = Nonetool_calls: Optional[List[ToolCall]] = None@@ -49,7 +69,7 @@ class Choice(BaseModel):finish_reason: strindex: intmessage: Message- logprobs: Optional[Dict[str, Union[List[MessageContentLogProb], None]]] = None+ logprobs: Optional[ChoiceLogprobs] = Noneseed: Optional[int] = None # found in TogetherAI@@ -134,7 +154,7 @@ class ChatCompletionResponse(BaseModel):class FunctionCallDelta(BaseModel):# arguments: Optional[str] = Nonename: Optional[str] = None- arguments: str+ arguments: Optional[str] = None# name: str@@ -179,7 +199,7 @@ class ChunkChoice(BaseModel):finish_reason: Optional[str] = None # NOTE: when streaming will be nullindex: intdelta: MessageDelta- logprobs: Optional[Dict[str, Union[List[MessageContentLogProb], None]]] = None+ logprobs: Optional[ChoiceLogprobs] = Noneclass ChatCompletionChunkResponse(BaseModel):diff --git a/letta/schemas/providers.py b/letta/schemas/providers.pyindex f1e9edd63..822b291a2 100644--- a/letta/schemas/providers.py+++ b/letta/schemas/providers.py@@ -4,7 +4,7 @@ from typing import List, Literal, Optionalfrom pydantic import BaseModel, Field, model_validator-from letta.constants import LETTA_MODEL_ENDPOINT, LLM_MAX_TOKENS, MIN_CONTEXT_WINDOW+from letta.constants import DEFAULT_EMBEDDING_CHUNK_SIZE, LETTA_MODEL_ENDPOINT, LLM_MAX_TOKENS, MIN_CONTEXT_WINDOWfrom letta.llm_api.azure_openai import get_azure_chat_completions_endpoint, get_azure_embeddings_endpointfrom letta.llm_api.azure_openai_constants import AZURE_MODEL_TO_CONTEXT_LENGTHfrom letta.schemas.embedding_config import EmbeddingConfig@@ -57,7 +57,7 @@ class Provider(ProviderBase):"""String representation of the provider for display purposes"""raise NotImplementedError- def get_handle(self, model_name: str, is_embedding: bool = False) -> str:+ def get_handle(self, model_name: str, is_embedding: bool = False, base_name: Optional[str] = None) -> str:"""Get the handle for a model, with support for custom overrides.@@ -68,11 +68,13 @@ class Provider(ProviderBase):Returns:str: The handle for the model."""+ base_name = base_name if base_name else self.name+overrides = EMBEDDING_HANDLE_OVERRIDES if is_embedding else LLM_HANDLE_OVERRIDES- if self.name in overrides and model_name in overrides[self.name]:- model_name = overrides[self.name][model_name]+ if base_name in overrides and model_name in overrides[base_name]:+ model_name = overrides[base_name][model_name]- return f"{self.name}/{model_name}"+ return f"{base_name}/{model_name}"def cast_to_subtype(self):match (self.provider_type):@@ -162,21 +164,34 @@ class OpenAIProvider(Provider):openai_check_valid_api_key(self.base_url, self.api_key)- def list_llm_models(self) -> List[LLMConfig]:+ def _get_models(self) -> List[dict]:from letta.llm_api.openai import openai_get_model_list# Some hardcoded support for OpenRouter (so that we only get models with tool calling support)...# See: https://openrouter.ai/docs/requestsextra_params = {"supported_parameters": "tools"} if "openrouter.ai" in self.base_url else None- response = openai_get_model_list(self.base_url, api_key=self.api_key, extra_params=extra_params)- # TogetherAI's response is missing the 'data' field- # assert "data" in response, f"OpenAI model query response missing 'data' field: {response}"+ # Similar to Nebius+ extra_params = {"verbose": True} if "nebius.com" in self.base_url else None++ response = openai_get_model_list(+ self.base_url,+ api_key=self.api_key,+ extra_params=extra_params,+ # fix_url=True, # NOTE: make sure together ends with /v1+ )+if "data" in response:data = response["data"]else:+ # TogetherAI's response is missing the 'data' fielddata = response+ return data++ def list_llm_models(self) -> List[LLMConfig]:+ data = self._get_models()+configs = []for model in data:assert "id" in model, f"OpenAI model missing 'id' field: {model}"@@ -192,8 +207,8 @@ class OpenAIProvider(Provider):continue# TogetherAI includes the type, which we can use to filter out embedding models- if self.base_url == "https://api.together.ai/v1":- if "type" in model and model["type"] != "chat":+ if "api.together.ai" in self.base_url or "api.together.xyz" in self.base_url:+ if "type" in model and model["type"] not in ["chat", "language"]:continue# for TogetherAI, we need to skip the models that don't support JSON mode / function calling@@ -211,11 +226,25 @@ class OpenAIProvider(Provider):continueif model["config"]["chat_template"] is None:continue+ if model["config"]["chat_template"] is not None and "tools" not in model["config"]["chat_template"]:+ # NOTE: this is a hack to filter out models that don't support tool calling+ continueif "tools" not in model["config"]["chat_template"]:continue# if "config" in data and "chat_template" in data["config"] and "tools" not in data["config"]["chat_template"]:# continue+ if "nebius.com" in self.base_url:+ # Nebius includes the type, which we can use to filter for text models+ try:+ model_type = model["architecture"]["modality"]+ if model_type not in ["text->text", "text+image->text"]:+ # print(f"Skipping model w/ modality {model_type}:\n{model}")+ continue+ except KeyError:+ print(f"Couldn't access architecture type field, skipping model:\n{model}")+ continue+# for openai, filter modelsif self.base_url == "https://api.openai.com/v1":allowed_types = ["gpt-4", "o1", "o3"]@@ -235,13 +264,19 @@ class OpenAIProvider(Provider):if skip:continue+ # set the handle to openai-proxy if the base URL isn't OpenAI+ if self.base_url != "https://api.openai.com/v1":+ handle = self.get_handle(model_name, base_name="openai-proxy")+ else:+ handle = self.get_handle(model_name)+configs.append(LLMConfig(model=model_name,model_endpoint_type="openai",model_endpoint=self.base_url,context_window=context_window_size,- handle=self.get_handle(model_name),+ handle=handle,provider_name=self.name,provider_category=self.provider_category,)@@ -256,33 +291,87 @@ class OpenAIProvider(Provider):def list_embedding_models(self) -> List[EmbeddingConfig]:- # TODO: actually automatically list models- return [- EmbeddingConfig(- embedding_model="text-embedding-ada-002",- embedding_endpoint_type="openai",- embedding_endpoint=self.base_url,- embedding_dim=1536,- embedding_chunk_size=300,- handle=self.get_handle("text-embedding-ada-002", is_embedding=True),- ),- EmbeddingConfig(- embedding_model="text-embedding-3-small",- embedding_endpoint_type="openai",- embedding_endpoint=self.base_url,- embedding_dim=2000,- embedding_chunk_size=300,- handle=self.get_handle("text-embedding-3-small", is_embedding=True),- ),- EmbeddingConfig(- embedding_model="text-embedding-3-large",- embedding_endpoint_type="openai",- embedding_endpoint=self.base_url,- embedding_dim=2000,- embedding_chunk_size=300,- handle=self.get_handle("text-embedding-3-large", is_embedding=True),- ),- ]+ if self.base_url == "https://api.openai.com/v1":+ # TODO: actually automatically list models for OpenAI+ return [+ EmbeddingConfig(+ embedding_model="text-embedding-ada-002",+ embedding_endpoint_type="openai",+ embedding_endpoint=self.base_url,+ embedding_dim=1536,+ embedding_chunk_size=300,+ handle=self.get_handle("text-embedding-ada-002", is_embedding=True),+ ),+ EmbeddingConfig(+ embedding_model="text-embedding-3-small",+ embedding_endpoint_type="openai",+ embedding_endpoint=self.base_url,+ embedding_dim=2000,+ embedding_chunk_size=300,+ handle=self.get_handle("text-embedding-3-small", is_embedding=True),+ ),+ EmbeddingConfig(+ embedding_model="text-embedding-3-large",+ embedding_endpoint_type="openai",+ embedding_endpoint=self.base_url,+ embedding_dim=2000,+ embedding_chunk_size=300,+ handle=self.get_handle("text-embedding-3-large", is_embedding=True),+ ),+ ]++ else:+ # Actually attempt to list+ data = self._get_models()++ configs = []+ for model in data:+ assert "id" in model, f"Model missing 'id' field: {model}"+ model_name = model["id"]++ if "context_length" in model:+ # Context length is returned in Nebius as "context_length"+ context_window_size = model["context_length"]+ else:+ context_window_size = self.get_model_context_window_size(model_name)++ # We need the context length for embeddings too+ if not context_window_size:+ continue++ if "nebius.com" in self.base_url:+ # Nebius includes the type, which we can use to filter for embedidng models+ try:+ model_type = model["architecture"]["modality"]+ if model_type not in ["text->embedding"]:+ # print(f"Skipping model w/ modality {model_type}:\n{model}")+ continue+ except KeyError:+ print(f"Couldn't access architecture type field, skipping model:\n{model}")+ continue++ elif "together.ai" in self.base_url or "together.xyz" in self.base_url:+ # TogetherAI includes the type, which we can use to filter for embedding models+ if "type" in model and model["type"] not in ["embedding"]:+ # print(f"Skipping model w/ modality {model_type}:\n{model}")+ continue++ else:+ # For other providers we should skip by default, since we don't want to assume embeddings are supported+ continue++ configs.append(+ EmbeddingConfig(+ embedding_model=model_name,+ embedding_endpoint_type=self.provider_type,+ embedding_endpoint=self.base_url,+ embedding_dim=context_window_size,+ embedding_chunk_size=DEFAULT_EMBEDDING_CHUNK_SIZE,+ handle=self.get_handle(model, is_embedding=True),+ )+ )++ return configsdef get_model_context_window_size(self, model_name: str):if model_name in LLM_MAX_TOKENS:diff --git a/letta/server/rest_api/interface.py b/letta/server/rest_api/interface.pyindex 9a89f9074..66cc77a91 100644--- a/letta/server/rest_api/interface.py+++ b/letta/server/rest_api/interface.py@@ -482,6 +482,10 @@ class StreamingServerInterface(AgentChunkStreamingInterface):data: {"function_return": "None", "status": "success", "date": "2024-02-29T06:07:50.847262+00:00"}"""+ if not chunk.choices or len(chunk.choices) == 0:+ warnings.warn(f"No choices in chunk: {chunk}")+ return None+choice = chunk.choices[0]message_delta = choice.deltaotid = Message.generate_otid_from_id(message_id, message_index)diff --git a/letta/server/server.py b/letta/server/server.pyindex 5bd8a4b9d..54c21ca6e 100644--- a/letta/server/server.py+++ b/letta/server/server.py@@ -1219,6 +1219,9 @@ class SyncServer(Server):try:llm_models.extend(provider.list_llm_models())except Exception as e:+ import traceback++ traceback.print_exc()warnings.warn(f"An error occurred while listing LLM models for provider {provider}: {e}")llm_models.extend(self.get_local_llm_configs())diff --git a/letta/settings.py b/letta/settings.pyindex 81551bacb..1daa2e88b 100644--- a/letta/settings.py+++ b/letta/settings.py@@ -2,7 +2,7 @@ import osfrom pathlib import Pathfrom typing import Optional-from pydantic import Field+from pydantic import AliasChoices, Fieldfrom pydantic_settings import BaseSettings, SettingsConfigDictfrom letta.local_llm.constants import DEFAULT_WRAPPER_NAME@@ -70,7 +70,13 @@ class ModelSettings(BaseSettings):# openaiopenai_api_key: Optional[str] = None- openai_api_base: str = "https://api.openai.com/v1"+ openai_api_base: str = Field(+ default="https://api.openai.com/v1",+ # NOTE: We previously used OPENAI_API_BASE, but this was deprecated in favor of OPENAI_BASE_URL+ # preferred first, fallback second+ # env=["OPENAI_BASE_URL", "OPENAI_API_BASE"], # pydantic-settings v2+ validation_alias=AliasChoices("OPENAI_BASE_URL", "OPENAI_API_BASE"), # pydantic-settings v1+ )# deepseekdeepseek_api_key: Optional[str] = None