Platform for stateful agents: AI with advanced memory that can learn and self-improve over time.
fix: patch o-series (#1699)
Charles Packere1fa7f0b40a9
diff --git a/letta/helpers/composio_helpers.py b/letta/helpers/composio_helpers.pyindex a3c518eca..2a0281e16 100644--- a/letta/helpers/composio_helpers.py+++ b/letta/helpers/composio_helpers.py@@ -10,7 +10,7 @@ def get_composio_api_key(actor: User, logger: Optional[Logger] = None) -> Optionapi_keys = SandboxConfigManager().list_sandbox_env_vars_by_key(key="COMPOSIO_API_KEY", actor=actor)if not api_keys:if logger:- logger.warning(f"No API keys found for Composio. Defaulting to the environment variable...")+ logger.debug(f"No API keys found for Composio. Defaulting to the environment variable...")if tool_settings.composio_api_key:return tool_settings.composio_api_keyelse:diff --git a/letta/llm_api/openai.py b/letta/llm_api/openai.pyindex ffb64a99c..428d7f11e 100644--- a/letta/llm_api/openai.py+++ b/letta/llm_api/openai.py@@ -5,6 +5,7 @@ import requestsfrom openai import OpenAIfrom letta.llm_api.helpers import add_inner_thoughts_to_functions, convert_to_structured_output, make_post_request+from letta.llm_api.openai_client import supports_parallel_tool_calling, supports_temperature_paramfrom letta.local_llm.constants import INNER_THOUGHTS_KWARG, INNER_THOUGHTS_KWARG_DESCRIPTION, INNER_THOUGHTS_KWARG_DESCRIPTION_GO_FIRSTfrom letta.local_llm.utils import num_tokens_from_functions, num_tokens_from_messagesfrom letta.log import get_logger@@ -135,7 +136,7 @@ def build_openai_chat_completions_request(tool_choice=tool_choice,user=str(user_id),max_completion_tokens=llm_config.max_tokens,- temperature=1.0 if llm_config.enable_reasoner else llm_config.temperature,+ temperature=llm_config.temperature if supports_temperature_param(model) else None,reasoning_effort=llm_config.reasoning_effort,)else:@@ -489,6 +490,7 @@ def prepare_openai_payload(chat_completion_request: ChatCompletionRequest):# except ValueError as e:# warnings.warn(f"Failed to convert tool function to structured output, tool={tool}, error={e}")- if "o3-mini" in chat_completion_request.model or "o1" in chat_completion_request.model:+ if not supports_parallel_tool_calling(chat_completion_request.model):data.pop("parallel_tool_calls", None)+return datadiff --git a/letta/llm_api/openai_client.py b/letta/llm_api/openai_client.pyindex 426069ab4..ada788caa 100644--- a/letta/llm_api/openai_client.py+++ b/letta/llm_api/openai_client.py@@ -34,6 +34,33 @@ from letta.settings import model_settingslogger = get_logger(__name__)+def is_openai_reasoning_model(model: str) -> bool:+ """Utility function to check if the model is a 'reasoner'"""++ # NOTE: needs to be updated with new model releases+ return model.startswith("o1") or model.startswith("o3")+++def supports_temperature_param(model: str) -> bool:+ """Certain OpenAI models don't support configuring the temperature.++ Example error: 400 - {'error': {'message': "Unsupported parameter: 'temperature' is not supported with this model.", 'type': 'invalid_request_error', 'param': 'temperature', 'code': 'unsupported_parameter'}}+ """+ if is_openai_reasoning_model(model):+ return False+ else:+ return True+++def supports_parallel_tool_calling(model: str) -> bool:+ """Certain OpenAI models don't support parallel tool calls."""++ if is_openai_reasoning_model(model):+ return False+ else:+ return True++class OpenAIClient(LLMClientBase):def _prepare_client_kwargs(self) -> dict:api_key = model_settings.openai_api_key or os.environ.get("OPENAI_API_KEY")@@ -66,7 +93,8 @@ class OpenAIClient(LLMClientBase):put_inner_thoughts_first=True,)- use_developer_message = llm_config.model.startswith("o1") or llm_config.model.startswith("o3") # o-series models+ use_developer_message = is_openai_reasoning_model(llm_config.model)+openai_message_list = [cast_message_to_subtype(m.to_openai_dict(@@ -103,7 +131,7 @@ class OpenAIClient(LLMClientBase):tool_choice=tool_choice,user=str(),max_completion_tokens=llm_config.max_tokens,- temperature=llm_config.temperature,+ temperature=llm_config.temperature if supports_temperature_param(model) else None,)if "inference.memgpt.ai" in llm_config.model_endpoint:@@ -160,6 +188,10 @@ class OpenAIClient(LLMClientBase):response=chat_completion_response, inner_thoughts_key=INNER_THOUGHTS_KWARG)+ # If we used a reasoning model, create a content part for the ommitted reasoning+ if is_openai_reasoning_model(self.llm_config.model):+ chat_completion_response.choices[0].message.ommitted_reasoning_content = True+return chat_completion_responsedef stream(self, request_data: dict) -> Stream[ChatCompletionChunk]:diff --git a/letta/schemas/letta_message_content.py b/letta/schemas/letta_message_content.pyindex 00ebfe788..400926989 100644--- a/letta/schemas/letta_message_content.py+++ b/letta/schemas/letta_message_content.py@@ -145,7 +145,8 @@ class OmittedReasoningContent(MessageContent):type: Literal[MessageContentType.omitted_reasoning] = Field(MessageContentType.omitted_reasoning, description="Indicates this is an omitted reasoning step.")- tokens: int = Field(..., description="The reasoning token count for intermediate reasoning content.")+ # NOTE: dropping because we don't track this kind of information for the other reasoning types+ # tokens: int = Field(..., description="The reasoning token count for intermediate reasoning content.")LettaMessageContentUnion = Annotated[diff --git a/letta/schemas/llm_config.py b/letta/schemas/llm_config.pyindex 77114ea3b..d6027a85e 100644--- a/letta/schemas/llm_config.py+++ b/letta/schemas/llm_config.py@@ -81,8 +81,11 @@ class LLMConfig(BaseModel):@model_validator(mode="before")@classmethoddef set_default_enable_reasoner(cls, values):- if any(openai_reasoner_model in values.get("model", "") for openai_reasoner_model in ["o3-mini", "o1"]):- values["enable_reasoner"] = True+ # NOTE: this is really only applicable for models that can toggle reasoning on-and-off, like 3.7+ # We can also use this field to identify if a model is a "reasoning" model (o1/o3, etc.) if we want+ # if any(openai_reasoner_model in values.get("model", "") for openai_reasoner_model in ["o3-mini", "o1"]):+ # values["enable_reasoner"] = True+ # values["put_inner_thoughts_in_kwargs"] = Falsereturn values@model_validator(mode="before")@@ -100,6 +103,13 @@ class LLMConfig(BaseModel):if values.get("put_inner_thoughts_in_kwargs") is None:values["put_inner_thoughts_in_kwargs"] = False if model in avoid_put_inner_thoughts_in_kwargs else True+ # For the o1/o3 series from OpenAI, set to False by default+ # We can set this flag to `true` if desired, which will enable "double-think"+ from letta.llm_api.openai_client import is_openai_reasoning_model++ if is_openai_reasoning_model(model):+ values["put_inner_thoughts_in_kwargs"] = False+return values@model_validator(mode="after")diff --git a/letta/schemas/message.py b/letta/schemas/message.pyindex dfc36fe2c..76d3dd05e 100644--- a/letta/schemas/message.py+++ b/letta/schemas/message.py@@ -31,6 +31,7 @@ from letta.schemas.letta_message import ()from letta.schemas.letta_message_content import (LettaMessageContentUnion,+ OmittedReasoningContent,ReasoningContent,RedactedReasoningContent,TextContent,@@ -295,6 +296,18 @@ class Message(BaseMessage):sender_id=self.sender_id,))+ elif isinstance(content_part, OmittedReasoningContent):+ # Special case for "hidden reasoning" models like o1/o3+ # NOTE: we also have to think about how to return this during streaming+ messages.append(+ HiddenReasoningMessage(+ id=self.id,+ date=self.created_at,+ state="omitted",+ name=self.name,+ otid=otid,+ )+ )else:warnings.warn(f"Unrecognized content part in assistant message: {content_part}")@@ -464,6 +477,10 @@ class Message(BaseMessage):data=openai_message_dict["redacted_reasoning_content"] if "redacted_reasoning_content" in openai_message_dict else None,),)+ if "omitted_reasoning_content" in openai_message_dict and openai_message_dict["omitted_reasoning_content"]:+ content.append(+ OmittedReasoningContent(),+ )# If we're going from deprecated function formif openai_message_dict["role"] == "function":diff --git a/letta/schemas/openai/chat_completion_response.py b/letta/schemas/openai/chat_completion_response.pyindex 920288ffe..c217f9736 100644--- a/letta/schemas/openai/chat_completion_response.py+++ b/letta/schemas/openai/chat_completion_response.py@@ -39,9 +39,10 @@ class Message(BaseModel):tool_calls: Optional[List[ToolCall]] = Nonerole: strfunction_call: Optional[FunctionCall] = None # Deprecated- reasoning_content: Optional[str] = None # Used in newer reasoning APIs+ reasoning_content: Optional[str] = None # Used in newer reasoning APIs, e.g. DeepSeekreasoning_content_signature: Optional[str] = None # NOTE: for Anthropicredacted_reasoning_content: Optional[str] = None # NOTE: for Anthropic+ ommitted_reasoning_content: bool = False # NOTE: for OpenAI o1/o3class Choice(BaseModel):@@ -52,16 +53,64 @@ class Choice(BaseModel):seed: Optional[int] = None # found in TogetherAI+class UsageStatisticsPromptTokenDetails(BaseModel):+ cached_tokens: int = 0+ # NOTE: OAI specific+ # audio_tokens: int = 0++ def __add__(self, other: "UsageStatisticsPromptTokenDetails") -> "UsageStatisticsPromptTokenDetails":+ return UsageStatisticsPromptTokenDetails(+ cached_tokens=self.cached_tokens + other.cached_tokens,+ )+++class UsageStatisticsCompletionTokenDetails(BaseModel):+ reasoning_tokens: int = 0+ # NOTE: OAI specific+ # audio_tokens: int = 0+ # accepted_prediction_tokens: int = 0+ # rejected_prediction_tokens: int = 0++ def __add__(self, other: "UsageStatisticsCompletionTokenDetails") -> "UsageStatisticsCompletionTokenDetails":+ return UsageStatisticsCompletionTokenDetails(+ reasoning_tokens=self.reasoning_tokens + other.reasoning_tokens,+ )++class UsageStatistics(BaseModel):completion_tokens: int = 0prompt_tokens: int = 0total_tokens: int = 0+ prompt_tokens_details: Optional[UsageStatisticsPromptTokenDetails] = None+ completion_tokens_details: Optional[UsageStatisticsCompletionTokenDetails] = None+def __add__(self, other: "UsageStatistics") -> "UsageStatistics":++ if self.prompt_tokens_details is None and other.prompt_tokens_details is None:+ total_prompt_tokens_details = None+ elif self.prompt_tokens_details is None:+ total_prompt_tokens_details = other.prompt_tokens_details+ elif other.prompt_tokens_details is None:+ total_prompt_tokens_details = self.prompt_tokens_details+ else:+ total_prompt_tokens_details = self.prompt_tokens_details + other.prompt_tokens_details++ if self.completion_tokens_details is None and other.completion_tokens_details is None:+ total_completion_tokens_details = None+ elif self.completion_tokens_details is None:+ total_completion_tokens_details = other.completion_tokens_details+ elif other.completion_tokens_details is None:+ total_completion_tokens_details = self.completion_tokens_details+ else:+ total_completion_tokens_details = self.completion_tokens_details + other.completion_tokens_details+return UsageStatistics(completion_tokens=self.completion_tokens + other.completion_tokens,prompt_tokens=self.prompt_tokens + other.prompt_tokens,total_tokens=self.total_tokens + other.total_tokens,+ prompt_tokens_details=total_prompt_tokens_details,+ completion_tokens_details=total_completion_tokens_details,)