Platform for stateful agents: AI with advanced memory that can learn and self-improve over time.
feat: handle flaky reasoning in v2 tests (#5133)
cthomas89321ff29a2b
diff --git a/letta/llm_api/google_vertex_client.py b/letta/llm_api/google_vertex_client.pyindex 687ac9373..de7c4d812 100644--- a/letta/llm_api/google_vertex_client.py+++ b/letta/llm_api/google_vertex_client.py@@ -563,9 +563,9 @@ class GoogleVertexClient(LLMClientBase):)else:openai_response_message.content = inner_thoughts- if response_message.thought_signature:- thought_signature = base64.b64encode(response_message.thought_signature).decode("utf-8")- openai_response_message.reasoning_content_signature = thought_signature+ if response_message.thought_signature:+ thought_signature = base64.b64encode(response_message.thought_signature).decode("utf-8")+ openai_response_message.reasoning_content_signature = thought_signature# Google AI API uses different finish reason strings than OpenAI# OpenAI: 'stop', 'length', 'function_call', 'content_filter', nulldiff --git a/letta/schemas/message.py b/letta/schemas/message.pyindex 1d24e6de2..7334f9d10 100644--- a/letta/schemas/message.py+++ b/letta/schemas/message.py@@ -302,12 +302,6 @@ class Message(BaseMessage):if self.role == MessageRole.assistant:if self.content:messages.extend(self._convert_reasoning_messages(text_is_assistant_message=text_is_assistant_message))- for i in range(len(messages) - 1, -1, -1):- if i > 0 and messages[i].message_type == messages[i - 1].message_type:- if messages[i].message_type == MessageType.reasoning_message:- messages[i - 1].reasoning = messages[i - 1].reasoning + messages.pop(i).reasoning- elif messages[i].message_type == MessageType.assistant_message:- messages[i - 1].content = messages[i - 1].content + messages.pop(i).contentif self.tool_calls is not None:messages.extend(@@ -361,11 +355,29 @@ class Message(BaseMessage):if isinstance(content_part, TextContent):if text_is_assistant_message:# .content is assistant message+ if messages and messages[-1].message_type == MessageType.assistant_message:+ messages[-1].content += content_part.text+ else:+ messages.append(+ AssistantMessage(+ id=self.id,+ date=self.created_at,+ content=content_part.text,+ name=self.name,+ otid=otid,+ sender_id=self.sender_id,+ step_id=self.step_id,+ is_err=self.is_err,+ run_id=self.run_id,+ )+ )+ else:+ # .content is COTmessages.append(- AssistantMessage(+ ReasoningMessage(id=self.id,date=self.created_at,- content=content_part.text,+ reasoning=content_part.text,name=self.name,otid=otid,sender_id=self.sender_id,@@ -374,39 +386,27 @@ class Message(BaseMessage):run_id=self.run_id,))++ elif isinstance(content_part, ReasoningContent):+ # "native" COT+ if messages and messages[-1].message_type == MessageType.reasoning_message:+ messages[-1].reasoning += content_part.reasoningelse:- # .content is COTmessages.append(ReasoningMessage(id=self.id,date=self.created_at,- reasoning=content_part.text,+ reasoning=content_part.reasoning,+ source="reasoner_model", # TODO do we want to tag like this?+ signature=content_part.signature,name=self.name,otid=otid,- sender_id=self.sender_id,step_id=self.step_id,is_err=self.is_err,run_id=self.run_id,))- elif isinstance(content_part, ReasoningContent):- # "native" COT- messages.append(- ReasoningMessage(- id=self.id,- date=self.created_at,- reasoning=content_part.reasoning,- source="reasoner_model", # TODO do we want to tag like this?- signature=content_part.signature,- name=self.name,- otid=otid,- step_id=self.step_id,- is_err=self.is_err,- run_id=self.run_id,- )- )-elif isinstance(content_part, SummarizedReasoningContent):# TODO remove the cast and just return the native typecasted_content_part = content_part.to_reasoning_content()@@ -1409,7 +1409,7 @@ class Message(BaseMessage):"name": content.name,"args": content.input,},- "thought_signature": content.signature,+ # "thought_signature": content.signature,})else:diff --git a/tests/integration_test_send_message_v2.py b/tests/integration_test_send_message_v2.pyindex eca630869..1ea9d9aca 100644--- a/tests/integration_test_send_message_v2.py+++ b/tests/integration_test_send_message_v2.py@@ -6,7 +6,7 @@ import timeimport uuidfrom contextlib import contextmanagerfrom http.server import BaseHTTPRequestHandler, HTTPServer-from typing import Any, Dict, List+from typing import Any, Dict, List, Tuplefrom unittest.mock import patchimport httpx@@ -115,19 +115,10 @@ def assert_greeting_response(msg for msg in messages if not (isinstance(msg, LettaPing) or (hasattr(msg, "message_type") and msg.message_type == "ping"))]- expected_message_count = get_expected_message_count(llm_config, streaming=streaming, from_db=from_db)- try:- assert len(messages) == expected_message_count- except:- # Reasoning summary in responses API when effort is high is still flaky, so don't throw if missing- if (- LLMConfig.is_openai_reasoning_model(llm_config)- or LLMConfig.is_google_vertex_reasoning_model(llm_config)- or LLMConfig.is_google_ai_reasoning_model(llm_config)- ):- assert len(messages) == expected_message_count - 1- else:- raise+ expected_message_count_min, expected_message_count_max = get_expected_message_count_range(+ llm_config, streaming=streaming, from_db=from_db+ )+ assert expected_message_count_min <= len(messages) <= expected_message_count_max# User message if loaded from dbindex = 0@@ -139,26 +130,14 @@ def assert_greeting_response(# Reasoning message if reasoning enabledotid_suffix = 0try:- if (- (LLMConfig.is_openai_reasoning_model(llm_config) and llm_config.reasoning_effort == "high")- or LLMConfig.is_anthropic_reasoning_model(llm_config)- or LLMConfig.is_google_vertex_reasoning_model(llm_config)- or LLMConfig.is_google_ai_reasoning_model(llm_config)- ):+ if is_reasoner_model(llm_config):assert isinstance(messages[index], ReasoningMessage)assert messages[index].otid and messages[index].otid[-1] == str(otid_suffix)index += 1otid_suffix += 1except:- # Reasoning summary in responses API when effort is high is still flaky, so don't throw if missing- if (- LLMConfig.is_openai_reasoning_model(llm_config)- or LLMConfig.is_google_vertex_reasoning_model(llm_config)- or LLMConfig.is_google_ai_reasoning_model(llm_config)- ):- pass- else:- raise+ # Reasoning is non-deterministic, so don't throw if missing+ pass# Assistant messageassert isinstance(messages[index], AssistantMessage)@@ -196,15 +175,10 @@ def assert_tool_call_response(msg for msg in messages if not (isinstance(msg, LettaPing) or (hasattr(msg, "message_type") and msg.message_type == "ping"))]- expected_message_count = get_expected_message_count(llm_config, tool_call=True, streaming=streaming, from_db=from_db)- try:- assert len(messages) == expected_message_count- except:- # Reasoning summary in responses API when effort is high is still flaky, so don't throw if missing- if LLMConfig.is_openai_reasoning_model(llm_config):- assert len(messages) == expected_message_count - 1- else:- raise+ expected_message_count_min, expected_message_count_max = get_expected_message_count_range(+ llm_config, tool_call=True, streaming=streaming, from_db=from_db+ )+ assert expected_message_count_min <= len(messages) <= expected_message_count_max# User message if loaded from dbindex = 0@@ -216,19 +190,14 @@ def assert_tool_call_response(# Reasoning message if reasoning enabledotid_suffix = 0try:- if (- LLMConfig.is_openai_reasoning_model(llm_config) and llm_config.reasoning_effort == "high"- ) or LLMConfig.is_anthropic_reasoning_model(llm_config):+ if is_reasoner_model(llm_config):assert isinstance(messages[index], ReasoningMessage)assert messages[index].otid and messages[index].otid[-1] == str(otid_suffix)index += 1otid_suffix += 1except:- # Reasoning summary in responses API when effort is high is still flaky, so don't throw if missing- if LLMConfig.is_openai_reasoning_model(llm_config):- pass- else:- raise+ # Reasoning is non-deterministic, so don't throw if missing+ pass# Assistant messageif llm_config.model_endpoint_type == "anthropic":@@ -248,6 +217,18 @@ def assert_tool_call_response(assert messages[index].otid and messages[index].otid[-1] == str(otid_suffix)index += 1+ # Reasoning message if reasoning enabled+ otid_suffix = 0+ try:+ if is_reasoner_model(llm_config):+ assert isinstance(messages[index], ReasoningMessage)+ assert messages[index].otid and messages[index].otid[-1] == str(otid_suffix)+ index += 1+ otid_suffix += 1+ except:+ # Reasoning is non-deterministic, so don't throw if missing+ pass+# Assistant messageassert isinstance(messages[index], AssistantMessage)assert messages[index].otid and messages[index].otid[-1] == str(otid_suffix)@@ -312,42 +293,41 @@ async def wait_for_run_completion(client: AsyncLetta, run_id: str, timeout: floatime.sleep(interval)-def get_expected_message_count(llm_config: LLMConfig, tool_call: bool = False, streaming: bool = False, from_db: bool = False) -> int:+def get_expected_message_count_range(+ llm_config: LLMConfig, tool_call: bool = False, streaming: bool = False, from_db: bool = False+) -> Tuple[int, int]:"""- Returns the expected number of messages for a given LLM configuration.+ Returns the expected range of number of messages for a given LLM configuration. Uses range to account for possible variations in the number of reasoning messages.Greeting:------------------------------------------------------------------------------------------------------------------------------------------------------------------| gpt-4o | gpt-o3 (med effort) | gpt-5 (high effort) | sonnet-3-5 | sonnet-3.7-thinking | flash-2.5-thinking || ------------------------ | ------------------------ | ------------------------ | ------------------------ | ------------------------ | ------------------------ |- | AssistantMessage | AssistantMessage | ReasoningMessage | AssistantMessage | ReasoningMessage | AssistantMessage |- | | | AssistantMessage | | AssistantMessage | |+ | AssistantMessage | AssistantMessage | ReasoningMessage | AssistantMessage | ReasoningMessage | ReasoningMessage |+ | | | AssistantMessage | | AssistantMessage | AssistantMessage |Tool Call:------------------------------------------------------------------------------------------------------------------------------------------------------------------| gpt-4o | gpt-o3 (med effort) | gpt-5 (high effort) | sonnet-3-5 | sonnet-3.7-thinking | flash-2.5-thinking || ------------------------ | ------------------------ | ------------------------ | ------------------------ | ------------------------ | ------------------------ |- | ToolCallMessage | ToolCallMessage | ReasoningMessage | AssistantMessage | ReasoningMessage | ToolCallMessage |- | ToolReturnMessage | ToolReturnMessage | ToolCallMessage | ToolCallMessage | AssistantMessage | ToolReturnMessage |- | AssistantMessage | AssistantMessage | ToolReturnMessage | ToolReturnMessage | ToolCallMessage | AssistantMessage |- | | | AssistantMessage | AssistantMessage | ToolReturnMessage | |- | | | | | AssistantMessage | |+ | ToolCallMessage | ToolCallMessage | ReasoningMessage | AssistantMessage | ReasoningMessage | ReasoningMessage |+ | ToolReturnMessage | ToolReturnMessage | ToolCallMessage | ToolCallMessage | AssistantMessage | ToolCallMessage |+ | AssistantMessage | AssistantMessage | ToolReturnMessage | ToolReturnMessage | ToolCallMessage | ToolReturnMessage |+ | | | ReasoningMessage | AssistantMessage | ToolReturnMessage | ReasoningMessage |+ | | | AssistantMessage | | AssistantMessage | AssistantMessage |"""- is_reasoner_model = (- (LLMConfig.is_openai_reasoning_model(llm_config) and llm_config.reasoning_effort == "high")- or LLMConfig.is_anthropic_reasoning_model(llm_config)- or LLMConfig.is_google_vertex_reasoning_model(llm_config)- or LLMConfig.is_google_ai_reasoning_model(llm_config)- )-# assistant messageexpected_message_count = 1+ expected_range = 0- if is_reasoner_model:+ if is_reasoner_model(llm_config):# reasoning message- expected_message_count += 1+ expected_range += 1+ if tool_call and not LLMConfig.is_anthropic_reasoning_model(llm_config):+ # reasoning message for additional turn, only for openai and google models+ expected_range += 1if tool_call:# tool call and tool return messages@@ -364,7 +344,16 @@ def get_expected_message_count(llm_config: LLMConfig, tool_call: bool = False, s# stop reason and usage statisticsexpected_message_count += 2- return expected_message_count+ return expected_message_count, expected_message_count + expected_range+++def is_reasoner_model(llm_config: LLMConfig) -> bool:+ return (+ (LLMConfig.is_openai_reasoning_model(llm_config) and llm_config.reasoning_effort == "high")+ or LLMConfig.is_anthropic_reasoning_model(llm_config)+ or LLMConfig.is_google_vertex_reasoning_model(llm_config)+ or LLMConfig.is_google_ai_reasoning_model(llm_config)+ )# ------------------------------