Platform for stateful agents: AI with advanced memory that can learn and self-improve over time.
fix(core): init attempt at patching the extended thinking bug (#4331)
Charles Packer56452ae42616
diff --git a/letta/llm_api/anthropic_client.py b/letta/llm_api/anthropic_client.pyindex 97198cd72..8c33f1627 100644--- a/letta/llm_api/anthropic_client.py+++ b/letta/llm_api/anthropic_client.py@@ -10,6 +10,7 @@ from anthropic.types.beta.message_create_params import MessageCreateParamsNonStrfrom anthropic.types.beta.messages import BetaMessageBatchfrom anthropic.types.beta.messages.batch_create_params import Request+from letta.constants import FUNC_FAILED_HEARTBEAT_MESSAGE, REQ_HEARTBEAT_MESSAGEfrom letta.errors import (ContextWindowExceededError,ErrorCode,@@ -59,7 +60,12 @@ class AnthropicClient(LLMClientBase):@trace_methodasync def request_async(self, request_data: dict, llm_config: LLMConfig) -> dict:client = await self._get_anthropic_client_async(llm_config, async_client=True)- response = await client.beta.messages.create(**request_data)++ if llm_config.enable_reasoner:+ response = await client.beta.messages.create(**request_data, betas=["interleaved-thinking-2025-05-14"])+ else:+ response = await client.beta.messages.create(**request_data)+return response.model_dump()@trace_method@@ -72,6 +78,11 @@ class AnthropicClient(LLMClientBase):# See: https://docs.anthropic.com/en/docs/build-with-claude/tool-use/fine-grained-streamingbetas = ["fine-grained-tool-streaming-2025-05-14"]+ # If extended thinking, turn on interleaved header+ # https://docs.anthropic.com/en/docs/build-with-claude/extended-thinking#interleaved-thinking+ if llm_config.enable_reasoner:+ betas.append("interleaved-thinking-2025-05-14")+return await client.beta.messages.create(**request_data, betas=betas)@trace_method@@ -270,6 +281,10 @@ class AnthropicClient(LLMClientBase):# Handle alternating messagesdata["messages"] = merge_tool_results_into_user_messages(data["messages"])+ # Strip heartbeat pings if extended thinking+ if llm_config.enable_reasoner:+ data["messages"] = merge_heartbeats_into_tool_responses(data["messages"])+# Prefix fill# https://docs.anthropic.com/en/api/messages#body-messages# NOTE: cannot prefill with tools for opus:@@ -615,6 +630,98 @@ def convert_tools_to_anthropic_format(tools: List[OpenAITool]) -> List[dict]:return formatted_tools+def is_heartbeat(message: dict, is_ping: bool = False) -> bool:+ """Check if the message is an automated heartbeat ping"""++ if "role" not in message or message["role"] != "user" or "content" not in message:+ return False++ try:+ message_json = json.loads(message["content"])+ except:+ return False++ if "reason" not in message_json:+ return False++ if message_json["type"] != "heartbeat":+ return False++ if not is_ping:+ # Just checking if 'type': 'heartbeat'+ return True+ else:+ # Also checking if it's specifically a 'ping' style message+ # NOTE: this will not catch tool rule heartbeats+ if REQ_HEARTBEAT_MESSAGE in message_json["reason"] or FUNC_FAILED_HEARTBEAT_MESSAGE in message_json["reason"]:+ return True+ else:+ return False+++def merge_heartbeats_into_tool_responses(messages: List[dict]):+ """For extended thinking mode, we don't want anything other than tool responses in-between assistant actions++ Otherwise, the thinking will silently get dropped.++ NOTE: assumes merge_tool_results_into_user_messages has already been called+ """++ merged_messages = []++ # Loop through messages+ # For messages with role 'user' and len(content) > 1,+ # Check if content[0].type == 'tool_result'+ # If so, iterate over content[1:] and while content.type == 'text' and is_heartbeat(content.text),+ # merge into content[0].content++ for message in messages:+ if "role" not in message or "content" not in message:+ # Skip invalid messages+ merged_messages.append(message)+ continue++ if message["role"] == "user" and len(message["content"]) > 1:+ content_parts = message["content"]++ # If the first content part is a tool result, merge the heartbeat content into index 0 of the content+ # Two end cases:+ # 1. It was [tool_result, heartbeat], in which case merged result is [tool_result+heartbeat] (len 1)+ # 2. It was [tool_result, user_text], in which case it should be unchanged (len 2)+ if "type" in content_parts[0] and "content" in content_parts[0] and content_parts[0]["type"] == "tool_result":+ new_content_parts = [content_parts[0]]++ # If the first content part is a tool result, merge the heartbeat content into index 0 of the content+ for i, content_part in enumerate(content_parts[1:]):+ # If it's a heartbeat, add it to the merge+ if (+ content_part["type"] == "text"+ and "text" in content_part+ and is_heartbeat({"role": "user", "content": content_part["text"]})+ ):+ # NOTE: joining with a ','+ new_content_parts[0]["content"] += ", " + content_part["text"]++ # If it's not, break, and concat to finish+ else:+ # Append the rest directly, no merging of content strings+ new_content_parts.extend(content_parts[i + 1 :])+ break++ # Set the content_parts+ message["content"] = new_content_parts+ merged_messages.append(message)++ else:+ # Skip invalid messages parts+ merged_messages.append(message)+ continue+ else:+ merged_messages.append(message)++ return merged_messages++def merge_tool_results_into_user_messages(messages: List[dict]):"""Anthropic API doesn't allow role 'tool'->'user' sequences@@ -653,7 +760,7 @@ def merge_tool_results_into_user_messages(messages: List[dict]):if isinstance(next_message["content"], list)else [{"type": "text", "text": next_message["content"]}])- merged_content = current_content + next_content+ merged_content: list = current_content + next_contentcurrent_message["content"] = merged_contentelse:# Append the current message to result as it's completediff --git a/letta/llm_api/bedrock_client.py b/letta/llm_api/bedrock_client.pyindex c89b57ff9..0d26e0f58 100644--- a/letta/llm_api/bedrock_client.py+++ b/letta/llm_api/bedrock_client.py@@ -27,7 +27,7 @@ class BedrockClient(AnthropicClient):llm_config.provider_name,actor=self.actor,)- return override_access_key_id, override_secret_access_key, override_default_regions+ return override_access_key_id, override_secret_access_key, override_default_region@trace_methodasync def _get_anthropic_client_async(diff --git a/tests/configs/llm_model_configs/claude-3-7-sonnet-extended.json b/tests/configs/llm_model_configs/claude-3-7-sonnet-extended.jsonindex f38ffb419..a7abf66a4 100644--- a/tests/configs/llm_model_configs/claude-3-7-sonnet-extended.json+++ b/tests/configs/llm_model_configs/claude-3-7-sonnet-extended.json@@ -4,7 +4,7 @@"model_endpoint": "https://api.anthropic.com/v1","model_wrapper": null,"context_window": 200000,- "put_inner_thoughts_in_kwargs": true,+ "put_inner_thoughts_in_kwargs": false,"enable_reasoner": true,"max_reasoning_tokens": 1024}diff --git a/tests/configs/llm_model_configs/claude-3-7-sonnet.json b/tests/configs/llm_model_configs/claude-3-7-sonnet.jsonnew file mode 100644index 000000000..3653a85d5--- /dev/null+++ b/tests/configs/llm_model_configs/claude-3-7-sonnet.json@@ -0,0 +1,9 @@+{+ "model": "claude-3-7-sonnet-20250219",+ "model_endpoint_type": "anthropic",+ "model_endpoint": "https://api.anthropic.com/v1",+ "model_wrapper": null,+ "context_window": 200000,+ "put_inner_thoughts_in_kwargs": true,+ "enable_reasoner": true+}diff --git a/tests/configs/llm_model_configs/claude-4-sonnet.json b/tests/configs/llm_model_configs/claude-4-sonnet.jsonnew file mode 100644index 000000000..a3b2a3c6d--- /dev/null+++ b/tests/configs/llm_model_configs/claude-4-sonnet.json@@ -0,0 +1,9 @@+{+ "model": "claude-sonnet-4-20250514",+ "model_endpoint_type": "anthropic",+ "model_endpoint": "https://api.anthropic.com/v1",+ "model_wrapper": null,+ "context_window": 200000,+ "put_inner_thoughts_in_kwargs": true,+ "enable_reasoner": true+}diff --git a/tests/integration_test_send_message.py b/tests/integration_test_send_message.pyindex 35a50f1b4..82363d07a 100644--- a/tests/integration_test_send_message.py+++ b/tests/integration_test_send_message.py@@ -120,6 +120,26 @@ USER_MESSAGE_ROLL_DICE_LONG: List[MessageCreate] = [otid=USER_MESSAGE_OTID,)]+USER_MESSAGE_ROLL_DICE_LONG_THINKING: List[MessageCreate] = [+ MessageCreate(+ role="user",+ content=(+ "This is an automated test message. First, think long and hard about about why you're here, and your creator. "+ "Then, call the roll_dice tool with 16 sides. "+ "Once you've rolled the die, think deeply about the meaning of the roll to you (but don't tell me, just think these thoughts privately). "+ "Then, once you're done thinking, send me a very detailed, comprehensive message about the outcome, using send_message. "+ "Your response must be at least 800 characters long. Start by explaining what dice rolling represents in games and probability theory. "+ "Discuss the mathematical probability of getting each number on a 16-sided die (1/16 or 6.25% for each face). "+ "Explain how 16-sided dice are commonly used in tabletop role-playing games like Dungeons & Dragons. "+ "Describe the specific number you rolled and what it might mean in different gaming contexts. "+ "Discuss how this particular roll compares to the expected value (8.5) of a 16-sided die. "+ "Explain the concept of randomness and how true random number generation works. "+ "End with some interesting facts about polyhedral dice and their history in gaming. "+ "Remember, make your response detailed and at least 800 characters long."+ ),+ otid=USER_MESSAGE_OTID,+ )+]URL_IMAGE = "https://upload.wikimedia.org/wikipedia/commons/a/a7/Camponotus_flavomarginatus_ant.jpg"USER_MESSAGE_URL_IMAGE: List[MessageCreate] = [MessageCreate(@@ -153,8 +173,10 @@ all_configs = ["openai-o4-mini.json","azure-gpt-4o-mini.json","claude-4-sonnet-extended.json",+ "claude-4-sonnet.json","claude-3-5-sonnet.json","claude-3-7-sonnet-extended.json",+ "claude-3-7-sonnet.json","bedrock-claude-4-sonnet.json","gemini-1.5-pro.json","gemini-2.5-flash-vertex.json",@@ -323,7 +345,7 @@ def assert_tool_call_response(ReasoningMessage -> AssistantMessage."""expected_message_count = 7 if streaming or from_db else 5- assert len(messages) == expected_message_count+ assert len(messages) == expected_message_count, messagesindex = 0if from_db:@@ -748,10 +770,15 @@ def test_tool_call("""last_message = client.agents.messages.list(agent_id=agent_state.id, limit=1)agent_state = client.agents.modify(agent_id=agent_state.id, llm_config=llm_config)+ # Use the thinking prompt for Anthropic models with extended reasoning to ensure second reasoning step+ if llm_config.model_endpoint_type == "anthropic" and llm_config.enable_reasoner:+ messages_to_send = USER_MESSAGE_ROLL_DICE_LONG_THINKING+ else:+ messages_to_send = USER_MESSAGE_ROLL_DICEtry:response = client.agents.messages.create(agent_id=agent_state.id,- messages=USER_MESSAGE_ROLL_DICE,+ messages=messages_to_send,request_options={"timeout_in_seconds": 300},)except Exception as e:@@ -965,9 +992,14 @@ def test_step_streaming_tool_call(last_message = client.agents.messages.list(agent_id=agent_state.id, limit=1)agent_state = client.agents.modify(agent_id=agent_state.id, llm_config=llm_config)+ # Use the thinking prompt for Anthropic models with extended reasoning to ensure second reasoning step+ if llm_config.model_endpoint_type == "anthropic" and llm_config.enable_reasoner:+ messages_to_send = USER_MESSAGE_ROLL_DICE_LONG_THINKING+ else:+ messages_to_send = USER_MESSAGE_ROLL_DICEresponse = client.agents.messages.create_stream(agent_id=agent_state.id,- messages=USER_MESSAGE_ROLL_DICE,+ messages=messages_to_send,request_options={"timeout_in_seconds": 300},)messages = accumulate_chunks(list(response))@@ -1110,7 +1142,11 @@ def test_token_streaming_tool_call(agent_state = client.agents.modify(agent_id=agent_state.id, llm_config=llm_config)# Use longer message for Anthropic models to force chunkingif llm_config.model_endpoint_type == "anthropic":- messages_to_send = USER_MESSAGE_ROLL_DICE_LONG+ if llm_config.enable_reasoner:+ # Without asking the model to think, Anthropic might decide to not think for the second step post-roll+ messages_to_send = USER_MESSAGE_ROLL_DICE_LONG_THINKING+ else:+ messages_to_send = USER_MESSAGE_ROLL_DICE_LONGelse:messages_to_send = USER_MESSAGE_ROLL_DICEresponse = client.agents.messages.create_stream(@@ -1285,7 +1321,11 @@ def test_background_token_streaming_tool_call(agent_state = client.agents.modify(agent_id=agent_state.id, llm_config=llm_config)# Use longer message for Anthropic models to force chunkingif llm_config.model_endpoint_type == "anthropic":- messages_to_send = USER_MESSAGE_ROLL_DICE_LONG+ if llm_config.enable_reasoner:+ # Without asking the model to think, Anthropic might decide to not think for the second step post-roll+ messages_to_send = USER_MESSAGE_ROLL_DICE_LONG_THINKING+ else:+ messages_to_send = USER_MESSAGE_ROLL_DICE_LONGelse:messages_to_send = USER_MESSAGE_ROLL_DICEresponse = client.agents.messages.create_stream(@@ -1419,9 +1459,14 @@ def test_async_tool_call(last_message = client.agents.messages.list(agent_id=agent_state.id, limit=1)client.agents.modify(agent_id=agent_state.id, llm_config=llm_config)+ # Use the thinking prompt for Anthropic models with extended reasoning to ensure second reasoning step+ if llm_config.model_endpoint_type == "anthropic" and llm_config.enable_reasoner:+ messages_to_send = USER_MESSAGE_ROLL_DICE_LONG_THINKING+ else:+ messages_to_send = USER_MESSAGE_ROLL_DICErun = client.agents.messages.create_async(agent_id=agent_state.id,- messages=USER_MESSAGE_ROLL_DICE,+ messages=messages_to_send,request_options={"timeout_in_seconds": 300},)run = wait_for_run_completion(client, run.id)