Skip to content

armgit / letta

Public Imported from GitHub
0

Platform for stateful agents: AI with advanced memory that can learn and self-improve over time.

fix(core): init attempt at patching the extended thinking bug (#4331)

Charles Packer56452ae42616
diff --git a/letta/llm_api/anthropic_client.py b/letta/llm_api/anthropic_client.py
index 97198cd72..8c33f1627 100644
--- a/letta/llm_api/anthropic_client.py
+++ b/letta/llm_api/anthropic_client.py
@@ -10,6 +10,7 @@ from anthropic.types.beta.message_create_params import MessageCreateParamsNonStr
from anthropic.types.beta.messages import BetaMessageBatch
from anthropic.types.beta.messages.batch_create_params import Request
+from letta.constants import FUNC_FAILED_HEARTBEAT_MESSAGE, REQ_HEARTBEAT_MESSAGE
from letta.errors import (
ContextWindowExceededError,
ErrorCode,
@@ -59,7 +60,12 @@ class AnthropicClient(LLMClientBase):
@trace_method
async def request_async(self, request_data: dict, llm_config: LLMConfig) -> dict:
client = await self._get_anthropic_client_async(llm_config, async_client=True)
- response = await client.beta.messages.create(**request_data)
+
+ if llm_config.enable_reasoner:
+ response = await client.beta.messages.create(**request_data, betas=["interleaved-thinking-2025-05-14"])
+ else:
+ response = await client.beta.messages.create(**request_data)
+
return response.model_dump()
@trace_method
@@ -72,6 +78,11 @@ class AnthropicClient(LLMClientBase):
# See: https://docs.anthropic.com/en/docs/build-with-claude/tool-use/fine-grained-streaming
betas = ["fine-grained-tool-streaming-2025-05-14"]
+ # If extended thinking, turn on interleaved header
+ # https://docs.anthropic.com/en/docs/build-with-claude/extended-thinking#interleaved-thinking
+ if llm_config.enable_reasoner:
+ betas.append("interleaved-thinking-2025-05-14")
+
return await client.beta.messages.create(**request_data, betas=betas)
@trace_method
@@ -270,6 +281,10 @@ class AnthropicClient(LLMClientBase):
# Handle alternating messages
data["messages"] = merge_tool_results_into_user_messages(data["messages"])
+ # Strip heartbeat pings if extended thinking
+ if llm_config.enable_reasoner:
+ data["messages"] = merge_heartbeats_into_tool_responses(data["messages"])
+
# Prefix fill
# https://docs.anthropic.com/en/api/messages#body-messages
# NOTE: cannot prefill with tools for opus:
@@ -615,6 +630,98 @@ def convert_tools_to_anthropic_format(tools: List[OpenAITool]) -> List[dict]:
return formatted_tools
+def is_heartbeat(message: dict, is_ping: bool = False) -> bool:
+ """Check if the message is an automated heartbeat ping"""
+
+ if "role" not in message or message["role"] != "user" or "content" not in message:
+ return False
+
+ try:
+ message_json = json.loads(message["content"])
+ except:
+ return False
+
+ if "reason" not in message_json:
+ return False
+
+ if message_json["type"] != "heartbeat":
+ return False
+
+ if not is_ping:
+ # Just checking if 'type': 'heartbeat'
+ return True
+ else:
+ # Also checking if it's specifically a 'ping' style message
+ # NOTE: this will not catch tool rule heartbeats
+ if REQ_HEARTBEAT_MESSAGE in message_json["reason"] or FUNC_FAILED_HEARTBEAT_MESSAGE in message_json["reason"]:
+ return True
+ else:
+ return False
+
+
+def merge_heartbeats_into_tool_responses(messages: List[dict]):
+ """For extended thinking mode, we don't want anything other than tool responses in-between assistant actions
+
+ Otherwise, the thinking will silently get dropped.
+
+ NOTE: assumes merge_tool_results_into_user_messages has already been called
+ """
+
+ merged_messages = []
+
+ # Loop through messages
+ # For messages with role 'user' and len(content) > 1,
+ # Check if content[0].type == 'tool_result'
+ # If so, iterate over content[1:] and while content.type == 'text' and is_heartbeat(content.text),
+ # merge into content[0].content
+
+ for message in messages:
+ if "role" not in message or "content" not in message:
+ # Skip invalid messages
+ merged_messages.append(message)
+ continue
+
+ if message["role"] == "user" and len(message["content"]) > 1:
+ content_parts = message["content"]
+
+ # If the first content part is a tool result, merge the heartbeat content into index 0 of the content
+ # Two end cases:
+ # 1. It was [tool_result, heartbeat], in which case merged result is [tool_result+heartbeat] (len 1)
+ # 2. It was [tool_result, user_text], in which case it should be unchanged (len 2)
+ if "type" in content_parts[0] and "content" in content_parts[0] and content_parts[0]["type"] == "tool_result":
+ new_content_parts = [content_parts[0]]
+
+ # If the first content part is a tool result, merge the heartbeat content into index 0 of the content
+ for i, content_part in enumerate(content_parts[1:]):
+ # If it's a heartbeat, add it to the merge
+ if (
+ content_part["type"] == "text"
+ and "text" in content_part
+ and is_heartbeat({"role": "user", "content": content_part["text"]})
+ ):
+ # NOTE: joining with a ','
+ new_content_parts[0]["content"] += ", " + content_part["text"]
+
+ # If it's not, break, and concat to finish
+ else:
+ # Append the rest directly, no merging of content strings
+ new_content_parts.extend(content_parts[i + 1 :])
+ break
+
+ # Set the content_parts
+ message["content"] = new_content_parts
+ merged_messages.append(message)
+
+ else:
+ # Skip invalid messages parts
+ merged_messages.append(message)
+ continue
+ else:
+ merged_messages.append(message)
+
+ return merged_messages
+
+
def merge_tool_results_into_user_messages(messages: List[dict]):
"""Anthropic API doesn't allow role 'tool'->'user' sequences
@@ -653,7 +760,7 @@ def merge_tool_results_into_user_messages(messages: List[dict]):
if isinstance(next_message["content"], list)
else [{"type": "text", "text": next_message["content"]}]
)
- merged_content = current_content + next_content
+ merged_content: list = current_content + next_content
current_message["content"] = merged_content
else:
# Append the current message to result as it's complete
diff --git a/letta/llm_api/bedrock_client.py b/letta/llm_api/bedrock_client.py
index c89b57ff9..0d26e0f58 100644
--- a/letta/llm_api/bedrock_client.py
+++ b/letta/llm_api/bedrock_client.py
@@ -27,7 +27,7 @@ class BedrockClient(AnthropicClient):
llm_config.provider_name,
actor=self.actor,
)
- return override_access_key_id, override_secret_access_key, override_default_regions
+ return override_access_key_id, override_secret_access_key, override_default_region
@trace_method
async def _get_anthropic_client_async(
diff --git a/tests/configs/llm_model_configs/claude-3-7-sonnet-extended.json b/tests/configs/llm_model_configs/claude-3-7-sonnet-extended.json
index f38ffb419..a7abf66a4 100644
--- a/tests/configs/llm_model_configs/claude-3-7-sonnet-extended.json
+++ b/tests/configs/llm_model_configs/claude-3-7-sonnet-extended.json
@@ -4,7 +4,7 @@
"model_endpoint": "https://api.anthropic.com/v1",
"model_wrapper": null,
"context_window": 200000,
- "put_inner_thoughts_in_kwargs": true,
+ "put_inner_thoughts_in_kwargs": false,
"enable_reasoner": true,
"max_reasoning_tokens": 1024
}
diff --git a/tests/configs/llm_model_configs/claude-3-7-sonnet.json b/tests/configs/llm_model_configs/claude-3-7-sonnet.json
new file mode 100644
index 000000000..3653a85d5
--- /dev/null
+++ b/tests/configs/llm_model_configs/claude-3-7-sonnet.json
@@ -0,0 +1,9 @@
+{
+ "model": "claude-3-7-sonnet-20250219",
+ "model_endpoint_type": "anthropic",
+ "model_endpoint": "https://api.anthropic.com/v1",
+ "model_wrapper": null,
+ "context_window": 200000,
+ "put_inner_thoughts_in_kwargs": true,
+ "enable_reasoner": true
+}
diff --git a/tests/configs/llm_model_configs/claude-4-sonnet.json b/tests/configs/llm_model_configs/claude-4-sonnet.json
new file mode 100644
index 000000000..a3b2a3c6d
--- /dev/null
+++ b/tests/configs/llm_model_configs/claude-4-sonnet.json
@@ -0,0 +1,9 @@
+{
+ "model": "claude-sonnet-4-20250514",
+ "model_endpoint_type": "anthropic",
+ "model_endpoint": "https://api.anthropic.com/v1",
+ "model_wrapper": null,
+ "context_window": 200000,
+ "put_inner_thoughts_in_kwargs": true,
+ "enable_reasoner": true
+}
diff --git a/tests/integration_test_send_message.py b/tests/integration_test_send_message.py
index 35a50f1b4..82363d07a 100644
--- a/tests/integration_test_send_message.py
+++ b/tests/integration_test_send_message.py
@@ -120,6 +120,26 @@ USER_MESSAGE_ROLL_DICE_LONG: List[MessageCreate] = [
otid=USER_MESSAGE_OTID,
)
]
+USER_MESSAGE_ROLL_DICE_LONG_THINKING: List[MessageCreate] = [
+ MessageCreate(
+ role="user",
+ content=(
+ "This is an automated test message. First, think long and hard about about why you're here, and your creator. "
+ "Then, call the roll_dice tool with 16 sides. "
+ "Once you've rolled the die, think deeply about the meaning of the roll to you (but don't tell me, just think these thoughts privately). "
+ "Then, once you're done thinking, send me a very detailed, comprehensive message about the outcome, using send_message. "
+ "Your response must be at least 800 characters long. Start by explaining what dice rolling represents in games and probability theory. "
+ "Discuss the mathematical probability of getting each number on a 16-sided die (1/16 or 6.25% for each face). "
+ "Explain how 16-sided dice are commonly used in tabletop role-playing games like Dungeons & Dragons. "
+ "Describe the specific number you rolled and what it might mean in different gaming contexts. "
+ "Discuss how this particular roll compares to the expected value (8.5) of a 16-sided die. "
+ "Explain the concept of randomness and how true random number generation works. "
+ "End with some interesting facts about polyhedral dice and their history in gaming. "
+ "Remember, make your response detailed and at least 800 characters long."
+ ),
+ otid=USER_MESSAGE_OTID,
+ )
+]
URL_IMAGE = "https://upload.wikimedia.org/wikipedia/commons/a/a7/Camponotus_flavomarginatus_ant.jpg"
USER_MESSAGE_URL_IMAGE: List[MessageCreate] = [
MessageCreate(
@@ -153,8 +173,10 @@ all_configs = [
"openai-o4-mini.json",
"azure-gpt-4o-mini.json",
"claude-4-sonnet-extended.json",
+ "claude-4-sonnet.json",
"claude-3-5-sonnet.json",
"claude-3-7-sonnet-extended.json",
+ "claude-3-7-sonnet.json",
"bedrock-claude-4-sonnet.json",
"gemini-1.5-pro.json",
"gemini-2.5-flash-vertex.json",
@@ -323,7 +345,7 @@ def assert_tool_call_response(
ReasoningMessage -> AssistantMessage.
"""
expected_message_count = 7 if streaming or from_db else 5
- assert len(messages) == expected_message_count
+ assert len(messages) == expected_message_count, messages
index = 0
if from_db:
@@ -748,10 +770,15 @@ def test_tool_call(
"""
last_message = client.agents.messages.list(agent_id=agent_state.id, limit=1)
agent_state = client.agents.modify(agent_id=agent_state.id, llm_config=llm_config)
+ # Use the thinking prompt for Anthropic models with extended reasoning to ensure second reasoning step
+ if llm_config.model_endpoint_type == "anthropic" and llm_config.enable_reasoner:
+ messages_to_send = USER_MESSAGE_ROLL_DICE_LONG_THINKING
+ else:
+ messages_to_send = USER_MESSAGE_ROLL_DICE
try:
response = client.agents.messages.create(
agent_id=agent_state.id,
- messages=USER_MESSAGE_ROLL_DICE,
+ messages=messages_to_send,
request_options={"timeout_in_seconds": 300},
)
except Exception as e:
@@ -965,9 +992,14 @@ def test_step_streaming_tool_call(
last_message = client.agents.messages.list(agent_id=agent_state.id, limit=1)
agent_state = client.agents.modify(agent_id=agent_state.id, llm_config=llm_config)
+ # Use the thinking prompt for Anthropic models with extended reasoning to ensure second reasoning step
+ if llm_config.model_endpoint_type == "anthropic" and llm_config.enable_reasoner:
+ messages_to_send = USER_MESSAGE_ROLL_DICE_LONG_THINKING
+ else:
+ messages_to_send = USER_MESSAGE_ROLL_DICE
response = client.agents.messages.create_stream(
agent_id=agent_state.id,
- messages=USER_MESSAGE_ROLL_DICE,
+ messages=messages_to_send,
request_options={"timeout_in_seconds": 300},
)
messages = accumulate_chunks(list(response))
@@ -1110,7 +1142,11 @@ def test_token_streaming_tool_call(
agent_state = client.agents.modify(agent_id=agent_state.id, llm_config=llm_config)
# Use longer message for Anthropic models to force chunking
if llm_config.model_endpoint_type == "anthropic":
- messages_to_send = USER_MESSAGE_ROLL_DICE_LONG
+ if llm_config.enable_reasoner:
+ # Without asking the model to think, Anthropic might decide to not think for the second step post-roll
+ messages_to_send = USER_MESSAGE_ROLL_DICE_LONG_THINKING
+ else:
+ messages_to_send = USER_MESSAGE_ROLL_DICE_LONG
else:
messages_to_send = USER_MESSAGE_ROLL_DICE
response = client.agents.messages.create_stream(
@@ -1285,7 +1321,11 @@ def test_background_token_streaming_tool_call(
agent_state = client.agents.modify(agent_id=agent_state.id, llm_config=llm_config)
# Use longer message for Anthropic models to force chunking
if llm_config.model_endpoint_type == "anthropic":
- messages_to_send = USER_MESSAGE_ROLL_DICE_LONG
+ if llm_config.enable_reasoner:
+ # Without asking the model to think, Anthropic might decide to not think for the second step post-roll
+ messages_to_send = USER_MESSAGE_ROLL_DICE_LONG_THINKING
+ else:
+ messages_to_send = USER_MESSAGE_ROLL_DICE_LONG
else:
messages_to_send = USER_MESSAGE_ROLL_DICE
response = client.agents.messages.create_stream(
@@ -1419,9 +1459,14 @@ def test_async_tool_call(
last_message = client.agents.messages.list(agent_id=agent_state.id, limit=1)
client.agents.modify(agent_id=agent_state.id, llm_config=llm_config)
+ # Use the thinking prompt for Anthropic models with extended reasoning to ensure second reasoning step
+ if llm_config.model_endpoint_type == "anthropic" and llm_config.enable_reasoner:
+ messages_to_send = USER_MESSAGE_ROLL_DICE_LONG_THINKING
+ else:
+ messages_to_send = USER_MESSAGE_ROLL_DICE
run = client.agents.messages.create_async(
agent_id=agent_state.id,
- messages=USER_MESSAGE_ROLL_DICE,
+ messages=messages_to_send,
request_options={"timeout_in_seconds": 300},
)
run = wait_for_run_completion(client, run.id)