Platform for stateful agents: AI with advanced memory that can learn and self-improve over time.
feat: Scrub inner thoughts from history on toggle (#3607)
Matthew Zhou272b36c63f6a
diff --git a/letta/agents/letta_agent.py b/letta/agents/letta_agent.pyindex bd02a261a..7d2f5b86f 100644--- a/letta/agents/letta_agent.py+++ b/letta/agents/letta_agent.py@@ -22,6 +22,7 @@ from letta.constants import DEFAULT_MAX_STEPS, NON_USER_MSG_PREFIXfrom letta.errors import ContextWindowExceededErrorfrom letta.helpers import ToolRulesSolverfrom letta.helpers.datetime_helpers import AsyncTimer, get_utc_time, get_utc_timestamp_ns, ns_to_ms+from letta.helpers.reasoning_helper import scrub_inner_thoughts_from_messagesfrom letta.helpers.tool_execution_helper import enable_strict_modefrom letta.interfaces.anthropic_streaming_interface import AnthropicStreamingInterfacefrom letta.interfaces.openai_streaming_interface import OpenAIStreamingInterface@@ -1169,6 +1170,9 @@ class LettaAgent(BaseAgent):tool_rules_solver=tool_rules_solver,)+ # scrub inner thoughts from messages if reasoning is completely disabled+ in_context_messages = scrub_inner_thoughts_from_messages(in_context_messages, agent_state.llm_config)+tools = [tfor t in agent_state.toolsdiff --git a/letta/helpers/reasoning_helper.py b/letta/helpers/reasoning_helper.pynew file mode 100644index 000000000..31f8b5727--- /dev/null+++ b/letta/helpers/reasoning_helper.py@@ -0,0 +1,48 @@+from typing import List++from letta.schemas.enums import MessageRole+from letta.schemas.letta_message_content import TextContent+from letta.schemas.llm_config import LLMConfig+from letta.schemas.message import Message+++def is_reasoning_completely_disabled(llm_config: LLMConfig) -> bool:+ """+ Check if reasoning is completely disabled by verifying all three conditions:+ - put_inner_thoughts_in_kwargs is False+ - enable_reasoner is False+ - max_reasoning_tokens is 0++ Args:+ llm_config: The LLM configuration to check++ Returns:+ True if reasoning is completely disabled, False otherwise+ """+ return llm_config.put_inner_thoughts_in_kwargs is False and llm_config.enable_reasoner is False and llm_config.max_reasoning_tokens == 0+++def scrub_inner_thoughts_from_messages(messages: List[Message], llm_config: LLMConfig) -> List[Message]:+ """+ Remove inner thoughts (reasoning text) from assistant messages when reasoning is completely disabled.+ This makes the LLM think reasoning was never enabled by presenting clean message history.++ Args:+ messages: List of messages to potentially scrub+ llm_config: The LLM configuration to check++ Returns:+ The message list with inner thoughts removed if reasoning is disabled, otherwise unchanged+ """+ # early return if reasoning is not completely disabled+ if not is_reasoning_completely_disabled(llm_config):+ return messages++ # process messages to remove inner thoughts from assistant messages+ for message in messages:+ if message.role == MessageRole.assistant and message.content and message.tool_calls:+ # remove text content from assistant messages that also have tool calls+ # keep only non-text content (if any)+ message.content = [content for content in message.content if not isinstance(content, TextContent)]++ return messagesdiff --git a/tests/integration_test_send_message.py b/tests/integration_test_send_message.pyindex b8c3b2451..b27f8b105 100644--- a/tests/integration_test_send_message.py+++ b/tests/integration_test_send_message.py@@ -12,7 +12,7 @@ import httpximport pytestimport requestsfrom dotenv import load_dotenv-from letta_client import AsyncLetta, Letta, MessageCreate, Run+from letta_client import AsyncLetta, Letta, LettaRequest, MessageCreate, Runfrom letta_client.core.api_error import ApiErrorfrom letta_client.types import (AssistantMessage,@@ -30,6 +30,7 @@ from letta_client.types import (UserMessage,)+from letta.helpers.reasoning_helper import is_reasoning_completely_disabledfrom letta.llm_api.openai_client import is_openai_reasoning_modelfrom letta.log import get_loggerfrom letta.schemas.agent import AgentState@@ -71,6 +72,13 @@ USER_MESSAGE_FORCE_REPLY: List[MessageCreate] = [otid=USER_MESSAGE_OTID,)]+USER_MESSAGE_GREETING: List[MessageCreate] = [+ MessageCreate(+ role="user",+ content=f"Hi!",+ otid=USER_MESSAGE_OTID,+ )+]USER_MESSAGE_ROLL_DICE: List[MessageCreate] = [MessageCreate(role="user",@@ -330,6 +338,99 @@ def assert_tool_call_response(assert messages[index].step_count > 0+def validate_openai_format_scrubbing(messages: List[Dict[str, Any]]) -> None:+ """+ Validate that OpenAI format assistant messages with tool calls have no inner thoughts content.++ Args:+ messages: List of message dictionaries in OpenAI format+ """+ assistant_messages_with_tools = []++ for msg in messages:+ if msg.get("role") == "assistant" and msg.get("tool_calls"):+ assistant_messages_with_tools.append(msg)++ # There should be at least one assistant message with tool calls+ assert len(assistant_messages_with_tools) > 0, "Expected at least one OpenAI assistant message with tool calls"++ # Check that assistant messages with tool calls have no text content (inner thoughts scrubbed)+ for msg in assistant_messages_with_tools:+ if "content" in msg:+ content = msg["content"]+ assert content is None+++def validate_anthropic_format_scrubbing(messages: List[Dict[str, Any]]) -> None:+ """+ Validate that Anthropic/Claude format assistant messages with tool_use have no <thinking> tags.++ Args:+ messages: List of message dictionaries in Anthropic format+ """+ claude_assistant_messages_with_tools = []++ for msg in messages:+ if (+ msg.get("role") == "assistant"+ and isinstance(msg.get("content"), list)+ and any(item.get("type") == "tool_use" for item in msg.get("content", []))+ ):+ claude_assistant_messages_with_tools.append(msg)++ # There should be at least one Claude assistant message with tool_use+ assert len(claude_assistant_messages_with_tools) > 0, "Expected at least one Claude assistant message with tool_use"++ # Check Claude format messages specifically+ for msg in claude_assistant_messages_with_tools:+ content_list = msg["content"]++ # Strict validation: assistant messages with tool_use should have NO text content items at all+ text_items = [item for item in content_list if item.get("type") == "text"]+ assert len(text_items) == 0, (+ f"Found {len(text_items)} text content item(s) in Claude assistant message with tool_use. "+ f"When reasoning is disabled, there should be NO text items. "+ f"Text items found: {[item.get('text', '') for item in text_items]}"+ )++ # Verify that the message only contains tool_use items+ tool_use_items = [item for item in content_list if item.get("type") == "tool_use"]+ assert len(tool_use_items) > 0, "Assistant message should have at least one tool_use item"+ assert len(content_list) == len(tool_use_items), (+ f"Assistant message should ONLY contain tool_use items when reasoning is disabled. "+ f"Found {len(content_list)} total items but only {len(tool_use_items)} are tool_use items."+ )+++def validate_google_format_scrubbing(contents: List[Dict[str, Any]]) -> None:+ """+ Validate that Google/Gemini format model messages with functionCall have no thinking field.++ Args:+ contents: List of content dictionaries in Google format (uses 'contents' instead of 'messages')+ """+ model_messages_with_function_calls = []++ for content in contents:+ if content.get("role") == "model" and isinstance(content.get("parts"), list):+ for part in content["parts"]:+ if "functionCall" in part:+ model_messages_with_function_calls.append(part)++ # There should be at least one model message with functionCall+ assert len(model_messages_with_function_calls) > 0, "Expected at least one Google model message with functionCall"++ # Check Google format messages specifically+ for part in model_messages_with_function_calls:+ function_call = part["functionCall"]+ args = function_call.get("args", {})++ # Assert that there is no 'thinking' field in the function call arguments+ assert (+ "thinking" not in args+ ), f"Found 'thinking' field in Google model functionCall args (inner thoughts not scrubbed): {args.get('thinking')}"++def assert_image_input_response(messages: List[Any],llm_config: LLMConfig,@@ -1583,3 +1684,70 @@ def test_inner_thoughts_false_non_reasoner_models_streaming(assert_greeting_no_reasoning_response(messages, streaming=True)messages_from_db = client.agents.messages.list(agent_id=agent_state.id, after=last_message[0].id)assert_greeting_no_reasoning_response(messages_from_db, from_db=True)+++ "llm_config",+ TESTED_LLM_CONFIGS,+ ids=[c.model for c in TESTED_LLM_CONFIGS],+)+def test_inner_thoughts_toggle_interleaved(+ disable_e2b_api_key: Any,+ client: Letta,+ agent_state: AgentState,+ llm_config: LLMConfig,+) -> None:+ # get the config filename+ config_filename = None+ for filename in filenames:+ config = get_llm_config(filename)+ if config.model_dump() == llm_config.model_dump():+ config_filename = filename+ break++ # skip if this is a reasoning model+ if not config_filename or config_filename in reasoning_configs:+ pytest.skip(f"Skipping test for reasoning model {llm_config.model}")++ # Only run on OpenAI, Anthropic, and Google models+ if llm_config.model_endpoint_type not in ["openai", "anthropic", "google_ai", "google_vertex"]:+ pytest.skip(f"Skipping `test_inner_thoughts_toggle_interleaved` for model endpoint type {llm_config.model_endpoint_type}")++ assert not is_reasoning_completely_disabled(llm_config), "Reasoning should be enabled"+ agent_state = client.agents.modify(agent_id=agent_state.id, llm_config=llm_config)++ # Send a message with inner thoughts+ client.agents.messages.create(+ agent_id=agent_state.id,+ messages=USER_MESSAGE_GREETING,+ )++ # create a new config with all reasoning fields turned off+ new_llm_config = llm_config.model_dump()+ new_llm_config["put_inner_thoughts_in_kwargs"] = False+ new_llm_config["enable_reasoner"] = False+ new_llm_config["max_reasoning_tokens"] = 0+ adjusted_llm_config = LLMConfig(**new_llm_config)+ agent_state = client.agents.modify(agent_id=agent_state.id, llm_config=adjusted_llm_config)++ # Preview the message payload of the next message+ response = client.agents.messages.preview_raw_payload(+ agent_id=agent_state.id,+ request=LettaRequest(messages=USER_MESSAGE_FORCE_REPLY),+ )++ # Test our helper functions+ assert is_reasoning_completely_disabled(adjusted_llm_config), "Reasoning should be completely disabled"++ # Verify that assistant messages with tool calls have been scrubbed of inner thoughts+ # Branch assertions based on model endpoint type+ if llm_config.model_endpoint_type == "openai":+ messages = response["messages"]+ validate_openai_format_scrubbing(messages)+ elif llm_config.model_endpoint_type == "anthropic":+ messages = response["messages"]+ validate_anthropic_format_scrubbing(messages)+ elif llm_config.model_endpoint_type in ["google_ai", "google_vertex"]:+ # Google uses 'contents' instead of 'messages'+ contents = response.get("contents", response.get("messages", []))+ validate_google_format_scrubbing(contents)