Platform for stateful agents: AI with advanced memory that can learn and self-improve over time.
feat: fix anthropic stream buffering issue (was a missing beta header) (#3933)
Charles Packerd5b62af62d5f
diff --git a/letta/llm_api/anthropic_client.py b/letta/llm_api/anthropic_client.pyindex 68a3feb1d..c8683e37e 100644--- a/letta/llm_api/anthropic_client.py+++ b/letta/llm_api/anthropic_client.py@@ -63,7 +63,13 @@ class AnthropicClient(LLMClientBase):async def stream_async(self, request_data: dict, llm_config: LLMConfig) -> AsyncStream[BetaRawMessageStreamEvent]:client = await self._get_anthropic_client_async(llm_config, async_client=True)request_data["stream"] = True- return await client.beta.messages.create(**request_data)++ # Add fine-grained tool streaming beta header for better streaming performance+ # This helps reduce buffering when streaming tool call parameters+ # See: https://docs.anthropic.com/en/docs/build-with-claude/tool-use/fine-grained-streaming+ betas = ["fine-grained-tool-streaming-2025-05-14"]++ return await client.beta.messages.create(**request_data, betas=betas)@trace_methodasync def send_llm_batch_request_async(diff --git a/tests/integration_test_send_message.py b/tests/integration_test_send_message.pyindex a8250e851..f2a0d68cc 100644--- a/tests/integration_test_send_message.py+++ b/tests/integration_test_send_message.py@@ -72,6 +72,23 @@ USER_MESSAGE_FORCE_REPLY: List[MessageCreate] = [otid=USER_MESSAGE_OTID,)]+USER_MESSAGE_LONG_RESPONSE: str = (+ "Teamwork makes the dream work. When people collaborate and combine their unique skills, perspectives, and experiences, they can achieve far more than any individual working alone. "+ "This synergy creates an environment where innovation flourishes, problems are solved more creatively, and goals are reached more efficiently. "+ "In a team setting, diverse viewpoints lead to better decision-making as different team members bring their unique backgrounds and expertise to the table. "+ "Communication becomes the backbone of success, allowing ideas to flow freely and ensuring everyone is aligned toward common objectives. "+ "Trust builds gradually as team members learn to rely on each other's strengths while supporting one another through challenges. "+ "The collective intelligence of a group often surpasses that of even the brightest individual, as collaboration sparks creativity and innovation. "+ "Successful teams celebrate victories together and learn from failures as a unit, creating a culture of continuous improvement. "+ "Together, we can overcome challenges that would be insurmountable alone, achieving extraordinary results through the power of collaboration."+)+USER_MESSAGE_FORCE_LONG_REPLY: List[MessageCreate] = [+ MessageCreate(+ role="user",+ content=f"This is an automated test message. Call the send_message tool with exactly this message: '{USER_MESSAGE_LONG_RESPONSE}'",+ otid=USER_MESSAGE_OTID,+ )+]USER_MESSAGE_GREETING: List[MessageCreate] = [MessageCreate(role="user",@@ -86,6 +103,23 @@ USER_MESSAGE_ROLL_DICE: List[MessageCreate] = [otid=USER_MESSAGE_OTID,)]+USER_MESSAGE_ROLL_DICE_LONG: List[MessageCreate] = [+ MessageCreate(+ role="user",+ content=(+ "This is an automated test message. Call the roll_dice tool with 16 sides and send me a very detailed, comprehensive message about the outcome. "+ "Your response must be at least 800 characters long. Start by explaining what dice rolling represents in games and probability theory. "+ "Discuss the mathematical probability of getting each number on a 16-sided die (1/16 or 6.25% for each face). "+ "Explain how 16-sided dice are commonly used in tabletop role-playing games like Dungeons & Dragons. "+ "Describe the specific number you rolled and what it might mean in different gaming contexts. "+ "Discuss how this particular roll compares to the expected value (8.5) of a 16-sided die. "+ "Explain the concept of randomness and how true random number generation works. "+ "End with some interesting facts about polyhedral dice and their history in gaming. "+ "Remember, make your response detailed and at least 800 characters long."+ ),+ otid=USER_MESSAGE_OTID,+ )+]URL_IMAGE = "https://upload.wikimedia.org/wikipedia/commons/a/a7/Camponotus_flavomarginatus_ant.jpg"USER_MESSAGE_URL_IMAGE: List[MessageCreate] = [MessageCreate(@@ -172,7 +206,8 @@ def assert_greeting_with_assistant_message_response(assert isinstance(messages[index], AssistantMessage)if not token_streaming:- assert USER_MESSAGE_RESPONSE in messages[index].content+ # Check for either short or long response+ assert USER_MESSAGE_RESPONSE in messages[index].content or USER_MESSAGE_LONG_RESPONSE in messages[index].contentassert messages[index].otid and messages[index].otid[-1] == "1"index += 1@@ -491,7 +526,7 @@ def accumulate_chunks(chunks: List[Any], verify_token_streaming: bool = False) -and verify_token_streamingand current_message.message_type in ["reasoning_message", "assistant_message", "tool_call_message"]):- assert chunk_count > 1, f"Expected more than one chunk for {current_message.message_type}"+ assert chunk_count > 1, f"Expected more than one chunk for {current_message.message_type}. Messages: {messages}"current_message = Nonechunk_count = 0if current_message is None:@@ -964,9 +999,14 @@ def test_token_streaming_greeting_with_assistant_message("""last_message = client.agents.messages.list(agent_id=agent_state.id, limit=1)agent_state = client.agents.modify(agent_id=agent_state.id, llm_config=llm_config)+ # Use longer message for Anthropic models to test if they stream in chunks+ if llm_config.model_endpoint_type == "anthropic":+ messages_to_send = USER_MESSAGE_FORCE_LONG_REPLY+ else:+ messages_to_send = USER_MESSAGE_FORCE_REPLYresponse = client.agents.messages.create_stream(agent_id=agent_state.id,- messages=USER_MESSAGE_FORCE_REPLY,+ messages=messages_to_send,stream_tokens=True,)messages = accumulate_chunks(@@ -994,9 +1034,14 @@ def test_token_streaming_greeting_without_assistant_message("""last_message = client.agents.messages.list(agent_id=agent_state.id, limit=1)agent_state = client.agents.modify(agent_id=agent_state.id, llm_config=llm_config)+ # Use longer message for Anthropic models to force chunking+ if llm_config.model_endpoint_type == "anthropic":+ messages_to_send = USER_MESSAGE_FORCE_LONG_REPLY+ else:+ messages_to_send = USER_MESSAGE_FORCE_REPLYresponse = client.agents.messages.create_stream(agent_id=agent_state.id,- messages=USER_MESSAGE_FORCE_REPLY,+ messages=messages_to_send,use_assistant_message=False,stream_tokens=True,)@@ -1037,9 +1082,14 @@ def test_token_streaming_tool_call(last_message = client.agents.messages.list(agent_id=agent_state.id, limit=1)agent_state = client.agents.modify(agent_id=agent_state.id, llm_config=llm_config)+ # Use longer message for Anthropic models to force chunking+ if llm_config.model_endpoint_type == "anthropic":+ messages_to_send = USER_MESSAGE_ROLL_DICE_LONG+ else:+ messages_to_send = USER_MESSAGE_ROLL_DICEresponse = client.agents.messages.create_stream(agent_id=agent_state.id,- messages=USER_MESSAGE_ROLL_DICE,+ messages=messages_to_send,stream_tokens=True,)messages = accumulate_chunks(diff --git a/tests/test_sonnet_nonnative_reasoning_buffering.py b/tests/test_sonnet_nonnative_reasoning_buffering.pynew file mode 100755index 000000000..a61254ca9--- /dev/null+++ b/tests/test_sonnet_nonnative_reasoning_buffering.py@@ -0,0 +1,308 @@+"""Test to verify streaming behavior of Anthropic models with and without native reasoning.++This test confirms:+1. Sonnet 3.5 (20241022) with non-native reasoning exhibits batch streaming (API limitation)+ - UPDATE: With fine-grained-tool-streaming beta header, this may improve+2. Sonnet 4 (20250514) with native reasoning should stream progressively+3. GPT-4.1 streams progressively as expected++Note: We've added the 'fine-grained-tool-streaming-2025-05-14' beta header to potentially+improve streaming performance with Anthropic models, especially for tool call parameters.+"""++import os+import time+from typing import List, Tuple++import pytest+from letta_client import Letta, MessageCreate++from tests.utils import wait_for_server+++def run_server():+ """Start the Letta server."""+ from dotenv import load_dotenv++ load_dotenv()+ from letta.server.rest_api.app import start_server++ print("Starting server...")+ start_server(debug=True)++[email protected](scope="module")+def client():+ """Create a Letta client for testing."""+ import threading++ # Get URL from environment or start server+ api_url = os.getenv("LETTA_API_URL")+ server_url = os.getenv("LETTA_SERVER_URL", "http://localhost:8283")+ if not os.getenv("LETTA_SERVER_URL"):+ print("Starting server thread")+ thread = threading.Thread(target=run_server, daemon=True)+ thread.start()+ wait_for_server(server_url)+ print("Running client tests with server:", server_url)++ # Override the base_url if the LETTA_API_URL is set+ base_url = api_url if api_url else server_url+ # create the Letta client+ yield Letta(base_url=base_url, token=None)+++def agent_factory(client: Letta):+ """Factory fixture to create agents with different models."""+ created_agents = []++ def _create_agent(model_name: str):+ """Create an agent with the specified model."""+ # Check for required API keys+ if "claude" in model_name.lower():+ anthropic_key = os.getenv("ANTHROPIC_API_KEY")+ if not anthropic_key:+ pytest.skip("ANTHROPIC_API_KEY not set, skipping Anthropic test")+ elif "gpt" in model_name.lower():+ openai_key = os.getenv("OPENAI_API_KEY")+ if not openai_key:+ pytest.skip("OPENAI_API_KEY not set, skipping OpenAI test")++ agent_state = client.agents.create(+ name=f"test_agent_{model_name.replace('/', '_').replace('.', '_')}",+ memory_blocks=[{"label": "human", "value": "Test user"}, {"label": "persona", "value": "You are a creative storyteller."}],+ model=model_name,+ embedding="letta/letta-free",+ )+ created_agents.append(agent_state)+ return agent_state++ yield _create_agent++ # Cleanup all created agents+ for agent_state in created_agents:+ try:+ client.agents.delete(agent_state.id)+ except:+ pass # Agent might have already been deleted+++def detect_burst_chunks(chunks: List[Tuple[float, any]], burst_threshold: float = 0.05) -> List[List[int]]:+ """+ Detect bursts of chunks arriving close together in time.++ Args:+ chunks: List of (timestamp, chunk) tuples+ burst_threshold: Maximum time difference (in seconds) to consider chunks as part of the same burst++ Returns:+ List of bursts, where each burst is a list of chunk indices+ """+ if not chunks:+ return []++ bursts = []+ current_burst = [0]++ for i in range(1, len(chunks)):+ time_diff = chunks[i][0] - chunks[i - 1][0]+ if time_diff <= burst_threshold:+ # Part of the same burst+ current_burst.append(i)+ else:+ # New burst+ if len(current_burst) > 1: # Only count as burst if more than 1 chunk+ bursts.append(current_burst)+ current_burst = [i]++ # Don't forget the last burst+ if len(current_burst) > 1:+ bursts.append(current_burst)++ return bursts+++ "model,expected_buffering",+ [+ ("anthropic/claude-3-5-sonnet-20241022", False), # With fine-grained streaming beta, should stream better+ ("anthropic/claude-sonnet-4-20250514", False), # Sonnet 4 should NOT show buffering (has native reasoning)+ ("openai/gpt-4.1", False), # GPT-4.1 should NOT show buffering (uses native reasoning)+ ],+)+def test_streaming_buffering_behavior(client: Letta, agent_factory, model: str, expected_buffering: bool):+ """+ Test streaming behavior for different models.++ With fine-grained-tool-streaming beta header:+ - Sonnet 3.5 (20241022) should now stream progressively (beta feature improves tool streaming)+ - Sonnet 4 (20250514) with native reasoning should stream progressively without buffering+ - GPT-4.1 should stream progressively without buffering+ """+ print(f"\n=== Testing Streaming Behavior for {model} ===")+ print(f"Expected buffering: {expected_buffering}")++ # Create agent with the specified model+ agent = agent_factory(model)++ # Send a message that should generate reasoning and tool calls+ # This prompt should trigger inner thoughts and then a response+ user_message = "Think step by step about what makes a good story, then write me a creative story about a toad named Ted. Make it exactly 3 paragraphs long."++ # Create the stream+ response_stream = client.agents.messages.create_stream(+ agent_id=agent.id, messages=[MessageCreate(role="user", content=user_message)], stream_tokens=True # Enable token streaming+ )++ # Collect chunks with timestamps+ chunks_with_time = []+ reasoning_chunks = []+ assistant_chunks = []+ tool_chunks = []++ start_time = time.time()++ try:+ for chunk in response_stream:+ elapsed = time.time() - start_time+ chunks_with_time.append((elapsed, chunk))++ # Categorize chunks by type+ chunk_type = type(chunk).__name__+ chunk_info = f"[{elapsed:.3f}s] {chunk_type}"++ # Check for different message types+ if hasattr(chunk, "message_type"):+ chunk_info += f" (message_type: {chunk.message_type})"+ if chunk.message_type == "reasoning_message":+ reasoning_chunks.append((elapsed, chunk))+ elif chunk.message_type == "assistant_message":+ assistant_chunks.append((elapsed, chunk))+ elif chunk.message_type == "tool_call_message":+ tool_chunks.append((elapsed, chunk))+ elif type(chunk).__name__ == "ReasoningMessage":+ chunk_info += " (ReasoningMessage)"+ reasoning_chunks.append((elapsed, chunk))+ elif type(chunk).__name__ == "AssistantMessage":+ chunk_info += " (AssistantMessage)"+ assistant_chunks.append((elapsed, chunk))+ elif type(chunk).__name__ == "ToolCallMessage":+ chunk_info += " (ToolCallMessage)"+ tool_chunks.append((elapsed, chunk))++ # Check for inner thoughts (in tool calls for non-native reasoning)+ if hasattr(chunk, "tool_calls") and chunk.tool_calls:+ for tool_call in chunk.tool_calls:+ if hasattr(tool_call, "function") and hasattr(tool_call.function, "arguments"):+ # Check if this is inner thoughts+ if "inner_thoughts" in str(tool_call.function.arguments):+ chunk_info += " [contains inner_thoughts]"+ tool_chunks.append((elapsed, chunk))++ print(chunk_info)++ # Optional: print chunk content snippet for debugging+ if hasattr(chunk, "content") and chunk.content:+ content_preview = str(chunk.content)[:100]+ if content_preview:+ print(f" Content: {content_preview}...")++ except Exception as e:+ print(f"Stream error: {e}")+ import traceback++ traceback.print_exc()++ # Analyze results+ print(f"\n=== Analysis ===")+ print(f"Total chunks: {len(chunks_with_time)}")+ print(f"Reasoning chunks: {len(reasoning_chunks)}")+ print(f"Assistant chunks: {len(assistant_chunks)}")+ print(f"Tool chunks: {len(tool_chunks)}")++ # Detect bursts for each type+ if reasoning_chunks:+ reasoning_bursts = detect_burst_chunks(reasoning_chunks)+ print(f"\nReasoning bursts detected: {len(reasoning_bursts)}")+ for i, burst in enumerate(reasoning_bursts):+ burst_times = [reasoning_chunks[idx][0] for idx in burst]+ print(f" Burst {i+1}: {len(burst)} chunks from {burst_times[0]:.3f}s to {burst_times[-1]:.3f}s")++ if assistant_chunks:+ assistant_bursts = detect_burst_chunks(assistant_chunks)+ print(f"\nAssistant bursts detected: {len(assistant_bursts)}")+ for i, burst in enumerate(assistant_bursts):+ burst_times = [assistant_chunks[idx][0] for idx in burst]+ print(f" Burst {i+1}: {len(burst)} chunks from {burst_times[0]:.3f}s to {burst_times[-1]:.3f}s")++ if tool_chunks:+ tool_bursts = detect_burst_chunks(tool_chunks)+ print(f"\nTool call bursts detected: {len(tool_bursts)}")+ for i, burst in enumerate(tool_bursts):+ burst_times = [tool_chunks[idx][0] for idx in burst]+ print(f" Burst {i+1}: {len(burst)} chunks from {burst_times[0]:.3f}s to {burst_times[-1]:.3f}s")++ # Analyze results based on expected behavior+ print(f"\n=== Test Results ===")++ # Check if we detected large bursts+ has_significant_bursts = False++ if reasoning_chunks:+ reasoning_bursts = detect_burst_chunks(reasoning_chunks, burst_threshold=0.1)+ if reasoning_bursts:+ largest_burst = max(reasoning_bursts, key=len)+ burst_percentage = len(largest_burst) / len(reasoning_chunks) * 100+ print(f"\nLargest reasoning burst: {len(largest_burst)}/{len(reasoning_chunks)} chunks ({burst_percentage:.1f}%)")++ if burst_percentage >= 80: # Consider 80%+ as significant buffering+ has_significant_bursts = True+ print(f" -> BUFFERING DETECTED: {burst_percentage:.1f}% of reasoning chunks in single burst")++ if assistant_chunks:+ assistant_bursts = detect_burst_chunks(assistant_chunks, burst_threshold=0.1)+ if assistant_bursts:+ largest_burst = max(assistant_bursts, key=len)+ burst_percentage = len(largest_burst) / len(assistant_chunks) * 100+ print(f"Largest assistant burst: {len(largest_burst)}/{len(assistant_chunks)} chunks ({burst_percentage:.1f}%)")++ if burst_percentage >= 80:+ has_significant_bursts = True+ print(f" -> BUFFERING DETECTED: {burst_percentage:.1f}% of assistant chunks in single burst")++ if tool_chunks:+ tool_bursts = detect_burst_chunks(tool_chunks, burst_threshold=0.1)+ if tool_bursts:+ largest_burst = max(tool_bursts, key=len)+ burst_percentage = len(largest_burst) / len(tool_chunks) * 100+ print(f"Largest tool burst: {len(largest_burst)}/{len(tool_chunks)} chunks ({burst_percentage:.1f}%)")++ if burst_percentage >= 80:+ has_significant_bursts = True+ print(f" -> BUFFERING DETECTED: {burst_percentage:.1f}% of tool chunks in single burst")++ # Overall streaming analysis+ total_time = chunks_with_time[-1][0] if chunks_with_time else 0+ avg_time_between = total_time / len(chunks_with_time) if chunks_with_time else 0+ print(f"\nTotal streaming time: {total_time:.2f}s")+ print(f"Average time between chunks: {avg_time_between:.3f}s")++ # Verify test expectations+ if expected_buffering:+ assert has_significant_bursts, (+ f"Expected buffering behavior for {model}, but streaming appeared progressive. "+ f"This suggests the issue may be fixed or the test isn't detecting it properly."+ )+ print(f"\n✓ Test PASSED: {model} shows expected buffering behavior")+ else:+ assert not has_significant_bursts, (+ f"Did NOT expect buffering for {model}, but detected significant burst behavior. "+ f"This suggests {model} may also have streaming issues."+ )+ print(f"\n✓ Test PASSED: {model} shows expected progressive streaming")+++if __name__ == "__main__":+ # Allow running directly for debugging+ pytest.main([__file__, "-v", "-s"])