Skip to content

armgit / letta

Public Imported from GitHub
0

Platform for stateful agents: AI with advanced memory that can learn and self-improve over time.

test: use patch for agent loop error mocking in tests (#2860)

cthomas1a18314c6f4c
diff --git a/letta/server/rest_api/routers/v1/agents.py b/letta/server/rest_api/routers/v1/agents.py
index 8bda580d5..417a88a0e 100644
--- a/letta/server/rest_api/routers/v1/agents.py
+++ b/letta/server/rest_api/routers/v1/agents.py
@@ -1360,6 +1360,9 @@ async def send_message_streaming(
except LLMError as e:
error_data = {"error": {"type": "llm_error", "message": "An error occurred with the LLM request.", "detail": str(e)}}
yield (f"data: {json.dumps(error_data)}\n\n", 502)
+ except Exception as e:
+ error_data = {"error": {"type": "internal_error", "message": "An internal server error occurred.", "detail": str(e)}}
+ yield (f"data: {json.dumps(error_data)}\n\n", 500)
raw_stream = error_aware_stream()
diff --git a/tests/integration_test_send_message.py b/tests/integration_test_send_message.py
index 5701548eb..87a1098a7 100644
--- a/tests/integration_test_send_message.py
+++ b/tests/integration_test_send_message.py
@@ -7,6 +7,7 @@ import uuid
from contextlib import contextmanager
from http.server import BaseHTTPRequestHandler, HTTPServer
from typing import Any, Dict, List
+from unittest.mock import patch
import httpx
import pytest
@@ -710,28 +711,6 @@ def agent_state(client: Letta) -> AgentState:
# logger.error(f"Failed to delete agent {agent_state_instance.name}: {str(e)}")
[email protected](scope="function")
-def agent_state_no_tools(client: Letta) -> AgentState:
- """
- Creates and returns an agent state for testing with a pre-configured agent.
- The agent is named 'supervisor' and is configured with no tools.
- """
- send_message_tool = client.tools.list(name="send_message")[0]
- agent_state_instance = client.agents.create(
- name="supervisor",
- include_base_tools=False,
- model="openai/gpt-4o",
- embedding="letta/letta-free",
- tags=["supervisor"],
- )
- yield agent_state_instance
-
- # try:
- # client.agents.delete(agent_state_instance.id)
- # except Exception as e:
- # logger.error(f"Failed to delete agent {agent_state_instance.name}: {str(e)}")
-
-
# ------------------------------
# Test Cases
# ------------------------------
@@ -925,23 +904,27 @@ def test_base64_image_input(
def test_agent_loop_error(
disable_e2b_api_key: Any,
client: Letta,
- agent_state_no_tools: AgentState,
+ agent_state: AgentState,
llm_config: LLMConfig,
) -> None:
"""
Tests sending a message with a synchronous client.
Verifies that no new messages are persisted on error.
"""
- last_message = client.agents.messages.list(agent_id=agent_state_no_tools.id, limit=1)
- agent_state_no_tools = client.agents.modify(agent_id=agent_state_no_tools.id, llm_config=llm_config)
- with pytest.raises(ApiError):
- client.agents.messages.create(
- agent_id=agent_state_no_tools.id,
- messages=USER_MESSAGE_FORCE_REPLY,
- )
+ last_message = client.agents.messages.list(agent_id=agent_state.id, limit=1)
+ agent_state = client.agents.modify(agent_id=agent_state.id, llm_config=llm_config)
+
+ with patch("letta.agents.letta_agent_v2.LettaAgentV2.step") as mock_step:
+ mock_step.side_effect = ValueError("No tool calls found in response, model must make a tool call")
+
+ with pytest.raises(ApiError):
+ client.agents.messages.create(
+ agent_id=agent_state.id,
+ messages=USER_MESSAGE_FORCE_REPLY,
+ )
time.sleep(0.5)
- messages_from_db = client.agents.messages.list(agent_id=agent_state_no_tools.id, after=last_message[0].id)
+ messages_from_db = client.agents.messages.list(agent_id=agent_state.id, after=last_message[0].id)
assert len(messages_from_db) == 0
@@ -1045,7 +1028,6 @@ def test_step_streaming_tool_call(
assert_tool_call_response(messages_from_db, from_db=True, llm_config=llm_config)
@pytest.mark.parametrize(
"llm_config",
TESTED_LLM_CONFIGS,
@@ -1054,25 +1036,27 @@ def test_step_streaming_tool_call(
def test_step_stream_agent_loop_error(
disable_e2b_api_key: Any,
client: Letta,
- agent_state_no_tools: AgentState,
+ agent_state: AgentState,
llm_config: LLMConfig,
) -> None:
"""
Tests sending a message with a synchronous client.
Verifies that no new messages are persisted on error.
"""
- last_message = client.agents.messages.list(agent_id=agent_state_no_tools.id, limit=1)
- agent_state_no_tools = client.agents.modify(agent_id=agent_state_no_tools.id, llm_config=llm_config)
- response = client.agents.messages.create_stream(
- agent_id=agent_state_no_tools.id,
- messages=USER_MESSAGE_FORCE_REPLY,
- )
- with pytest.raises(Exception) as exc_info:
- for chunk in response:
- print(chunk)
- print("error info:", exc_info)
- assert type(exc_info.value) in (ApiError, ValueError)
- messages_from_db = client.agents.messages.list(agent_id=agent_state_no_tools.id, after=last_message[0].id)
+ last_message = client.agents.messages.list(agent_id=agent_state.id, limit=1)
+ agent_state = client.agents.modify(agent_id=agent_state.id, llm_config=llm_config)
+
+ with patch("letta.agents.letta_agent_v2.LettaAgentV2.stream") as mock_step:
+ mock_step.side_effect = ValueError("No tool calls found in response, model must make a tool call")
+
+ with pytest.raises(ApiError):
+ response = client.agents.messages.create_stream(
+ agent_id=agent_state.id,
+ messages=USER_MESSAGE_FORCE_REPLY,
+ )
+ list(response) # This should trigger the error
+
+ messages_from_db = client.agents.messages.list(agent_id=agent_state.id, after=last_message[0].id)
assert len(messages_from_db) == 0
@@ -1210,26 +1194,28 @@ def test_token_streaming_tool_call(
def test_token_streaming_agent_loop_error(
disable_e2b_api_key: Any,
client: Letta,
- agent_state_no_tools: AgentState,
+ agent_state: AgentState,
llm_config: LLMConfig,
) -> None:
"""
- Tests sending a message with a synchronous client.
+ Tests sending a streaming message with a synchronous client.
Verifies that no new messages are persisted on error.
"""
- last_message = client.agents.messages.list(agent_id=agent_state_no_tools.id, limit=1)
- agent_state_no_tools = client.agents.modify(agent_id=agent_state_no_tools.id, llm_config=llm_config, tool_ids=[])
- try:
- response = client.agents.messages.create_stream(
- agent_id=agent_state_no_tools.id,
- messages=USER_MESSAGE_FORCE_REPLY,
- stream_tokens=True,
- )
- list(response)
- except:
- pass # only some models throw an error TODO: make this consistent
+ last_message = client.agents.messages.list(agent_id=agent_state.id, limit=1)
+ agent_state = client.agents.modify(agent_id=agent_state.id, llm_config=llm_config)
+
+ with patch("letta.agents.letta_agent_v2.LettaAgentV2.stream") as mock_step:
+ mock_step.side_effect = ValueError("No tool calls found in response, model must make a tool call")
- messages_from_db = client.agents.messages.list(agent_id=agent_state_no_tools.id, after=last_message[0].id)
+ with pytest.raises(ApiError):
+ response = client.agents.messages.create_stream(
+ agent_id=agent_state.id,
+ messages=USER_MESSAGE_FORCE_REPLY,
+ stream_tokens=True,
+ )
+ list(response) # This should trigger the error
+
+ messages_from_db = client.agents.messages.list(agent_id=agent_state.id, after=last_message[0].id)
assert len(messages_from_db) == 0