Platform for stateful agents: AI with advanced memory that can learn and self-improve over time.
test: add agent loop tests (#2088)
cthomasa8d7e2fcc867
diff --git a/letta/server/rest_api/routers/v1/agents.py b/letta/server/rest_api/routers/v1/agents.pyindex 8d9fd7f65..113beb68b 100644--- a/letta/server/rest_api/routers/v1/agents.py+++ b/letta/server/rest_api/routers/v1/agents.py@@ -3,7 +3,7 @@ import tracebackfrom datetime import datetime, timezonefrom typing import Annotated, Any, List, Optional-from fastapi import APIRouter, BackgroundTasks, Body, Depends, File, Header, HTTPException, Query, UploadFile, status+from fastapi import APIRouter, BackgroundTasks, Body, Depends, File, Header, HTTPException, Query, Request, UploadFile, statusfrom fastapi.responses import JSONResponsefrom marshmallow import ValidationErrorfrom orjson import orjson@@ -619,6 +619,7 @@ def modify_message()async def send_message(agent_id: str,+ request_obj: Request, # FastAPI Requestserver: SyncServer = Depends(get_letta_server),request: LettaRequest = Body(...),actor_id: Optional[str] = Header(None, alias="user_id"), # Extract user_id from header, default to None if not present@@ -630,19 +631,12 @@ async def send_message(actor = server.user_manager.get_user_or_default(user_id=actor_id)# TODO: This is redundant, remove soonagent = server.agent_manager.get_agent_by_id(agent_id, actor)+ agent_eligible = not agent.enable_sleeptime and not agent.multi_agent_group and agent.agent_type != AgentType.sleeptime_agent+ experimental_header = request_obj.headers.get("x-experimental")+ feature_enabled = settings.use_experimental or experimental_header+ model_compatible = agent.llm_config.model_endpoint_type in ["anthropic", "openai", "google_vertex", "google_ai"]- if all(- (- settings.use_experimental,- not agent.enable_sleeptime,- not agent.multi_agent_group,- not agent.agent_type == AgentType.sleeptime_agent,- )- ) and (- # LLM Model Check: (1) Anthropic or (2) Google Vertex + Flag- agent.llm_config.model_endpoint_type == "anthropic"- or (agent.llm_config.model_endpoint_type == "google_vertex" and settings.use_vertex_async_loop_experimental)- ):+ if agent_eligible and feature_enabled and model_compatible:experimental_agent = LettaAgent(agent_id=agent_id,message_manager=server.message_manager,@@ -683,6 +677,7 @@ async def send_message()async def send_message_streaming(agent_id: str,+ request_obj: Request, # FastAPI Requestserver: SyncServer = Depends(get_letta_server),request: LettaStreamingRequest = Body(...),actor_id: Optional[str] = Header(None, alias="user_id"), # Extract user_id from header, default to None if not present@@ -696,14 +691,12 @@ async def send_message_streaming(actor = server.user_manager.get_user_or_default(user_id=actor_id)# TODO: This is redundant, remove soonagent = server.agent_manager.get_agent_by_id(agent_id, actor)+ agent_eligible = not agent.enable_sleeptime and not agent.multi_agent_group and agent.agent_type != AgentType.sleeptime_agent+ experimental_header = request_obj.headers.get("x-experimental")+ feature_enabled = settings.use_experimental or experimental_header+ model_compatible = agent.llm_config.model_endpoint_type in ["anthropic", "openai"]- if (- agent.llm_config.model_endpoint_type == "anthropic"- and not agent.enable_sleeptime- and not agent.multi_agent_group- and not agent.agent_type == AgentType.sleeptime_agent- and settings.use_experimental- ):+ if agent_eligible and feature_enabled and model_compatible:experimental_agent = LettaAgent(agent_id=agent_id,message_manager=server.message_manager,diff --git a/tests/configs/llm_model_configs/gemini-1.5-pro.json b/tests/configs/llm_model_configs/gemini-1.5-pro.jsonnew file mode 100644index 000000000..356d199b2--- /dev/null+++ b/tests/configs/llm_model_configs/gemini-1.5-pro.json@@ -0,0 +1,8 @@+{+ "context_window": 2097152,+ "model": "gemini-1.5-pro-latest",+ "model_endpoint_type": "google_ai",+ "model_endpoint": "https://generativelanguage.googleapis.com",+ "model_wrapper": null,+ "put_inner_thoughts_in_kwargs": true+}diff --git a/tests/configs/llm_model_configs/gemini-2.5-flash-vertex.json b/tests/configs/llm_model_configs/gemini-2.5-flash-vertex.jsonnew file mode 100644index 000000000..082aa1522--- /dev/null+++ b/tests/configs/llm_model_configs/gemini-2.5-flash-vertex.json@@ -0,0 +1,7 @@+{+ "model": "gemini-2.5-flash-preview-04-17",+ "model_endpoint_type": "google_vertex",+ "model_endpoint": "https://us-central1-aiplatform.googleapis.com/v1/projects/memgpt-428419/locations/us-central1",+ "context_window": 1048576,+ "put_inner_thoughts_in_kwargs": true+}diff --git a/tests/configs/llm_model_configs/gemini-vertex.json b/tests/configs/llm_model_configs/gemini-2.5-pro-vertex.jsonsimilarity index 100%rename from tests/configs/llm_model_configs/gemini-vertex.jsonrename to tests/configs/llm_model_configs/gemini-2.5-pro-vertex.jsondiff --git a/tests/configs/llm_model_configs/gemini-pro.json b/tests/configs/llm_model_configs/gemini-pro.jsondeleted file mode 100644index e59252822..000000000--- a/tests/configs/llm_model_configs/gemini-pro.json+++ /dev/null@@ -1,8 +0,0 @@-{- "context_window": 2097152,- "model": "gemini-1.5-pro-latest",- "model_endpoint_type": "google_ai",- "model_endpoint": "https://generativelanguage.googleapis.com",- "model_wrapper": null,- "put_inner_thoughts_in_kwargs": true-}diff --git a/tests/conftest.py b/tests/conftest.pyindex 314e006db..e44d2fecc 100644--- a/tests/conftest.py+++ b/tests/conftest.py@@ -13,6 +13,9 @@ from letta.settings import tool_settingsdef pytest_configure(config):logging.basicConfig(level=logging.DEBUG)+ # Register custom markers+ config.addinivalue_line("markers", "async_client_test: mark test as an async client test that is skipped by default")+@pytest.fixturedef disable_e2b_api_key() -> Generator[None, None, None]:diff --git a/tests/integration_test_send_message.py b/tests/integration_test_send_message.pyindex c9ee862a4..cb8b5d3b7 100644--- a/tests/integration_test_send_message.py+++ b/tests/integration_test_send_message.py@@ -8,7 +8,7 @@ import pytestimport requestsfrom dotenv import load_dotenvfrom letta_client import AsyncLetta, Letta, Run-from letta_client.types import AssistantMessage, ReasoningMessage+from letta_client.types import AssistantMessage, LettaUsageStatistics, ReasoningMessage, ToolCallMessage, ToolReturnMessagefrom letta.schemas.agent import AgentStatefrom letta.schemas.llm_config import LLMConfig@@ -55,7 +55,7 @@ def server_url() -> str:return url[email protected](scope="module")def client(server_url: str) -> Letta:"""Creates and returns a synchronous Letta REST client for testing.@@ -64,7 +64,7 @@ def client(server_url: str) -> Letta:yield client_instance[email protected](scope="function")def async_client(server_url: str) -> AsyncLetta:"""Creates and returns an asynchronous Letta REST client for testing.@@ -73,7 +73,7 @@ def async_client(server_url: str) -> AsyncLetta:yield async_client_instance[email protected](scope="module")def agent_state(client: Letta) -> AgentState:"""Creates and returns an agent state for testing with a pre-configured agent.@@ -92,6 +92,8 @@ def agent_state(client: Letta) -> AgentState:)yield agent_state_instance+ client.agents.delete(agent_state_instance.id)+# ------------------------------# Helper Functions and Constants@@ -105,64 +107,110 @@ def get_llm_config(filename: str, llm_config_dir: str = "tests/configs/llm_modelreturn llm_config-USER_MESSAGE: List[Dict[str, str]] = [{"role": "user", "content": "Hi there."}]+def roll_dice(num_sides: int) -> int:+ """+ Returns a random number between 1 and num_sides.+ Args:+ num_sides (int): The number of sides on the die.+ Returns:+ int: A random integer between 1 and num_sides, representing the die roll.+ """+ import random++ return random.randint(1, num_sides)+++USER_MESSAGE_GREETING: List[Dict[str, str]] = [{"role": "user", "content": "Hi there."}]+USER_MESSAGE_TOOL_CALL: List[Dict[str, str]] = [+ {"role": "user", "content": "Call the roll_dice tool with 16 sides and tell me the outcome."}+]all_configs = ["openai-gpt-4o-mini.json","azure-gpt-4o-mini.json","claude-3-5-sonnet.json","claude-3-7-sonnet.json","claude-3-7-sonnet-extended.json",- "gemini-pro.json",- "gemini-vertex.json",+ "gemini-1.5-pro.json",+ "gemini-2.5-flash-vertex.json",+ "gemini-2.5-pro-vertex.json",]requested = os.getenv("LLM_CONFIG_FILE")filenames = [requested] if requested else all_configsTESTED_LLM_CONFIGS: List[LLMConfig] = [get_llm_config(fn) for fn in filenames]-def assert_tool_response_messages(messages: List[Any]) -> None:+def assert_greeting_with_assistant_message_response(messages: List[Any], streaming: bool = False) -> None:"""Asserts that the messages list follows the expected sequence:- ReasoningMessage -> ToolCallMessage -> ToolReturnMessage ->ReasoningMessage -> AssistantMessage."""+ expected_message_count = 3 if streaming else 2+ assert len(messages) == expected_message_count+assert isinstance(messages[0], ReasoningMessage)assert isinstance(messages[1], AssistantMessage)+ if streaming:+ assert isinstance(messages[2], LettaUsageStatistics)+-def assert_streaming_tool_response_messages(chunks: List[Any]) -> None:+def assert_greeting_without_assistant_message_response(messages: List[Any], streaming: bool = False) -> None:"""- Validates that streaming responses contain at least one reasoning message,- one tool call, one tool return, one assistant message, and one usage statistics message.+ Asserts that the messages list follows the expected sequence:+ ReasoningMessage -> ToolCallMessage -> ToolReturnMessage."""+ expected_message_count = 4 if streaming else 3+ assert len(messages) == expected_message_count- def msg_groups(msg_type: Any) -> List[Any]:- return [c for c in chunks if isinstance(c, msg_type)]-- reasoning_msgs = msg_groups(ReasoningMessage)- assistant_msgs = msg_groups(AssistantMessage)+ assert isinstance(messages[0], ReasoningMessage)+ assert isinstance(messages[1], ToolCallMessage)+ assert isinstance(messages[2], ToolReturnMessage)- assert len(reasoning_msgs) == 1- assert len(assistant_msgs) == 1+ if streaming:+ assert isinstance(messages[3], LettaUsageStatistics)-def wait_for_run_completion(client: Letta, run_id: str, timeout: float = 30.0, interval: float = 0.5) -> Run:+def assert_tool_call_response(messages: List[Any], streaming: bool = False) -> None:+ """+ Asserts that the messages list follows the expected sequence:+ ReasoningMessage -> ToolCallMessage -> ToolReturnMessage ->+ ReasoningMessage -> AssistantMessage."""- Polls the run status until it completes or fails.+ expected_message_count = 6 if streaming else 5+ assert len(messages) == expected_message_count- Args:- client (Letta): The synchronous Letta client.- run_id (str): The identifier of the run to wait for.- timeout (float): Maximum time to wait (in seconds).- interval (float): Interval between status checks (in seconds).+ assert isinstance(messages[0], ReasoningMessage)+ assert isinstance(messages[1], ToolCallMessage)+ assert isinstance(messages[2], ToolReturnMessage)+ assert isinstance(messages[3], ReasoningMessage)+ assert isinstance(messages[4], AssistantMessage)++ if streaming:+ assert isinstance(messages[5], LettaUsageStatistics)- Returns:- Run: The completed run object.- Raises:- RuntimeError: If the run fails.- TimeoutError: If the run does not complete within the specified timeout.+def accumulate_chunks(chunks: List[Any]) -> List[Any]:"""+ Accumulates chunks into a list of messages.+ """+ messages = []+ current_message = None+ prev_message_type = None+ for chunk in chunks:+ current_message_type = chunk.message_type+ if prev_message_type != current_message_type:+ messages.append(current_message)+ current_message = None+ if current_message is None:+ current_message = chunk+ else:+ pass # TODO: actually accumulate the chunks. For now we only care about the count+ prev_message_type = current_message_type+ messages.append(current_message)+ return [m for m in messages if m is not None]+++def wait_for_run_completion(client: Letta, run_id: str, timeout: float = 30.0, interval: float = 0.5) -> Run:start = time.time()while True:run = client.runs.retrieve(run_id)@@ -196,8 +244,12 @@ def assert_tool_response_dict_messages(messages: List[Dict[str, Any]]) -> None:# ------------------------------[email protected]("llm_config", TESTED_LLM_CONFIGS)-def test_send_message_sync_client(+ "llm_config",+ TESTED_LLM_CONFIGS,+ ids=[c.model for c in TESTED_LLM_CONFIGS],+)+def test_greeting_with_assistant_message(disable_e2b_api_key: Any,client: Letta,agent_state: AgentState,@@ -207,17 +259,71 @@ def test_send_message_sync_client(Tests sending a message with a synchronous client.Verifies that the response messages follow the expected order."""- client.agents.modify(agent_id=agent_state.id, llm_config=llm_config)+ agent_state = client.agents.modify(agent_id=agent_state.id, llm_config=llm_config)response = client.agents.messages.create(agent_id=agent_state.id,- messages=USER_MESSAGE,+ messages=USER_MESSAGE_GREETING,)- assert_tool_response_messages(response.messages)+ assert_greeting_with_assistant_message_response(response.messages)+++ "llm_config",+ TESTED_LLM_CONFIGS,+ ids=[c.model for c in TESTED_LLM_CONFIGS],+)+def test_greeting_without_assistant_message(+ disable_e2b_api_key: Any,+ client: Letta,+ agent_state: AgentState,+ llm_config: LLMConfig,+) -> None:+ """+ Tests sending a message with a synchronous client.+ Verifies that the response messages follow the expected order.+ """+ agent_state = client.agents.modify(agent_id=agent_state.id, llm_config=llm_config)+ response = client.agents.messages.create(+ agent_id=agent_state.id,+ messages=USER_MESSAGE_GREETING,+ use_assistant_message=False,+ )+ assert_greeting_without_assistant_message_response(response.messages)+++ "llm_config",+ TESTED_LLM_CONFIGS,+ ids=[c.model for c in TESTED_LLM_CONFIGS],+)+def test_tool_call(+ disable_e2b_api_key: Any,+ client: Letta,+ agent_state: AgentState,+ llm_config: LLMConfig,+) -> None:+ """+ Tests sending a message with a synchronous client.+ Verifies that the response messages follow the expected order.+ """+ dice_tool = client.tools.upsert_from_function(func=roll_dice)+ client.agents.tools.attach(agent_id=agent_state.id, tool_id=dice_tool.id)+ agent_state = client.agents.modify(agent_id=agent_state.id, llm_config=llm_config)+ response = client.agents.messages.create(+ agent_id=agent_state.id,+ messages=USER_MESSAGE_TOOL_CALL,+ )+ assert_tool_call_response(response.messages)@pytest.mark.asyncio[email protected]("llm_config", TESTED_LLM_CONFIGS)-async def test_send_message_async_client([email protected]_client_test+ "llm_config",+ TESTED_LLM_CONFIGS,+ ids=[c.model for c in TESTED_LLM_CONFIGS],+)+async def test_greeting_with_assistant_message_async_client(disable_e2b_api_key: Any,async_client: AsyncLetta,agent_state: AgentState,@@ -227,16 +333,73 @@ async def test_send_message_async_client(Tests sending a message with an asynchronous client.Validates that the response messages match the expected sequence."""- await async_client.agents.modify(agent_id=agent_state.id, llm_config=llm_config)+ agent_state = await async_client.agents.modify(agent_id=agent_state.id, llm_config=llm_config)response = await async_client.agents.messages.create(agent_id=agent_state.id,- messages=USER_MESSAGE,+ messages=USER_MESSAGE_GREETING,)- assert_tool_response_messages(response.messages)+ assert_greeting_with_assistant_message_response(response.messages)[email protected]("llm_config", TESTED_LLM_CONFIGS)-def test_send_message_streaming_sync_client([email protected]_client_test+ "llm_config",+ TESTED_LLM_CONFIGS,+ ids=[c.model for c in TESTED_LLM_CONFIGS],+)+async def test_greeting_without_assistant_message_async_client(+ disable_e2b_api_key: Any,+ async_client: AsyncLetta,+ agent_state: AgentState,+ llm_config: LLMConfig,+) -> None:+ """+ Tests sending a message with an asynchronous client.+ Validates that the response messages match the expected sequence.+ """+ agent_state = await async_client.agents.modify(agent_id=agent_state.id, llm_config=llm_config)+ response = await async_client.agents.messages.create(+ agent_id=agent_state.id,+ messages=USER_MESSAGE_GREETING,+ use_assistant_message=False,+ )+ assert_greeting_without_assistant_message_response(response.messages)++[email protected]_client_test+ "llm_config",+ TESTED_LLM_CONFIGS,+ ids=[c.model for c in TESTED_LLM_CONFIGS],+)+async def test_tool_call_async_client(+ disable_e2b_api_key: Any,+ async_client: AsyncLetta,+ agent_state: AgentState,+ llm_config: LLMConfig,+) -> None:+ """+ Tests sending a message with an asynchronous client.+ Validates that the response messages match the expected sequence.+ """+ dice_tool = await async_client.tools.upsert_from_function(func=roll_dice)+ agent_state = await async_client.agents.tools.attach(agent_id=agent_state.id, tool_id=dice_tool.id)+ agent_state = await async_client.agents.modify(agent_id=agent_state.id, llm_config=llm_config)+ response = await async_client.agents.messages.create(+ agent_id=agent_state.id,+ messages=USER_MESSAGE_TOOL_CALL,+ )+ assert_tool_call_response(response.messages)+++ "llm_config",+ TESTED_LLM_CONFIGS,+ ids=[c.model for c in TESTED_LLM_CONFIGS],+)+def test_streaming_greeting_with_assistant_message(disable_e2b_api_key: Any,client: Letta,agent_state: AgentState,@@ -246,18 +409,270 @@ def test_send_message_streaming_sync_client(Tests sending a streaming message with a synchronous client.Checks that each chunk in the stream has the correct message types."""- client.agents.modify(agent_id=agent_state.id, llm_config=llm_config)+ agent_state = client.agents.modify(agent_id=agent_state.id, llm_config=llm_config)+ response = client.agents.messages.create_stream(+ agent_id=agent_state.id,+ messages=USER_MESSAGE_GREETING,+ )+ chunks = list(response)+ messages = accumulate_chunks(chunks)+ assert_greeting_with_assistant_message_response(messages, streaming=True)+++ "llm_config",+ TESTED_LLM_CONFIGS,+ ids=[c.model for c in TESTED_LLM_CONFIGS],+)+def test_streaming_greeting_without_assistant_message(+ disable_e2b_api_key: Any,+ client: Letta,+ agent_state: AgentState,+ llm_config: LLMConfig,+) -> None:+ """+ Tests sending a streaming message with a synchronous client.+ Checks that each chunk in the stream has the correct message types.+ """+ agent_state = client.agents.modify(agent_id=agent_state.id, llm_config=llm_config)+ response = client.agents.messages.create_stream(+ agent_id=agent_state.id,+ messages=USER_MESSAGE_GREETING,+ use_assistant_message=False,+ )+ chunks = list(response)+ messages = accumulate_chunks(chunks)+ assert_greeting_without_assistant_message_response(messages, streaming=True)+++ "llm_config",+ TESTED_LLM_CONFIGS,+ ids=[c.model for c in TESTED_LLM_CONFIGS],+)+def test_streaming_tool_call(+ disable_e2b_api_key: Any,+ client: Letta,+ agent_state: AgentState,+ llm_config: LLMConfig,+) -> None:+ """+ Tests sending a streaming message with a synchronous client.+ Checks that each chunk in the stream has the correct message types.+ """+ dice_tool = client.tools.upsert_from_function(func=roll_dice)+ agent_state = client.agents.tools.attach(agent_id=agent_state.id, tool_id=dice_tool.id)+ agent_state = client.agents.modify(agent_id=agent_state.id, llm_config=llm_config)+ response = client.agents.messages.create_stream(+ agent_id=agent_state.id,+ messages=USER_MESSAGE_TOOL_CALL,+ )+ chunks = list(response)+ messages = accumulate_chunks(chunks)+ assert_tool_call_response(messages, streaming=True)++[email protected]_client_test+ "llm_config",+ TESTED_LLM_CONFIGS,+ ids=[c.model for c in TESTED_LLM_CONFIGS],+)+async def test_streaming_greeting_with_assistant_message_async_client(+ disable_e2b_api_key: Any,+ async_client: AsyncLetta,+ agent_state: AgentState,+ llm_config: LLMConfig,+) -> None:+ """+ Tests sending a streaming message with an asynchronous client.+ Validates that the streaming response chunks include the correct message types.+ """+ await async_client.agents.modify(agent_id=agent_state.id, llm_config=llm_config)+ response = async_client.agents.messages.create_stream(+ agent_id=agent_state.id,+ messages=USER_MESSAGE_GREETING,+ )+ chunks = [chunk async for chunk in response]+ messages = accumulate_chunks(chunks)+ assert_greeting_with_assistant_message_response(messages, streaming=True)++[email protected]_client_test+ "llm_config",+ TESTED_LLM_CONFIGS,+ ids=[c.model for c in TESTED_LLM_CONFIGS],+)+async def test_streaming_greeting_without_assistant_message_async_client(+ disable_e2b_api_key: Any,+ async_client: AsyncLetta,+ agent_state: AgentState,+ llm_config: LLMConfig,+) -> None:+ """+ Tests sending a streaming message with an asynchronous client.+ Validates that the streaming response chunks include the correct message types.+ """+ await async_client.agents.modify(agent_id=agent_state.id, llm_config=llm_config)+ response = async_client.agents.messages.create_stream(+ agent_id=agent_state.id,+ messages=USER_MESSAGE_GREETING,+ use_assistant_message=False,+ )+ chunks = [chunk async for chunk in response]+ messages = accumulate_chunks(chunks)+ assert_greeting_without_assistant_message_response(messages, streaming=True)++[email protected]_client_test+ "llm_config",+ TESTED_LLM_CONFIGS,+ ids=[c.model for c in TESTED_LLM_CONFIGS],+)+async def test_streaming_tool_call_async_client(+ disable_e2b_api_key: Any,+ async_client: AsyncLetta,+ agent_state: AgentState,+ llm_config: LLMConfig,+) -> None:+ """+ Tests sending a streaming message with an asynchronous client.+ Validates that the streaming response chunks include the correct message types.+ """+ dice_tool = await async_client.tools.upsert_from_function(func=roll_dice)+ agent_state = await async_client.agents.tools.attach(agent_id=agent_state.id, tool_id=dice_tool.id)+ agent_state = await async_client.agents.modify(agent_id=agent_state.id, llm_config=llm_config)+ response = async_client.agents.messages.create_stream(+ agent_id=agent_state.id,+ messages=USER_MESSAGE_TOOL_CALL,+ )+ chunks = [chunk async for chunk in response]+ messages = accumulate_chunks(chunks)+ assert_tool_call_response(messages, streaming=True)+++ "llm_config",+ TESTED_LLM_CONFIGS,+ ids=[c.model for c in TESTED_LLM_CONFIGS],+)+def test_token_streaming_greeting_with_assistant_message(+ disable_e2b_api_key: Any,+ client: Letta,+ agent_state: AgentState,+ llm_config: LLMConfig,+) -> None:+ """+ Tests sending a streaming message with a synchronous client.+ Checks that each chunk in the stream has the correct message types.+ """+ agent_state = client.agents.modify(agent_id=agent_state.id, llm_config=llm_config)response = client.agents.messages.create_stream(agent_id=agent_state.id,- messages=USER_MESSAGE,+ messages=USER_MESSAGE_GREETING,+ stream_tokens=True,)chunks = list(response)- assert_streaming_tool_response_messages(chunks)+ messages = accumulate_chunks(chunks)+ assert_greeting_with_assistant_message_response(messages, streaming=True)+++ "llm_config",+ TESTED_LLM_CONFIGS,+ ids=[c.model for c in TESTED_LLM_CONFIGS],+)+def test_token_streaming_greeting_without_assistant_message(+ disable_e2b_api_key: Any,+ client: Letta,+ agent_state: AgentState,+ llm_config: LLMConfig,+) -> None:+ """+ Tests sending a streaming message with a synchronous client.+ Checks that each chunk in the stream has the correct message types.+ """+ agent_state = client.agents.modify(agent_id=agent_state.id, llm_config=llm_config)+ response = client.agents.messages.create_stream(+ agent_id=agent_state.id,+ messages=USER_MESSAGE_GREETING,+ use_assistant_message=False,+ stream_tokens=True,+ )+ chunks = list(response)+ messages = accumulate_chunks(chunks)+ assert_greeting_without_assistant_message_response(messages, streaming=True)+++ "llm_config",+ TESTED_LLM_CONFIGS,+ ids=[c.model for c in TESTED_LLM_CONFIGS],+)+def test_token_streaming_tool_call(+ disable_e2b_api_key: Any,+ client: Letta,+ agent_state: AgentState,+ llm_config: LLMConfig,+) -> None:+ """+ Tests sending a streaming message with a synchronous client.+ Checks that each chunk in the stream has the correct message types.+ """+ dice_tool = client.tools.upsert_from_function(func=roll_dice)+ agent_state = client.agents.tools.attach(agent_id=agent_state.id, tool_id=dice_tool.id)+ agent_state = client.agents.modify(agent_id=agent_state.id, llm_config=llm_config)+ response = client.agents.messages.create_stream(+ agent_id=agent_state.id,+ messages=USER_MESSAGE_TOOL_CALL,+ stream_tokens=True,+ )+ chunks = list(response)+ messages = accumulate_chunks(chunks)+ assert_tool_call_response(messages, streaming=True)++[email protected]_client_test+ "llm_config",+ TESTED_LLM_CONFIGS,+ ids=[c.model for c in TESTED_LLM_CONFIGS],+)+async def test_token_streaming_greeting_with_assistant_message_async_client(+ disable_e2b_api_key: Any,+ async_client: AsyncLetta,+ agent_state: AgentState,+ llm_config: LLMConfig,+) -> None:+ """+ Tests sending a streaming message with an asynchronous client.+ Validates that the streaming response chunks include the correct message types.+ """+ await async_client.agents.modify(agent_id=agent_state.id, llm_config=llm_config)+ response = async_client.agents.messages.create_stream(+ agent_id=agent_state.id,+ messages=USER_MESSAGE_GREETING,+ stream_tokens=True,+ )+ chunks = [chunk async for chunk in response]+ messages = accumulate_chunks(chunks)+ assert_greeting_with_assistant_message_response(messages, streaming=True)@pytest.mark.asyncio[email protected]("llm_config", TESTED_LLM_CONFIGS)-async def test_send_message_streaming_async_client([email protected]_client_test+ "llm_config",+ TESTED_LLM_CONFIGS,+ ids=[c.model for c in TESTED_LLM_CONFIGS],+)+async def test_token_streaming_greeting_without_assistant_message_async_client(disable_e2b_api_key: Any,async_client: AsyncLetta,agent_state: AgentState,@@ -270,14 +685,51 @@ async def test_send_message_streaming_async_client(await async_client.agents.modify(agent_id=agent_state.id, llm_config=llm_config)response = async_client.agents.messages.create_stream(agent_id=agent_state.id,- messages=USER_MESSAGE,+ messages=USER_MESSAGE_GREETING,+ use_assistant_message=False,+ stream_tokens=True,+ )+ chunks = [chunk async for chunk in response]+ messages = accumulate_chunks(chunks)+ assert_greeting_without_assistant_message_response(messages, streaming=True)++[email protected]_client_test+ "llm_config",+ TESTED_LLM_CONFIGS,+ ids=[c.model for c in TESTED_LLM_CONFIGS],+)+async def test_token_streaming_tool_call_async_client(+ disable_e2b_api_key: Any,+ async_client: AsyncLetta,+ agent_state: AgentState,+ llm_config: LLMConfig,+) -> None:+ """+ Tests sending a streaming message with an asynchronous client.+ Validates that the streaming response chunks include the correct message types.+ """+ dice_tool = await async_client.tools.upsert_from_function(func=roll_dice)+ agent_state = await async_client.agents.tools.attach(agent_id=agent_state.id, tool_id=dice_tool.id)+ agent_state = await async_client.agents.modify(agent_id=agent_state.id, llm_config=llm_config)+ response = async_client.agents.messages.create_stream(+ agent_id=agent_state.id,+ messages=USER_MESSAGE_TOOL_CALL,+ stream_tokens=True,)chunks = [chunk async for chunk in response]- assert_streaming_tool_response_messages(chunks)+ messages = accumulate_chunks(chunks)+ assert_tool_call_response(messages, streaming=True)[email protected]("llm_config", TESTED_LLM_CONFIGS)-def test_send_message_job_sync_client(+ "llm_config",+ TESTED_LLM_CONFIGS,+ ids=[c.model for c in TESTED_LLM_CONFIGS],+)+def test_async_greeting_with_assistant_message(disable_e2b_api_key: Any,client: Letta,agent_state: AgentState,@@ -291,7 +743,7 @@ def test_send_message_job_sync_client(run = client.agents.messages.create_async(agent_id=agent_state.id,- messages=USER_MESSAGE,+ messages=USER_MESSAGE_GREETING,)run = wait_for_run_completion(client, run.id)@@ -303,8 +755,13 @@ def test_send_message_job_sync_client(@pytest.mark.asyncio[email protected]("llm_config", TESTED_LLM_CONFIGS)-async def test_send_message_job_async_client([email protected]_client_test+ "llm_config",+ TESTED_LLM_CONFIGS,+ ids=[c.model for c in TESTED_LLM_CONFIGS],+)+async def test_async_greeting_with_assistant_message_async_client(disable_e2b_api_key: Any,client: Letta,async_client: AsyncLetta,@@ -319,7 +776,7 @@ async def test_send_message_job_async_client(run = await async_client.agents.messages.create_async(agent_id=agent_state.id,- messages=USER_MESSAGE,+ messages=USER_MESSAGE_GREETING,)# Use the synchronous client to check job completionrun = wait_for_run_completion(client, run.id)diff --git a/tests/pytest.ini b/tests/pytest.iniindex 299f27d4e..bce6a52b6 100644--- a/tests/pytest.ini+++ b/tests/pytest.ini@@ -11,3 +11,6 @@ markers =anthropic_basic: Tests for Anthropic endpointsazure_basic: Tests for Azure endpointsgemini_basic: Tests for Gemini endpoints+ async_client_test: mark test as an async client test that is skipped by default++addopts = -m "not async_client_test"