Skip to content

armgit / letta

Public Imported from GitHub
0

Platform for stateful agents: AI with advanced memory that can learn and self-improve over time.

test: add agent loop tests (#2088)

cthomasa8d7e2fcc867
diff --git a/letta/server/rest_api/routers/v1/agents.py b/letta/server/rest_api/routers/v1/agents.py
index 8d9fd7f65..113beb68b 100644
--- a/letta/server/rest_api/routers/v1/agents.py
+++ b/letta/server/rest_api/routers/v1/agents.py
@@ -3,7 +3,7 @@ import traceback
from datetime import datetime, timezone
from typing import Annotated, Any, List, Optional
-from fastapi import APIRouter, BackgroundTasks, Body, Depends, File, Header, HTTPException, Query, UploadFile, status
+from fastapi import APIRouter, BackgroundTasks, Body, Depends, File, Header, HTTPException, Query, Request, UploadFile, status
from fastapi.responses import JSONResponse
from marshmallow import ValidationError
from orjson import orjson
@@ -619,6 +619,7 @@ def modify_message(
)
async def send_message(
agent_id: str,
+ request_obj: Request, # FastAPI Request
server: SyncServer = Depends(get_letta_server),
request: LettaRequest = Body(...),
actor_id: Optional[str] = Header(None, alias="user_id"), # Extract user_id from header, default to None if not present
@@ -630,19 +631,12 @@ async def send_message(
actor = server.user_manager.get_user_or_default(user_id=actor_id)
# TODO: This is redundant, remove soon
agent = server.agent_manager.get_agent_by_id(agent_id, actor)
+ agent_eligible = not agent.enable_sleeptime and not agent.multi_agent_group and agent.agent_type != AgentType.sleeptime_agent
+ experimental_header = request_obj.headers.get("x-experimental")
+ feature_enabled = settings.use_experimental or experimental_header
+ model_compatible = agent.llm_config.model_endpoint_type in ["anthropic", "openai", "google_vertex", "google_ai"]
- if all(
- (
- settings.use_experimental,
- not agent.enable_sleeptime,
- not agent.multi_agent_group,
- not agent.agent_type == AgentType.sleeptime_agent,
- )
- ) and (
- # LLM Model Check: (1) Anthropic or (2) Google Vertex + Flag
- agent.llm_config.model_endpoint_type == "anthropic"
- or (agent.llm_config.model_endpoint_type == "google_vertex" and settings.use_vertex_async_loop_experimental)
- ):
+ if agent_eligible and feature_enabled and model_compatible:
experimental_agent = LettaAgent(
agent_id=agent_id,
message_manager=server.message_manager,
@@ -683,6 +677,7 @@ async def send_message(
)
async def send_message_streaming(
agent_id: str,
+ request_obj: Request, # FastAPI Request
server: SyncServer = Depends(get_letta_server),
request: LettaStreamingRequest = Body(...),
actor_id: Optional[str] = Header(None, alias="user_id"), # Extract user_id from header, default to None if not present
@@ -696,14 +691,12 @@ async def send_message_streaming(
actor = server.user_manager.get_user_or_default(user_id=actor_id)
# TODO: This is redundant, remove soon
agent = server.agent_manager.get_agent_by_id(agent_id, actor)
+ agent_eligible = not agent.enable_sleeptime and not agent.multi_agent_group and agent.agent_type != AgentType.sleeptime_agent
+ experimental_header = request_obj.headers.get("x-experimental")
+ feature_enabled = settings.use_experimental or experimental_header
+ model_compatible = agent.llm_config.model_endpoint_type in ["anthropic", "openai"]
- if (
- agent.llm_config.model_endpoint_type == "anthropic"
- and not agent.enable_sleeptime
- and not agent.multi_agent_group
- and not agent.agent_type == AgentType.sleeptime_agent
- and settings.use_experimental
- ):
+ if agent_eligible and feature_enabled and model_compatible:
experimental_agent = LettaAgent(
agent_id=agent_id,
message_manager=server.message_manager,
diff --git a/tests/configs/llm_model_configs/gemini-1.5-pro.json b/tests/configs/llm_model_configs/gemini-1.5-pro.json
new file mode 100644
index 000000000..356d199b2
--- /dev/null
+++ b/tests/configs/llm_model_configs/gemini-1.5-pro.json
@@ -0,0 +1,8 @@
+{
+ "context_window": 2097152,
+ "model": "gemini-1.5-pro-latest",
+ "model_endpoint_type": "google_ai",
+ "model_endpoint": "https://generativelanguage.googleapis.com",
+ "model_wrapper": null,
+ "put_inner_thoughts_in_kwargs": true
+}
diff --git a/tests/configs/llm_model_configs/gemini-2.5-flash-vertex.json b/tests/configs/llm_model_configs/gemini-2.5-flash-vertex.json
new file mode 100644
index 000000000..082aa1522
--- /dev/null
+++ b/tests/configs/llm_model_configs/gemini-2.5-flash-vertex.json
@@ -0,0 +1,7 @@
+{
+ "model": "gemini-2.5-flash-preview-04-17",
+ "model_endpoint_type": "google_vertex",
+ "model_endpoint": "https://us-central1-aiplatform.googleapis.com/v1/projects/memgpt-428419/locations/us-central1",
+ "context_window": 1048576,
+ "put_inner_thoughts_in_kwargs": true
+}
diff --git a/tests/configs/llm_model_configs/gemini-vertex.json b/tests/configs/llm_model_configs/gemini-2.5-pro-vertex.json
similarity index 100%
rename from tests/configs/llm_model_configs/gemini-vertex.json
rename to tests/configs/llm_model_configs/gemini-2.5-pro-vertex.json
diff --git a/tests/configs/llm_model_configs/gemini-pro.json b/tests/configs/llm_model_configs/gemini-pro.json
deleted file mode 100644
index e59252822..000000000
--- a/tests/configs/llm_model_configs/gemini-pro.json
+++ /dev/null
@@ -1,8 +0,0 @@
-{
- "context_window": 2097152,
- "model": "gemini-1.5-pro-latest",
- "model_endpoint_type": "google_ai",
- "model_endpoint": "https://generativelanguage.googleapis.com",
- "model_wrapper": null,
- "put_inner_thoughts_in_kwargs": true
-}
diff --git a/tests/conftest.py b/tests/conftest.py
index 314e006db..e44d2fecc 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -13,6 +13,9 @@ from letta.settings import tool_settings
def pytest_configure(config):
logging.basicConfig(level=logging.DEBUG)
+ # Register custom markers
+ config.addinivalue_line("markers", "async_client_test: mark test as an async client test that is skipped by default")
+
@pytest.fixture
def disable_e2b_api_key() -> Generator[None, None, None]:
diff --git a/tests/integration_test_send_message.py b/tests/integration_test_send_message.py
index c9ee862a4..cb8b5d3b7 100644
--- a/tests/integration_test_send_message.py
+++ b/tests/integration_test_send_message.py
@@ -8,7 +8,7 @@ import pytest
import requests
from dotenv import load_dotenv
from letta_client import AsyncLetta, Letta, Run
-from letta_client.types import AssistantMessage, ReasoningMessage
+from letta_client.types import AssistantMessage, LettaUsageStatistics, ReasoningMessage, ToolCallMessage, ToolReturnMessage
from letta.schemas.agent import AgentState
from letta.schemas.llm_config import LLMConfig
@@ -55,7 +55,7 @@ def server_url() -> str:
return url
[email protected](scope="module")
def client(server_url: str) -> Letta:
"""
Creates and returns a synchronous Letta REST client for testing.
@@ -64,7 +64,7 @@ def client(server_url: str) -> Letta:
yield client_instance
[email protected](scope="function")
def async_client(server_url: str) -> AsyncLetta:
"""
Creates and returns an asynchronous Letta REST client for testing.
@@ -73,7 +73,7 @@ def async_client(server_url: str) -> AsyncLetta:
yield async_client_instance
[email protected](scope="module")
def agent_state(client: Letta) -> AgentState:
"""
Creates and returns an agent state for testing with a pre-configured agent.
@@ -92,6 +92,8 @@ def agent_state(client: Letta) -> AgentState:
)
yield agent_state_instance
+ client.agents.delete(agent_state_instance.id)
+
# ------------------------------
# Helper Functions and Constants
@@ -105,64 +107,110 @@ def get_llm_config(filename: str, llm_config_dir: str = "tests/configs/llm_model
return llm_config
-USER_MESSAGE: List[Dict[str, str]] = [{"role": "user", "content": "Hi there."}]
+def roll_dice(num_sides: int) -> int:
+ """
+ Returns a random number between 1 and num_sides.
+ Args:
+ num_sides (int): The number of sides on the die.
+ Returns:
+ int: A random integer between 1 and num_sides, representing the die roll.
+ """
+ import random
+
+ return random.randint(1, num_sides)
+
+
+USER_MESSAGE_GREETING: List[Dict[str, str]] = [{"role": "user", "content": "Hi there."}]
+USER_MESSAGE_TOOL_CALL: List[Dict[str, str]] = [
+ {"role": "user", "content": "Call the roll_dice tool with 16 sides and tell me the outcome."}
+]
all_configs = [
"openai-gpt-4o-mini.json",
"azure-gpt-4o-mini.json",
"claude-3-5-sonnet.json",
"claude-3-7-sonnet.json",
"claude-3-7-sonnet-extended.json",
- "gemini-pro.json",
- "gemini-vertex.json",
+ "gemini-1.5-pro.json",
+ "gemini-2.5-flash-vertex.json",
+ "gemini-2.5-pro-vertex.json",
]
requested = os.getenv("LLM_CONFIG_FILE")
filenames = [requested] if requested else all_configs
TESTED_LLM_CONFIGS: List[LLMConfig] = [get_llm_config(fn) for fn in filenames]
-def assert_tool_response_messages(messages: List[Any]) -> None:
+def assert_greeting_with_assistant_message_response(messages: List[Any], streaming: bool = False) -> None:
"""
Asserts that the messages list follows the expected sequence:
- ReasoningMessage -> ToolCallMessage -> ToolReturnMessage ->
ReasoningMessage -> AssistantMessage.
"""
+ expected_message_count = 3 if streaming else 2
+ assert len(messages) == expected_message_count
+
assert isinstance(messages[0], ReasoningMessage)
assert isinstance(messages[1], AssistantMessage)
+ if streaming:
+ assert isinstance(messages[2], LettaUsageStatistics)
+
-def assert_streaming_tool_response_messages(chunks: List[Any]) -> None:
+def assert_greeting_without_assistant_message_response(messages: List[Any], streaming: bool = False) -> None:
"""
- Validates that streaming responses contain at least one reasoning message,
- one tool call, one tool return, one assistant message, and one usage statistics message.
+ Asserts that the messages list follows the expected sequence:
+ ReasoningMessage -> ToolCallMessage -> ToolReturnMessage.
"""
+ expected_message_count = 4 if streaming else 3
+ assert len(messages) == expected_message_count
- def msg_groups(msg_type: Any) -> List[Any]:
- return [c for c in chunks if isinstance(c, msg_type)]
-
- reasoning_msgs = msg_groups(ReasoningMessage)
- assistant_msgs = msg_groups(AssistantMessage)
+ assert isinstance(messages[0], ReasoningMessage)
+ assert isinstance(messages[1], ToolCallMessage)
+ assert isinstance(messages[2], ToolReturnMessage)
- assert len(reasoning_msgs) == 1
- assert len(assistant_msgs) == 1
+ if streaming:
+ assert isinstance(messages[3], LettaUsageStatistics)
-def wait_for_run_completion(client: Letta, run_id: str, timeout: float = 30.0, interval: float = 0.5) -> Run:
+def assert_tool_call_response(messages: List[Any], streaming: bool = False) -> None:
+ """
+ Asserts that the messages list follows the expected sequence:
+ ReasoningMessage -> ToolCallMessage -> ToolReturnMessage ->
+ ReasoningMessage -> AssistantMessage.
"""
- Polls the run status until it completes or fails.
+ expected_message_count = 6 if streaming else 5
+ assert len(messages) == expected_message_count
- Args:
- client (Letta): The synchronous Letta client.
- run_id (str): The identifier of the run to wait for.
- timeout (float): Maximum time to wait (in seconds).
- interval (float): Interval between status checks (in seconds).
+ assert isinstance(messages[0], ReasoningMessage)
+ assert isinstance(messages[1], ToolCallMessage)
+ assert isinstance(messages[2], ToolReturnMessage)
+ assert isinstance(messages[3], ReasoningMessage)
+ assert isinstance(messages[4], AssistantMessage)
+
+ if streaming:
+ assert isinstance(messages[5], LettaUsageStatistics)
- Returns:
- Run: The completed run object.
- Raises:
- RuntimeError: If the run fails.
- TimeoutError: If the run does not complete within the specified timeout.
+def accumulate_chunks(chunks: List[Any]) -> List[Any]:
"""
+ Accumulates chunks into a list of messages.
+ """
+ messages = []
+ current_message = None
+ prev_message_type = None
+ for chunk in chunks:
+ current_message_type = chunk.message_type
+ if prev_message_type != current_message_type:
+ messages.append(current_message)
+ current_message = None
+ if current_message is None:
+ current_message = chunk
+ else:
+ pass # TODO: actually accumulate the chunks. For now we only care about the count
+ prev_message_type = current_message_type
+ messages.append(current_message)
+ return [m for m in messages if m is not None]
+
+
+def wait_for_run_completion(client: Letta, run_id: str, timeout: float = 30.0, interval: float = 0.5) -> Run:
start = time.time()
while True:
run = client.runs.retrieve(run_id)
@@ -196,8 +244,12 @@ def assert_tool_response_dict_messages(messages: List[Dict[str, Any]]) -> None:
# ------------------------------
[email protected]("llm_config", TESTED_LLM_CONFIGS)
-def test_send_message_sync_client(
+ "llm_config",
+ TESTED_LLM_CONFIGS,
+ ids=[c.model for c in TESTED_LLM_CONFIGS],
+)
+def test_greeting_with_assistant_message(
disable_e2b_api_key: Any,
client: Letta,
agent_state: AgentState,
@@ -207,17 +259,71 @@ def test_send_message_sync_client(
Tests sending a message with a synchronous client.
Verifies that the response messages follow the expected order.
"""
- client.agents.modify(agent_id=agent_state.id, llm_config=llm_config)
+ agent_state = client.agents.modify(agent_id=agent_state.id, llm_config=llm_config)
response = client.agents.messages.create(
agent_id=agent_state.id,
- messages=USER_MESSAGE,
+ messages=USER_MESSAGE_GREETING,
)
- assert_tool_response_messages(response.messages)
+ assert_greeting_with_assistant_message_response(response.messages)
+
+
+ "llm_config",
+ TESTED_LLM_CONFIGS,
+ ids=[c.model for c in TESTED_LLM_CONFIGS],
+)
+def test_greeting_without_assistant_message(
+ disable_e2b_api_key: Any,
+ client: Letta,
+ agent_state: AgentState,
+ llm_config: LLMConfig,
+) -> None:
+ """
+ Tests sending a message with a synchronous client.
+ Verifies that the response messages follow the expected order.
+ """
+ agent_state = client.agents.modify(agent_id=agent_state.id, llm_config=llm_config)
+ response = client.agents.messages.create(
+ agent_id=agent_state.id,
+ messages=USER_MESSAGE_GREETING,
+ use_assistant_message=False,
+ )
+ assert_greeting_without_assistant_message_response(response.messages)
+
+
+ "llm_config",
+ TESTED_LLM_CONFIGS,
+ ids=[c.model for c in TESTED_LLM_CONFIGS],
+)
+def test_tool_call(
+ disable_e2b_api_key: Any,
+ client: Letta,
+ agent_state: AgentState,
+ llm_config: LLMConfig,
+) -> None:
+ """
+ Tests sending a message with a synchronous client.
+ Verifies that the response messages follow the expected order.
+ """
+ dice_tool = client.tools.upsert_from_function(func=roll_dice)
+ client.agents.tools.attach(agent_id=agent_state.id, tool_id=dice_tool.id)
+ agent_state = client.agents.modify(agent_id=agent_state.id, llm_config=llm_config)
+ response = client.agents.messages.create(
+ agent_id=agent_state.id,
+ messages=USER_MESSAGE_TOOL_CALL,
+ )
+ assert_tool_call_response(response.messages)
@pytest.mark.asyncio
[email protected]("llm_config", TESTED_LLM_CONFIGS)
-async def test_send_message_async_client(
[email protected]_client_test
+ "llm_config",
+ TESTED_LLM_CONFIGS,
+ ids=[c.model for c in TESTED_LLM_CONFIGS],
+)
+async def test_greeting_with_assistant_message_async_client(
disable_e2b_api_key: Any,
async_client: AsyncLetta,
agent_state: AgentState,
@@ -227,16 +333,73 @@ async def test_send_message_async_client(
Tests sending a message with an asynchronous client.
Validates that the response messages match the expected sequence.
"""
- await async_client.agents.modify(agent_id=agent_state.id, llm_config=llm_config)
+ agent_state = await async_client.agents.modify(agent_id=agent_state.id, llm_config=llm_config)
response = await async_client.agents.messages.create(
agent_id=agent_state.id,
- messages=USER_MESSAGE,
+ messages=USER_MESSAGE_GREETING,
)
- assert_tool_response_messages(response.messages)
+ assert_greeting_with_assistant_message_response(response.messages)
[email protected]("llm_config", TESTED_LLM_CONFIGS)
-def test_send_message_streaming_sync_client(
[email protected]_client_test
+ "llm_config",
+ TESTED_LLM_CONFIGS,
+ ids=[c.model for c in TESTED_LLM_CONFIGS],
+)
+async def test_greeting_without_assistant_message_async_client(
+ disable_e2b_api_key: Any,
+ async_client: AsyncLetta,
+ agent_state: AgentState,
+ llm_config: LLMConfig,
+) -> None:
+ """
+ Tests sending a message with an asynchronous client.
+ Validates that the response messages match the expected sequence.
+ """
+ agent_state = await async_client.agents.modify(agent_id=agent_state.id, llm_config=llm_config)
+ response = await async_client.agents.messages.create(
+ agent_id=agent_state.id,
+ messages=USER_MESSAGE_GREETING,
+ use_assistant_message=False,
+ )
+ assert_greeting_without_assistant_message_response(response.messages)
+
+
[email protected]_client_test
+ "llm_config",
+ TESTED_LLM_CONFIGS,
+ ids=[c.model for c in TESTED_LLM_CONFIGS],
+)
+async def test_tool_call_async_client(
+ disable_e2b_api_key: Any,
+ async_client: AsyncLetta,
+ agent_state: AgentState,
+ llm_config: LLMConfig,
+) -> None:
+ """
+ Tests sending a message with an asynchronous client.
+ Validates that the response messages match the expected sequence.
+ """
+ dice_tool = await async_client.tools.upsert_from_function(func=roll_dice)
+ agent_state = await async_client.agents.tools.attach(agent_id=agent_state.id, tool_id=dice_tool.id)
+ agent_state = await async_client.agents.modify(agent_id=agent_state.id, llm_config=llm_config)
+ response = await async_client.agents.messages.create(
+ agent_id=agent_state.id,
+ messages=USER_MESSAGE_TOOL_CALL,
+ )
+ assert_tool_call_response(response.messages)
+
+
+ "llm_config",
+ TESTED_LLM_CONFIGS,
+ ids=[c.model for c in TESTED_LLM_CONFIGS],
+)
+def test_streaming_greeting_with_assistant_message(
disable_e2b_api_key: Any,
client: Letta,
agent_state: AgentState,
@@ -246,18 +409,270 @@ def test_send_message_streaming_sync_client(
Tests sending a streaming message with a synchronous client.
Checks that each chunk in the stream has the correct message types.
"""
- client.agents.modify(agent_id=agent_state.id, llm_config=llm_config)
+ agent_state = client.agents.modify(agent_id=agent_state.id, llm_config=llm_config)
+ response = client.agents.messages.create_stream(
+ agent_id=agent_state.id,
+ messages=USER_MESSAGE_GREETING,
+ )
+ chunks = list(response)
+ messages = accumulate_chunks(chunks)
+ assert_greeting_with_assistant_message_response(messages, streaming=True)
+
+
+ "llm_config",
+ TESTED_LLM_CONFIGS,
+ ids=[c.model for c in TESTED_LLM_CONFIGS],
+)
+def test_streaming_greeting_without_assistant_message(
+ disable_e2b_api_key: Any,
+ client: Letta,
+ agent_state: AgentState,
+ llm_config: LLMConfig,
+) -> None:
+ """
+ Tests sending a streaming message with a synchronous client.
+ Checks that each chunk in the stream has the correct message types.
+ """
+ agent_state = client.agents.modify(agent_id=agent_state.id, llm_config=llm_config)
+ response = client.agents.messages.create_stream(
+ agent_id=agent_state.id,
+ messages=USER_MESSAGE_GREETING,
+ use_assistant_message=False,
+ )
+ chunks = list(response)
+ messages = accumulate_chunks(chunks)
+ assert_greeting_without_assistant_message_response(messages, streaming=True)
+
+
+ "llm_config",
+ TESTED_LLM_CONFIGS,
+ ids=[c.model for c in TESTED_LLM_CONFIGS],
+)
+def test_streaming_tool_call(
+ disable_e2b_api_key: Any,
+ client: Letta,
+ agent_state: AgentState,
+ llm_config: LLMConfig,
+) -> None:
+ """
+ Tests sending a streaming message with a synchronous client.
+ Checks that each chunk in the stream has the correct message types.
+ """
+ dice_tool = client.tools.upsert_from_function(func=roll_dice)
+ agent_state = client.agents.tools.attach(agent_id=agent_state.id, tool_id=dice_tool.id)
+ agent_state = client.agents.modify(agent_id=agent_state.id, llm_config=llm_config)
+ response = client.agents.messages.create_stream(
+ agent_id=agent_state.id,
+ messages=USER_MESSAGE_TOOL_CALL,
+ )
+ chunks = list(response)
+ messages = accumulate_chunks(chunks)
+ assert_tool_call_response(messages, streaming=True)
+
+
[email protected]_client_test
+ "llm_config",
+ TESTED_LLM_CONFIGS,
+ ids=[c.model for c in TESTED_LLM_CONFIGS],
+)
+async def test_streaming_greeting_with_assistant_message_async_client(
+ disable_e2b_api_key: Any,
+ async_client: AsyncLetta,
+ agent_state: AgentState,
+ llm_config: LLMConfig,
+) -> None:
+ """
+ Tests sending a streaming message with an asynchronous client.
+ Validates that the streaming response chunks include the correct message types.
+ """
+ await async_client.agents.modify(agent_id=agent_state.id, llm_config=llm_config)
+ response = async_client.agents.messages.create_stream(
+ agent_id=agent_state.id,
+ messages=USER_MESSAGE_GREETING,
+ )
+ chunks = [chunk async for chunk in response]
+ messages = accumulate_chunks(chunks)
+ assert_greeting_with_assistant_message_response(messages, streaming=True)
+
+
[email protected]_client_test
+ "llm_config",
+ TESTED_LLM_CONFIGS,
+ ids=[c.model for c in TESTED_LLM_CONFIGS],
+)
+async def test_streaming_greeting_without_assistant_message_async_client(
+ disable_e2b_api_key: Any,
+ async_client: AsyncLetta,
+ agent_state: AgentState,
+ llm_config: LLMConfig,
+) -> None:
+ """
+ Tests sending a streaming message with an asynchronous client.
+ Validates that the streaming response chunks include the correct message types.
+ """
+ await async_client.agents.modify(agent_id=agent_state.id, llm_config=llm_config)
+ response = async_client.agents.messages.create_stream(
+ agent_id=agent_state.id,
+ messages=USER_MESSAGE_GREETING,
+ use_assistant_message=False,
+ )
+ chunks = [chunk async for chunk in response]
+ messages = accumulate_chunks(chunks)
+ assert_greeting_without_assistant_message_response(messages, streaming=True)
+
+
[email protected]_client_test
+ "llm_config",
+ TESTED_LLM_CONFIGS,
+ ids=[c.model for c in TESTED_LLM_CONFIGS],
+)
+async def test_streaming_tool_call_async_client(
+ disable_e2b_api_key: Any,
+ async_client: AsyncLetta,
+ agent_state: AgentState,
+ llm_config: LLMConfig,
+) -> None:
+ """
+ Tests sending a streaming message with an asynchronous client.
+ Validates that the streaming response chunks include the correct message types.
+ """
+ dice_tool = await async_client.tools.upsert_from_function(func=roll_dice)
+ agent_state = await async_client.agents.tools.attach(agent_id=agent_state.id, tool_id=dice_tool.id)
+ agent_state = await async_client.agents.modify(agent_id=agent_state.id, llm_config=llm_config)
+ response = async_client.agents.messages.create_stream(
+ agent_id=agent_state.id,
+ messages=USER_MESSAGE_TOOL_CALL,
+ )
+ chunks = [chunk async for chunk in response]
+ messages = accumulate_chunks(chunks)
+ assert_tool_call_response(messages, streaming=True)
+
+
+ "llm_config",
+ TESTED_LLM_CONFIGS,
+ ids=[c.model for c in TESTED_LLM_CONFIGS],
+)
+def test_token_streaming_greeting_with_assistant_message(
+ disable_e2b_api_key: Any,
+ client: Letta,
+ agent_state: AgentState,
+ llm_config: LLMConfig,
+) -> None:
+ """
+ Tests sending a streaming message with a synchronous client.
+ Checks that each chunk in the stream has the correct message types.
+ """
+ agent_state = client.agents.modify(agent_id=agent_state.id, llm_config=llm_config)
response = client.agents.messages.create_stream(
agent_id=agent_state.id,
- messages=USER_MESSAGE,
+ messages=USER_MESSAGE_GREETING,
+ stream_tokens=True,
)
chunks = list(response)
- assert_streaming_tool_response_messages(chunks)
+ messages = accumulate_chunks(chunks)
+ assert_greeting_with_assistant_message_response(messages, streaming=True)
+
+
+ "llm_config",
+ TESTED_LLM_CONFIGS,
+ ids=[c.model for c in TESTED_LLM_CONFIGS],
+)
+def test_token_streaming_greeting_without_assistant_message(
+ disable_e2b_api_key: Any,
+ client: Letta,
+ agent_state: AgentState,
+ llm_config: LLMConfig,
+) -> None:
+ """
+ Tests sending a streaming message with a synchronous client.
+ Checks that each chunk in the stream has the correct message types.
+ """
+ agent_state = client.agents.modify(agent_id=agent_state.id, llm_config=llm_config)
+ response = client.agents.messages.create_stream(
+ agent_id=agent_state.id,
+ messages=USER_MESSAGE_GREETING,
+ use_assistant_message=False,
+ stream_tokens=True,
+ )
+ chunks = list(response)
+ messages = accumulate_chunks(chunks)
+ assert_greeting_without_assistant_message_response(messages, streaming=True)
+
+
+ "llm_config",
+ TESTED_LLM_CONFIGS,
+ ids=[c.model for c in TESTED_LLM_CONFIGS],
+)
+def test_token_streaming_tool_call(
+ disable_e2b_api_key: Any,
+ client: Letta,
+ agent_state: AgentState,
+ llm_config: LLMConfig,
+) -> None:
+ """
+ Tests sending a streaming message with a synchronous client.
+ Checks that each chunk in the stream has the correct message types.
+ """
+ dice_tool = client.tools.upsert_from_function(func=roll_dice)
+ agent_state = client.agents.tools.attach(agent_id=agent_state.id, tool_id=dice_tool.id)
+ agent_state = client.agents.modify(agent_id=agent_state.id, llm_config=llm_config)
+ response = client.agents.messages.create_stream(
+ agent_id=agent_state.id,
+ messages=USER_MESSAGE_TOOL_CALL,
+ stream_tokens=True,
+ )
+ chunks = list(response)
+ messages = accumulate_chunks(chunks)
+ assert_tool_call_response(messages, streaming=True)
+
+
[email protected]_client_test
+ "llm_config",
+ TESTED_LLM_CONFIGS,
+ ids=[c.model for c in TESTED_LLM_CONFIGS],
+)
+async def test_token_streaming_greeting_with_assistant_message_async_client(
+ disable_e2b_api_key: Any,
+ async_client: AsyncLetta,
+ agent_state: AgentState,
+ llm_config: LLMConfig,
+) -> None:
+ """
+ Tests sending a streaming message with an asynchronous client.
+ Validates that the streaming response chunks include the correct message types.
+ """
+ await async_client.agents.modify(agent_id=agent_state.id, llm_config=llm_config)
+ response = async_client.agents.messages.create_stream(
+ agent_id=agent_state.id,
+ messages=USER_MESSAGE_GREETING,
+ stream_tokens=True,
+ )
+ chunks = [chunk async for chunk in response]
+ messages = accumulate_chunks(chunks)
+ assert_greeting_with_assistant_message_response(messages, streaming=True)
@pytest.mark.asyncio
[email protected]("llm_config", TESTED_LLM_CONFIGS)
-async def test_send_message_streaming_async_client(
[email protected]_client_test
+ "llm_config",
+ TESTED_LLM_CONFIGS,
+ ids=[c.model for c in TESTED_LLM_CONFIGS],
+)
+async def test_token_streaming_greeting_without_assistant_message_async_client(
disable_e2b_api_key: Any,
async_client: AsyncLetta,
agent_state: AgentState,
@@ -270,14 +685,51 @@ async def test_send_message_streaming_async_client(
await async_client.agents.modify(agent_id=agent_state.id, llm_config=llm_config)
response = async_client.agents.messages.create_stream(
agent_id=agent_state.id,
- messages=USER_MESSAGE,
+ messages=USER_MESSAGE_GREETING,
+ use_assistant_message=False,
+ stream_tokens=True,
+ )
+ chunks = [chunk async for chunk in response]
+ messages = accumulate_chunks(chunks)
+ assert_greeting_without_assistant_message_response(messages, streaming=True)
+
+
[email protected]_client_test
+ "llm_config",
+ TESTED_LLM_CONFIGS,
+ ids=[c.model for c in TESTED_LLM_CONFIGS],
+)
+async def test_token_streaming_tool_call_async_client(
+ disable_e2b_api_key: Any,
+ async_client: AsyncLetta,
+ agent_state: AgentState,
+ llm_config: LLMConfig,
+) -> None:
+ """
+ Tests sending a streaming message with an asynchronous client.
+ Validates that the streaming response chunks include the correct message types.
+ """
+ dice_tool = await async_client.tools.upsert_from_function(func=roll_dice)
+ agent_state = await async_client.agents.tools.attach(agent_id=agent_state.id, tool_id=dice_tool.id)
+ agent_state = await async_client.agents.modify(agent_id=agent_state.id, llm_config=llm_config)
+ response = async_client.agents.messages.create_stream(
+ agent_id=agent_state.id,
+ messages=USER_MESSAGE_TOOL_CALL,
+ stream_tokens=True,
)
chunks = [chunk async for chunk in response]
- assert_streaming_tool_response_messages(chunks)
+ messages = accumulate_chunks(chunks)
+ assert_tool_call_response(messages, streaming=True)
[email protected]("llm_config", TESTED_LLM_CONFIGS)
-def test_send_message_job_sync_client(
+ "llm_config",
+ TESTED_LLM_CONFIGS,
+ ids=[c.model for c in TESTED_LLM_CONFIGS],
+)
+def test_async_greeting_with_assistant_message(
disable_e2b_api_key: Any,
client: Letta,
agent_state: AgentState,
@@ -291,7 +743,7 @@ def test_send_message_job_sync_client(
run = client.agents.messages.create_async(
agent_id=agent_state.id,
- messages=USER_MESSAGE,
+ messages=USER_MESSAGE_GREETING,
)
run = wait_for_run_completion(client, run.id)
@@ -303,8 +755,13 @@ def test_send_message_job_sync_client(
@pytest.mark.asyncio
[email protected]("llm_config", TESTED_LLM_CONFIGS)
-async def test_send_message_job_async_client(
[email protected]_client_test
+ "llm_config",
+ TESTED_LLM_CONFIGS,
+ ids=[c.model for c in TESTED_LLM_CONFIGS],
+)
+async def test_async_greeting_with_assistant_message_async_client(
disable_e2b_api_key: Any,
client: Letta,
async_client: AsyncLetta,
@@ -319,7 +776,7 @@ async def test_send_message_job_async_client(
run = await async_client.agents.messages.create_async(
agent_id=agent_state.id,
- messages=USER_MESSAGE,
+ messages=USER_MESSAGE_GREETING,
)
# Use the synchronous client to check job completion
run = wait_for_run_completion(client, run.id)
diff --git a/tests/pytest.ini b/tests/pytest.ini
index 299f27d4e..bce6a52b6 100644
--- a/tests/pytest.ini
+++ b/tests/pytest.ini
@@ -11,3 +11,6 @@ markers =
anthropic_basic: Tests for Anthropic endpoints
azure_basic: Tests for Azure endpoints
gemini_basic: Tests for Gemini endpoints
+ async_client_test: mark test as an async client test that is skipped by default
+
+addopts = -m "not async_client_test"