Skip to content

armgit / letta

Public Imported from GitHub
0

Platform for stateful agents: AI with advanced memory that can learn and self-improve over time.

test: clean up send message tests (#2700)

cthomas2a50ea224fa9
diff --git a/tests/conftest.py b/tests/conftest.py
index cb25bb85b..cfca0ebfc 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -13,9 +13,6 @@ from letta.settings import tool_settings
def pytest_configure(config):
logging.basicConfig(level=logging.DEBUG)
- # Register custom markers
- config.addinivalue_line("markers", "async_client_test: mark test as an async client test that is skipped by default")
-
@pytest.fixture
def disable_e2b_api_key() -> Generator[None, None, None]:
diff --git a/tests/integration_test_send_message.py b/tests/integration_test_send_message.py
index efa48720f..4efc9ed7c 100644
--- a/tests/integration_test_send_message.py
+++ b/tests/integration_test_send_message.py
@@ -420,90 +420,12 @@ def test_tool_call(
assert_tool_call_response(messages_from_db, from_db=True)
[email protected]_client_test
@pytest.mark.parametrize(
"llm_config",
TESTED_LLM_CONFIGS,
ids=[c.model for c in TESTED_LLM_CONFIGS],
)
-async def test_greeting_with_assistant_message_async_client(
- disable_e2b_api_key: Any,
- async_client: AsyncLetta,
- agent_state: AgentState,
- llm_config: LLMConfig,
-) -> None:
- """
- Tests sending a message with an asynchronous client.
- Validates that the response messages match the expected sequence.
- """
- agent_state = await async_client.agents.modify(agent_id=agent_state.id, llm_config=llm_config)
- response = await async_client.agents.messages.create(
- agent_id=agent_state.id,
- messages=USER_MESSAGE_FORCE_REPLY,
- )
- assert_greeting_with_assistant_message_response(response.messages)
-
-
[email protected]_client_test
- "llm_config",
- TESTED_LLM_CONFIGS,
- ids=[c.model for c in TESTED_LLM_CONFIGS],
-)
-async def test_greeting_without_assistant_message_async_client(
- disable_e2b_api_key: Any,
- async_client: AsyncLetta,
- agent_state: AgentState,
- llm_config: LLMConfig,
-) -> None:
- """
- Tests sending a message with an asynchronous client.
- Validates that the response messages match the expected sequence.
- """
- agent_state = await async_client.agents.modify(agent_id=agent_state.id, llm_config=llm_config)
- response = await async_client.agents.messages.create(
- agent_id=agent_state.id,
- messages=USER_MESSAGE_FORCE_REPLY,
- use_assistant_message=False,
- )
- assert_greeting_without_assistant_message_response(response.messages)
-
-
[email protected]_client_test
- "llm_config",
- TESTED_LLM_CONFIGS,
- ids=[c.model for c in TESTED_LLM_CONFIGS],
-)
-async def test_tool_call_async_client(
- disable_e2b_api_key: Any,
- async_client: AsyncLetta,
- agent_state: AgentState,
- llm_config: LLMConfig,
-) -> None:
- """
- Tests sending a message with an asynchronous client.
- Validates that the response messages match the expected sequence.
- """
- dice_tool = await async_client.tools.upsert_from_function(func=roll_dice)
- agent_state = await async_client.agents.tools.attach(agent_id=agent_state.id, tool_id=dice_tool.id)
- agent_state = await async_client.agents.modify(agent_id=agent_state.id, llm_config=llm_config)
- response = await async_client.agents.messages.create(
- agent_id=agent_state.id,
- messages=USER_MESSAGE_ROLL_DICE,
- )
- assert_tool_call_response(response.messages)
-
-
- "llm_config",
- TESTED_LLM_CONFIGS,
- ids=[c.model for c in TESTED_LLM_CONFIGS],
-)
-def test_streaming_greeting_with_assistant_message(
+def test_step_streaming_greeting_with_assistant_message(
disable_e2b_api_key: Any,
client: Letta,
agent_state: AgentState,
@@ -513,6 +435,7 @@ def test_streaming_greeting_with_assistant_message(
Tests sending a streaming message with a synchronous client.
Checks that each chunk in the stream has the correct message types.
"""
+ last_message = client.agents.messages.list(agent_id=agent_state.id, limit=1)
agent_state = client.agents.modify(agent_id=agent_state.id, llm_config=llm_config)
response = client.agents.messages.create_stream(
agent_id=agent_state.id,
@@ -521,6 +444,8 @@ def test_streaming_greeting_with_assistant_message(
chunks = list(response)
messages = accumulate_chunks(chunks)
assert_greeting_with_assistant_message_response(messages, streaming=True)
+ messages_from_db = client.agents.messages.list(agent_id=agent_state.id, after=last_message[0].id)
+ assert_greeting_with_assistant_message_response(messages_from_db, from_db=True)
@pytest.mark.parametrize(
@@ -528,7 +453,7 @@ def test_streaming_greeting_with_assistant_message(
TESTED_LLM_CONFIGS,
ids=[c.model for c in TESTED_LLM_CONFIGS],
)
-def test_streaming_greeting_without_assistant_message(
+def test_step_streaming_greeting_without_assistant_message(
disable_e2b_api_key: Any,
client: Letta,
agent_state: AgentState,
@@ -538,6 +463,7 @@ def test_streaming_greeting_without_assistant_message(
Tests sending a streaming message with a synchronous client.
Checks that each chunk in the stream has the correct message types.
"""
+ last_message = client.agents.messages.list(agent_id=agent_state.id, limit=1)
agent_state = client.agents.modify(agent_id=agent_state.id, llm_config=llm_config)
response = client.agents.messages.create_stream(
agent_id=agent_state.id,
@@ -547,6 +473,8 @@ def test_streaming_greeting_without_assistant_message(
chunks = list(response)
messages = accumulate_chunks(chunks)
assert_greeting_without_assistant_message_response(messages, streaming=True)
+ messages_from_db = client.agents.messages.list(agent_id=agent_state.id, after=last_message[0].id, use_assistant_message=False)
+ assert_greeting_without_assistant_message_response(messages_from_db, from_db=True)
@pytest.mark.parametrize(
@@ -554,7 +482,7 @@ def test_streaming_greeting_without_assistant_message(
TESTED_LLM_CONFIGS,
ids=[c.model for c in TESTED_LLM_CONFIGS],
)
-def test_streaming_tool_call(
+def test_step_streaming_tool_call(
disable_e2b_api_key: Any,
client: Letta,
agent_state: AgentState,
@@ -566,6 +494,7 @@ def test_streaming_tool_call(
"""
dice_tool = client.tools.upsert_from_function(func=roll_dice)
agent_state = client.agents.tools.attach(agent_id=agent_state.id, tool_id=dice_tool.id)
+ last_message = client.agents.messages.list(agent_id=agent_state.id, limit=1)
agent_state = client.agents.modify(agent_id=agent_state.id, llm_config=llm_config)
response = client.agents.messages.create_stream(
agent_id=agent_state.id,
@@ -574,117 +503,8 @@ def test_streaming_tool_call(
chunks = list(response)
messages = accumulate_chunks(chunks)
assert_tool_call_response(messages, streaming=True)
-
-
[email protected]_client_test
- "llm_config",
- TESTED_LLM_CONFIGS,
- ids=[c.model for c in TESTED_LLM_CONFIGS],
-)
-async def test_streaming_greeting_with_assistant_message_async_client(
- disable_e2b_api_key: Any,
- async_client: AsyncLetta,
- agent_state: AgentState,
- llm_config: LLMConfig,
-) -> None:
- """
- Tests sending a streaming message with an asynchronous client.
- Validates that the streaming response chunks include the correct message types.
- """
- await async_client.agents.modify(agent_id=agent_state.id, llm_config=llm_config)
- response = async_client.agents.messages.create_stream(
- agent_id=agent_state.id,
- messages=USER_MESSAGE_FORCE_REPLY,
- )
- chunks = [chunk async for chunk in response]
- messages = accumulate_chunks(chunks)
- assert_greeting_with_assistant_message_response(messages, streaming=True)
-
-
[email protected]_client_test
- "llm_config",
- TESTED_LLM_CONFIGS,
- ids=[c.model for c in TESTED_LLM_CONFIGS],
-)
-async def test_streaming_greeting_without_assistant_message_async_client(
- disable_e2b_api_key: Any,
- async_client: AsyncLetta,
- agent_state: AgentState,
- llm_config: LLMConfig,
-) -> None:
- """
- Tests sending a streaming message with an asynchronous client.
- Validates that the streaming response chunks include the correct message types.
- """
- await async_client.agents.modify(agent_id=agent_state.id, llm_config=llm_config)
- response = async_client.agents.messages.create_stream(
- agent_id=agent_state.id,
- messages=USER_MESSAGE_FORCE_REPLY,
- use_assistant_message=False,
- )
- chunks = [chunk async for chunk in response]
- messages = accumulate_chunks(chunks)
- assert_greeting_without_assistant_message_response(messages, streaming=True)
-
-
[email protected]_client_test
- "llm_config",
- TESTED_LLM_CONFIGS,
- ids=[c.model for c in TESTED_LLM_CONFIGS],
-)
-async def test_streaming_tool_call_async_client(
- disable_e2b_api_key: Any,
- async_client: AsyncLetta,
- agent_state: AgentState,
- llm_config: LLMConfig,
-) -> None:
- """
- Tests sending a streaming message with an asynchronous client.
- Validates that the streaming response chunks include the correct message types.
- """
- dice_tool = await async_client.tools.upsert_from_function(func=roll_dice)
- agent_state = await async_client.agents.tools.attach(agent_id=agent_state.id, tool_id=dice_tool.id)
- agent_state = await async_client.agents.modify(agent_id=agent_state.id, llm_config=llm_config)
- response = async_client.agents.messages.create_stream(
- agent_id=agent_state.id,
- messages=USER_MESSAGE_ROLL_DICE,
- )
- chunks = [chunk async for chunk in response]
- messages = accumulate_chunks(chunks)
- assert_tool_call_response(messages, streaming=True)
-
-
- "llm_config",
- TESTED_LLM_CONFIGS,
- ids=[c.model for c in TESTED_LLM_CONFIGS],
-)
-def test_step_streaming_greeting_with_assistant_message(
- disable_e2b_api_key: Any,
- client: Letta,
- agent_state: AgentState,
- llm_config: LLMConfig,
-) -> None:
- """
- Tests sending a streaming message with a synchronous client.
- Checks that each chunk in the stream has the correct message types.
- """
- agent_state = client.agents.modify(agent_id=agent_state.id, llm_config=llm_config)
- response = client.agents.messages.create_stream(
- agent_id=agent_state.id,
- messages=USER_MESSAGE_FORCE_REPLY,
- stream_tokens=False,
- )
- messages = []
- for message in response:
- messages.append(message)
- assert_greeting_with_assistant_message_response(messages, streaming=True)
+ messages_from_db = client.agents.messages.list(agent_id=agent_state.id, after=last_message[0].id)
+ assert_tool_call_response(messages_from_db, from_db=True)
@pytest.mark.parametrize(
@@ -702,6 +522,7 @@ def test_token_streaming_greeting_with_assistant_message(
Tests sending a streaming message with a synchronous client.
Checks that each chunk in the stream has the correct message types.
"""
+ last_message = client.agents.messages.list(agent_id=agent_state.id, limit=1)
agent_state = client.agents.modify(agent_id=agent_state.id, llm_config=llm_config)
response = client.agents.messages.create_stream(
agent_id=agent_state.id,
@@ -711,6 +532,8 @@ def test_token_streaming_greeting_with_assistant_message(
chunks = list(response)
messages = accumulate_chunks(chunks)
assert_greeting_with_assistant_message_response(messages, streaming=True, token_streaming=True)
+ messages_from_db = client.agents.messages.list(agent_id=agent_state.id, after=last_message[0].id)
+ assert_greeting_with_assistant_message_response(messages_from_db, from_db=True)
@pytest.mark.parametrize(
@@ -728,6 +551,7 @@ def test_token_streaming_greeting_without_assistant_message(
Tests sending a streaming message with a synchronous client.
Checks that each chunk in the stream has the correct message types.
"""
+ last_message = client.agents.messages.list(agent_id=agent_state.id, limit=1)
agent_state = client.agents.modify(agent_id=agent_state.id, llm_config=llm_config)
response = client.agents.messages.create_stream(
agent_id=agent_state.id,
@@ -738,6 +562,8 @@ def test_token_streaming_greeting_without_assistant_message(
chunks = list(response)
messages = accumulate_chunks(chunks)
assert_greeting_without_assistant_message_response(messages, streaming=True, token_streaming=True)
+ messages_from_db = client.agents.messages.list(agent_id=agent_state.id, after=last_message[0].id, use_assistant_message=False)
+ assert_greeting_without_assistant_message_response(messages_from_db, from_db=True)
@pytest.mark.parametrize(
@@ -757,6 +583,7 @@ def test_token_streaming_tool_call(
"""
dice_tool = client.tools.upsert_from_function(func=roll_dice)
agent_state = client.agents.tools.attach(agent_id=agent_state.id, tool_id=dice_tool.id)
+ last_message = client.agents.messages.list(agent_id=agent_state.id, limit=1)
agent_state = client.agents.modify(agent_id=agent_state.id, llm_config=llm_config)
response = client.agents.messages.create_stream(
agent_id=agent_state.id,
@@ -766,93 +593,8 @@ def test_token_streaming_tool_call(
chunks = list(response)
messages = accumulate_chunks(chunks)
assert_tool_call_response(messages, streaming=True)
-
-
[email protected]_client_test
- "llm_config",
- TESTED_LLM_CONFIGS,
- ids=[c.model for c in TESTED_LLM_CONFIGS],
-)
-async def test_token_streaming_greeting_with_assistant_message_async_client(
- disable_e2b_api_key: Any,
- async_client: AsyncLetta,
- agent_state: AgentState,
- llm_config: LLMConfig,
-) -> None:
- """
- Tests sending a streaming message with an asynchronous client.
- Validates that the streaming response chunks include the correct message types.
- """
- await async_client.agents.modify(agent_id=agent_state.id, llm_config=llm_config)
- response = async_client.agents.messages.create_stream(
- agent_id=agent_state.id,
- messages=USER_MESSAGE_FORCE_REPLY,
- stream_tokens=True,
- )
- chunks = [chunk async for chunk in response]
- messages = accumulate_chunks(chunks)
- assert_greeting_with_assistant_message_response(messages, streaming=True)
-
-
[email protected]_client_test
- "llm_config",
- TESTED_LLM_CONFIGS,
- ids=[c.model for c in TESTED_LLM_CONFIGS],
-)
-async def test_token_streaming_greeting_without_assistant_message_async_client(
- disable_e2b_api_key: Any,
- async_client: AsyncLetta,
- agent_state: AgentState,
- llm_config: LLMConfig,
-) -> None:
- """
- Tests sending a streaming message with an asynchronous client.
- Validates that the streaming response chunks include the correct message types.
- """
- await async_client.agents.modify(agent_id=agent_state.id, llm_config=llm_config)
- response = async_client.agents.messages.create_stream(
- agent_id=agent_state.id,
- messages=USER_MESSAGE_FORCE_REPLY,
- use_assistant_message=False,
- stream_tokens=True,
- )
- chunks = [chunk async for chunk in response]
- messages = accumulate_chunks(chunks)
- assert_greeting_without_assistant_message_response(messages, streaming=True)
-
-
[email protected]_client_test
- "llm_config",
- TESTED_LLM_CONFIGS,
- ids=[c.model for c in TESTED_LLM_CONFIGS],
-)
-async def test_token_streaming_tool_call_async_client(
- disable_e2b_api_key: Any,
- async_client: AsyncLetta,
- agent_state: AgentState,
- llm_config: LLMConfig,
-) -> None:
- """
- Tests sending a streaming message with an asynchronous client.
- Validates that the streaming response chunks include the correct message types.
- """
- dice_tool = await async_client.tools.upsert_from_function(func=roll_dice)
- agent_state = await async_client.agents.tools.attach(agent_id=agent_state.id, tool_id=dice_tool.id)
- agent_state = await async_client.agents.modify(agent_id=agent_state.id, llm_config=llm_config)
- response = async_client.agents.messages.create_stream(
- agent_id=agent_state.id,
- messages=USER_MESSAGE_ROLL_DICE,
- stream_tokens=True,
- )
- chunks = [chunk async for chunk in response]
- messages = accumulate_chunks(chunks)
- assert_tool_call_response(messages, streaming=True)
+ messages_from_db = client.agents.messages.list(agent_id=agent_state.id, after=last_message[0].id)
+ assert_tool_call_response(messages_from_db, from_db=True)
@pytest.mark.parametrize(
@@ -885,47 +627,12 @@ def test_async_greeting_with_assistant_message(
assert_tool_response_dict_messages(messages)
[email protected]_client_test
- "llm_config",
- TESTED_LLM_CONFIGS,
- ids=[c.model for c in TESTED_LLM_CONFIGS],
-)
-async def test_async_greeting_with_assistant_message_async_client(
- disable_e2b_api_key: Any,
- client: Letta,
- async_client: AsyncLetta,
- agent_state: AgentState,
- llm_config: LLMConfig,
-) -> None:
- """
- Tests sending a message as an asynchronous job using the asynchronous client.
- Waits for job completion and verifies that the resulting messages meet the expected format.
- """
- await async_client.agents.modify(agent_id=agent_state.id, llm_config=llm_config)
-
- run = await async_client.agents.messages.create_async(
- agent_id=agent_state.id,
- messages=USER_MESSAGE_FORCE_REPLY,
- )
- # Use the synchronous client to check job completion
- run = wait_for_run_completion(client, run.id)
-
- result = run.metadata.get("result")
- assert result is not None, "Run metadata missing 'result' key"
-
- messages = result["messages"]
- assert_tool_response_dict_messages(messages)
-
-
@pytest.mark.parametrize(
"llm_config",
TESTED_LLM_CONFIGS,
ids=[c.model for c in TESTED_LLM_CONFIGS],
)
-async def test_auto_summarize(disable_e2b_api_key: Any, client: Letta, llm_config: LLMConfig):
+def test_auto_summarize(disable_e2b_api_key: Any, client: Letta, llm_config: LLMConfig):
"""Test that summarization is automatically triggered."""
llm_config.context_window = 3000
client.tools.upsert_base_tools()
diff --git a/tests/pytest.ini b/tests/pytest.ini
index bce6a52b6..299f27d4e 100644
--- a/tests/pytest.ini
+++ b/tests/pytest.ini
@@ -11,6 +11,3 @@ markers =
anthropic_basic: Tests for Anthropic endpoints
azure_basic: Tests for Azure endpoints
gemini_basic: Tests for Gemini endpoints
- async_client_test: mark test as an async client test that is skipped by default
-
-addopts = -m "not async_client_test"