Platform for stateful agents: AI with advanced memory that can learn and self-improve over time.
test(include_pings): Add long-running tests (#3899)
Kian Jones8aceaf677be0
diff --git a/tests/helpers/utils.py b/tests/helpers/utils.pyindex 36918df76..f47b7e27c 100644--- a/tests/helpers/utils.py+++ b/tests/helpers/utils.py@@ -1,7 +1,10 @@import functools+import osimport timefrom typing import Optional, Union+import requests+from letta.functions.functions import parse_source_codefrom letta.functions.schema_generator import generate_schemafrom letta.schemas.agent import AgentState, CreateAgent, UpdateAgent@@ -11,6 +14,7 @@ from letta.schemas.memory import ContextWindowOverviewfrom letta.schemas.tool import Toolfrom letta.schemas.user import Userfrom letta.schemas.user import User as PydanticUser+from letta.server.rest_api.routers.v1.agents import ImportedAgentsResponsefrom letta.server.server import SyncServer@@ -248,3 +252,29 @@ def validate_context_window_overview(# Check for toolsassert overview.num_tokens_functions_definitions > 0assert len(overview.functions_definitions) > 0+++def upload_test_agentfile_from_disk(server_url: str, filename: str) -> ImportedAgentsResponse:+ """+ Upload a given .af file to live FastAPI server.+ """+ path_to_current_file = os.path.dirname(__file__)+ path_to_test_agent_files = path_to_current_file.removesuffix("/helpers") + "/test_agent_files"+ file_path = os.path.join(path_to_test_agent_files, filename)++ with open(file_path, "rb") as f:+ files = {"file": (filename, f, "application/json")}++ # Send parameters as form data instead of query parameters+ form_data = {+ "append_copy_suffix": "true",+ "override_existing_tools": "false",+ }++ response = requests.post(+ f"{server_url}/v1/agents/import",+ headers={"user_id": ""},+ files=files,+ data=form_data, # Send as form data+ )+ return ImportedAgentsResponse(**response.json())diff --git a/tests/integration_test_pinecone_tool.py b/tests/integration_test_pinecone_tool.pyindex 3caf4d5f9..c37bd3829 100644--- a/tests/integration_test_pinecone_tool.py+++ b/tests/integration_test_pinecone_tool.py@@ -10,6 +10,8 @@ from dotenv import load_dotenvfrom letta_client import AsyncLetta, MessageCreate, ReasoningMessage, ToolCallMessagefrom letta_client.core import RequestOptions+from tests.helpers.utils import upload_test_agentfile_from_disk+REASONING_THROTTLE_MS = 100TEST_USER_MESSAGE = "What products or services does 11x AI sell?"@@ -19,7 +21,7 @@ def server_url() -> str:"""Provides the URL for the Letta server.If LETTA_SERVER_URL is not set, starts the server in a background thread- and polls until it’s accepting connections.+ and polls until it's accepting connections."""def _run_server() -> None:@@ -60,12 +62,11 @@ def client(server_url: str):yield async_client_instance-async def test_pinecone_tool(client: AsyncLetta) -> None:+async def test_pinecone_tool(client: AsyncLetta, server_url: str) -> None:"""Test the Pinecone tool integration with the Letta client."""- with open("../../scripts/test-afs/knowledge-base.af", "rb") as f:- response = await client.agents.import_file(file=f)+ response = upload_test_agentfile_from_disk(server_url, "knowledge-base.af")agent_id = response.agent_ids[0]diff --git a/tests/test_long_running_agents.py b/tests/test_long_running_agents.pynew file mode 100644index 000000000..04f0b9d68--- /dev/null+++ b/tests/test_long_running_agents.py@@ -0,0 +1,121 @@+import os+import threading+import time++import httpx+import pytest+import requests+from dotenv import load_dotenv+from letta_client import Letta, MessageCreate, TextContent++from tests.helpers.utils import upload_test_agentfile_from_disk++RESEARCH_INSTRUCTIONS = "\n Lead Name: Kian Jones\n Lead Title: Software Engineer\n Lead LinkedIn URL: https://www.linkedin.com/in/kian-jones\n Company Name: Letta\n Company Domain: letta.com\n Company Industry: technology/software/ai\n \n**Research Instructions**\n"+DEEP_RESEARCH_INSTRUCTIONS = "Let's get started, we have to research mantis shrimps. I need to know everything there is, or my grandmother will die. Please begin immediately and do a great job, they are scaring me."++[email protected](scope="module")+def server_url() -> str:+ """+ Provides the URL for the Letta server.+ If LETTA_SERVER_URL is not set, starts the server in a background thread+ and polls until it's accepting connections.+ """++ def _run_server() -> None:+ load_dotenv()+ from letta.server.rest_api.app import start_server++ start_server(debug=True)++ url: str = os.getenv("LETTA_SERVER_URL", "http://localhost:8283")++ if not os.getenv("LETTA_SERVER_URL"):+ thread = threading.Thread(target=_run_server, daemon=True)+ thread.start()++ # Poll until the server is up (or timeout)+ timeout_seconds = 30+ deadline = time.time() + timeout_seconds+ while time.time() < deadline:+ try:+ resp = requests.get(url + "/v1/health")+ if resp.status_code < 500:+ break+ except requests.exceptions.RequestException:+ pass+ time.sleep(0.1)+ else:+ raise RuntimeError(f"Could not reach {url} within {timeout_seconds}s")++ return url++[email protected](scope="module")+def client(server_url: str) -> Letta:+ """+ Creates and returns a synchronous Letta REST client for testing.+ """+ client_instance = Letta(base_url=server_url)+ return client_instance+++async def test_deep_research_agent(client, server_url, disable_e2b_api_key):+ imported_af = upload_test_agentfile_from_disk(server_url, "deep-thought.af")++ agent_id = imported_af.agent_ids[0]++ try:+ response = client.agents.messages.create_stream(+ agent_id=agent_id,+ stream_tokens=True,+ messages=[+ MessageCreate(+ role="user",+ content=[+ TextContent(+ text=DEEP_RESEARCH_INSTRUCTIONS,+ )+ ],+ )+ ],+ )+ for chunk in response:+ if chunk.message_type is not None:+ print(chunk)+ except httpx.ReadTimeout as e:+ print("Timeout on create_stream. Consider enabling pings in create_stream if you have long running agents. ", e)+ assert False+ finally:+ client.agents.delete(agent_id=agent_id)+++async def test_11x_agent(client, server_url, disable_e2b_api_key):+ imported_af = upload_test_agentfile_from_disk(server_url, "mock_alice.af")++ agent_id = imported_af.agent_ids[0]++ try:+ response = client.agents.messages.create_stream(+ agent_id=agent_id,+ include_pings=True,+ stream_tokens=True,+ messages=[+ MessageCreate(+ role="user",+ content=[+ TextContent(+ text=RESEARCH_INSTRUCTIONS,+ )+ ],+ )+ ],+ )+ for chunk in response:+ if chunk.message_type is not None:+ print(chunk)+ except httpx.ReadTimeout as e:+ print("Timeout on create_stream. Consider enabling pings in create_stream if you have long running agents. ", e)+ assert False+ finally:+ client.agents.delete(agent_id=agent_id)