Platform for stateful agents: AI with advanced memory that can learn and self-improve over time.
feat: Fix test batch sdkpy [LET-4507] (#4917)
Matthew Zhoud78e0ccb583d
diff --git a/.github/workflows/core-integration-tests.yml b/.github/workflows/core-integration-tests.ymlindex 7be6a4523..a0d806b2e 100644--- a/.github/workflows/core-integration-tests.yml+++ b/.github/workflows/core-integration-tests.yml@@ -41,7 +41,6 @@ jobs:"integration_test_chat_completions.py","integration_test_multi_agent.py","integration_test_batch_api_cron_jobs.py",- "integration_test_batch_sdk.py","integration_test_builtin_tools.py","integration_test_turbopuffer.py","integration_test_human_in_the_loop.py"diff --git a/letta/server/rest_api/routers/v1/messages.py b/letta/server/rest_api/routers/v1/messages.pyindex f5c52cae1..e610df283 100644--- a/letta/server/rest_api/routers/v1/messages.py+++ b/letta/server/rest_api/routers/v1/messages.py@@ -122,7 +122,7 @@ async def list_batches("""actor = await server.user_manager.get_actor_or_default_async(actor_id=headers.actor_id)- jobs = server.job_manager.list_jobs(+ jobs = await server.job_manager.list_jobs_async(actor=actor,statuses=[JobStatus.created, JobStatus.running],job_type=JobType.BATCH,diff --git a/tests/integration_test_batch_sdk.py b/tests/integration_test_batch_sdk.pydeleted file mode 100644index e70435ded..000000000--- a/tests/integration_test_batch_sdk.py+++ /dev/null@@ -1,139 +0,0 @@-import os-import threading-import time--import pytest-from dotenv import load_dotenv-from letta_client import Letta, LettaBatchRequest, MessageCreate, TextContent--from letta.config import LettaConfig-from letta.jobs.llm_batch_job_polling import poll_running_llm_batches-from letta.orm import Base-from letta.schemas.enums import JobStatus-from letta.server.db import db_context-from letta.server.server import SyncServer--[email protected](autouse=True)-def clear_batch_tables():- """Clear batch-related tables before each test."""- with db_context() as session:- for table in reversed(Base.metadata.sorted_tables):- if table.name in {"jobs", "llm_batch_job", "llm_batch_items"}:- session.execute(table.delete()) # Truncate table- session.commit()---def run_server():- """Starts the Letta server in a background thread."""- load_dotenv()- from letta.server.rest_api.app import start_server-- start_server(debug=True)--[email protected](scope="session")-def server_url():- """- Ensures a server is running and returns its base URL.-- Uses environment variable if available, otherwise starts a server- in a background thread.- """- url = os.getenv("LETTA_SERVER_URL", "http://localhost:8283")-- if not os.getenv("LETTA_SERVER_URL"):- thread = threading.Thread(target=run_server, daemon=True)- thread.start()- time.sleep(5) # Give server time to start-- return url--[email protected](scope="module")-def server():- """- Creates a SyncServer instance for testing.-- Loads and saves config to ensure proper initialization.- """- config = LettaConfig.load()- config.save()- return SyncServer()--[email protected](scope="session")-def client(server_url):- """Creates a REST client connected to the test server."""- return Letta(base_url=server_url)---async def test_create_batch(client: Letta, server: SyncServer):- # create agents- agent1 = client.agents.create(- name="agent1_batch",- memory_blocks=[{"label": "persona", "value": "you are agent 1"}],- model="anthropic/claude-3-7-sonnet-20250219",- embedding="letta/letta-free",- )- agent2 = client.agents.create(- name="agent2_batch",- memory_blocks=[{"label": "persona", "value": "you are agent 2"}],- model="anthropic/claude-3-7-sonnet-20250219",- embedding="letta/letta-free",- )-- # create a run- run = client.batches.create(- requests=[- LettaBatchRequest(- messages=[- MessageCreate(- role="user",- content=[- TextContent(- text="hi",- )- ],- )- ],- agent_id=agent1.id,- ),- LettaBatchRequest(- messages=[- MessageCreate(- role="user",- content=[- TextContent(- text="hi",- )- ],- )- ],- agent_id=agent2.id,- ),- ]- )- assert run is not None-- # list batches- batches = client.batches.list()- assert len(batches) == 1, f"Expected 1 batch, got {len(batches)}"- assert batches[0].status == JobStatus.running-- # Poll it once- await poll_running_llm_batches(server)-- # get the batch results- results = client.batches.retrieve(- batch_id=run.id,- )- assert results is not None-- # cancel- client.batches.cancel(batch_id=run.id)- batch_job = client.batches.retrieve(- batch_id=run.id,- )- assert batch_job.status == JobStatus.cancelleddiff --git a/tests/test_agent_files/test_basic_agent_with_blocks_tools_messages_v2.af b/tests/test_agent_files/test_basic_agent_with_blocks_tools_messages_v2.afindex 6db6511bc..5d1142e5a 100644--- a/tests/test_agent_files/test_basic_agent_with_blocks_tools_messages_v2.af+++ b/tests/test_agent_files/test_basic_agent_with_blocks_tools_messages_v2.af@@ -1,7 +1,7 @@{"agents": [{- "name": "test_export_import_f72735f5-a08b-4ff0-8f8e-761af82dee33",+ "name": "test_export_import_431ac32f-ffd1-40a7-8152-9733470c951d","memory_blocks": [],"tools": [],"tool_ids": [@@ -21,24 +21,24 @@],"tool_rules": [{- "tool_name": "conversation_search",+ "tool_name": "memory_replace","type": "continue_loop",- "prompt_template": "<tool_rule>\n{{ tool_name }} requires continuing your response when called\n</tool_rule>"+ "prompt_template": null},{- "tool_name": "memory_replace",- "type": "continue_loop",- "prompt_template": "<tool_rule>\n{{ tool_name }} requires continuing your response when called\n</tool_rule>"+ "tool_name": "send_message",+ "type": "exit_loop",+ "prompt_template": null},{"tool_name": "memory_insert","type": "continue_loop",- "prompt_template": "<tool_rule>\n{{ tool_name }} requires continuing your response when called\n</tool_rule>"+ "prompt_template": null},{- "tool_name": "send_message",- "type": "exit_loop",- "prompt_template": "<tool_rule>\n{{ tool_name }} ends your response (yields control) when called\n</tool_rule>"+ "tool_name": "conversation_search",+ "type": "continue_loop",+ "prompt_template": null}],"tags": [@@ -99,6 +99,7 @@"template": false,"project": null,"tool_exec_environment_variables": {},+ "secrets": null,"memory_variables": null,"project_id": null,"template_id": null,@@ -128,7 +129,7 @@"content": [{"type": "text",- "text": "You are a helpful assistant specializing in data analysis and mathematical computations.\n\n<memory_blocks>\nThe following memory blocks are currently engaged in your core memory unit:\n\n<project_context>\n<description>\nNone\n</description>\n<metadata>\n- chars_current=210\n- chars_limit=6000\n</metadata>\n<value>\n# NOTE: Line numbers shown below are to help during editing. Do NOT include line number prefixes in your memory edit tool calls.\nLine 1: Current project: Building predictive models for financial markets. Sarah is working on sequence analysis and pattern recognition. Recently interested in mathematical sequences like Fibonacci for trend analysis.\n</value>\n</project_context>\n\n<human>\n<description>\nThe human block: Stores key details about the person you are conversing with, allowing for more personalized and friend-like conversation.\n</description>\n<metadata>\n- chars_current=175\n- chars_limit=4000\n</metadata>\n<value>\n# NOTE: Line numbers shown below are to help during editing. Do NOT include line number prefixes in your memory edit tool calls.\nLine 1: username: sarah_researcher\nLine 2: occupation: data scientist\nLine 3: interests: machine learning, statistics, fibonacci sequences\nLine 4: preferred_communication: detailed explanations with examples\n</value>\n</human>\n\n<persona>\n<description>\nThe persona block: Stores details about your current persona, guiding how you behave and respond. This helps you to maintain consistency and personality in your interactions.\n</description>\n<metadata>\n- chars_current=195\n- chars_limit=8000\n</metadata>\n<value>\n# NOTE: Line numbers shown below are to help during editing. Do NOT include line number prefixes in your memory edit tool calls.\nLine 1: You are Alex, a data analyst and mathematician who helps users with calculations and insights. You have extensive experience in statistical analysis and prefer to provide clear, accurate results.\n</value>\n</persona>\n\n</memory_blocks>\n\n<tool_usage_rules>\nThe following constraints define rules for tool usage and guide desired behavior. These rules must be followed to ensure proper tool execution and workflow. A single response may contain multiple tool calls.\n\n<tool_rule>\nconversation_search requires continuing your response when called\n</tool_rule>\n<tool_rule>\nmemory_replace requires continuing your response when called\n</tool_rule>\n<tool_rule>\nmemory_insert requires continuing your response when called\n</tool_rule>\n<tool_rule>\nsend_message ends your response (yields control) when called\n</tool_rule>\n</tool_usage_rules>\n\n\n\n<memory_metadata>\n- The current system date is: September 08, 2025\n- Memory blocks were last modified: 2025-09-08 05:53:05 PM UTC+0000\n- -1 previous messages between you and the user are stored in recall memory (use tools to access them)\n- 2 total memories you created are stored in archival memory (use tools to access them)\n</memory_metadata>"+ "text": "You are a helpful assistant specializing in data analysis and mathematical computations.\n\n<memory_blocks>\nThe following memory blocks are currently engaged in your core memory unit:\n\n<persona>\n<description>\nThe persona block: Stores details about your current persona, guiding how you behave and respond. This helps you to maintain consistency and personality in your interactions.\n</description>\n<metadata>\n- chars_current=195\n- chars_limit=8000\n</metadata>\n<value>\n# NOTE: Line numbers shown below are to help during editing. Do NOT include line number prefixes in your memory edit tool calls.\nLine 1: You are Alex, a data analyst and mathematician who helps users with calculations and insights. You have extensive experience in statistical analysis and prefer to provide clear, accurate results.\n</value>\n</persona>\n\n<human>\n<description>\nThe human block: Stores key details about the person you are conversing with, allowing for more personalized and friend-like conversation.\n</description>\n<metadata>\n- chars_current=175\n- chars_limit=4000\n</metadata>\n<value>\n# NOTE: Line numbers shown below are to help during editing. Do NOT include line number prefixes in your memory edit tool calls.\nLine 1: username: sarah_researcher\nLine 2: occupation: data scientist\nLine 3: interests: machine learning, statistics, fibonacci sequences\nLine 4: preferred_communication: detailed explanations with examples\n</value>\n</human>\n\n<project_context>\n<description>\n\n</description>\n<metadata>\n- chars_current=210\n- chars_limit=6000\n</metadata>\n<value>\n# NOTE: Line numbers shown below are to help during editing. Do NOT include line number prefixes in your memory edit tool calls.\nLine 1: Current project: Building predictive models for financial markets. Sarah is working on sequence analysis and pattern recognition. Recently interested in mathematical sequences like Fibonacci for trend analysis.\n</value>\n</project_context>\n\n</memory_blocks>\n\n<tool_usage_rules>\nThe following constraints define rules for tool usage and guide desired behavior. These rules must be followed to ensure proper tool execution and workflow. A single response may contain multiple tool calls.\n\n<tool_rule>\nmemory_replace requires continuing your response when called\n</tool_rule>\n<tool_rule>\nmemory_insert requires continuing your response when called\n</tool_rule>\n<tool_rule>\nconversation_search requires continuing your response when called\n</tool_rule>\n<tool_rule>\nsend_message ends your response (yields control) when called\n</tool_rule>\n</tool_usage_rules>\n\n<memory_metadata>\n- The current system date is: September 24, 2025\n- Memory blocks were last modified: 2025-09-24 10:57:40 PM UTC+0000\n- -1 previous messages between you and the user are stored in recall memory (use tools to access them)\n- 2 total memories you created are stored in archival memory (use tools to access them)\n</memory_metadata>"}],"name": null,@@ -142,7 +143,7 @@"tool_calls": null,"tool_call_id": null,"tool_returns": [],- "created_at": "2025-09-08T17:53:02.499443+00:00"+ "created_at": "2025-09-24T22:57:39.493431+00:00"},{"type": "message",@@ -163,7 +164,7 @@"agent_id": "agent-0","tool_calls": [{- "id": "c1ec5c60-7dbc-4b53-bfbb-c2af5d5a222f",+ "id": "f1c5f8b4-c57c-4641-8d58-903248a31f7b","function": {"arguments": "{\n \"message\": \"More human than human is our motto.\"\n}","name": "send_message"@@ -173,7 +174,7 @@],"tool_call_id": null,"tool_returns": [],- "created_at": "2025-09-08T17:53:02.499491+00:00"+ "created_at": "2025-09-24T22:57:39.493461+00:00"},{"type": "message",@@ -181,7 +182,7 @@"content": [{"type": "text",- "text": "{\n \"status\": \"OK\",\n \"message\": null,\n \"time\": \"2025-09-08 05:53:02 PM UTC+0000\"\n}"+ "text": "{\n \"status\": \"OK\",\n \"message\": null,\n \"time\": \"2025-09-24 10:57:39 PM UTC+0000\"\n}"}],"name": "send_message",@@ -193,9 +194,9 @@"model": "gpt-4.1-mini","agent_id": "agent-0","tool_calls": null,- "tool_call_id": "c1ec5c60-7dbc-4b53-bfbb-c2af5d5a222f",+ "tool_call_id": "f1c5f8b4-c57c-4641-8d58-903248a31f7b","tool_returns": [],- "created_at": "2025-09-08T17:53:02.499526+00:00"+ "created_at": "2025-09-24T22:57:39.493485+00:00"},{"type": "message",@@ -203,7 +204,7 @@"content": [{"type": "text",- "text": "{\n \"type\": \"login\",\n \"last_login\": \"Never (first login)\",\n \"time\": \"2025-09-08 05:53:02 PM UTC+0000\"\n}"+ "text": "{\n \"type\": \"login\",\n \"last_login\": \"Never (first login)\",\n \"time\": \"2025-09-24 10:57:39 PM UTC+0000\"\n}"}],"name": null,@@ -217,7 +218,7 @@"tool_calls": null,"tool_call_id": null,"tool_returns": [],- "created_at": "2025-09-08T17:53:02.499541+00:00"+ "created_at": "2025-09-24T22:57:39.493496+00:00"},{"type": "message",@@ -239,7 +240,7 @@"tool_calls": null,"tool_call_id": null,"tool_returns": [],- "created_at": "2025-09-08T17:53:05.017966+00:00"+ "created_at": "2025-09-24T22:57:40.764277+00:00"},{"type": "message",@@ -247,7 +248,7 @@"content": [{"type": "text",- "text": "Responding to test message for export."+ "text": "User sent a test message for export functionality verification."}],"name": null,@@ -260,17 +261,17 @@"agent_id": "agent-0","tool_calls": [{- "id": "call_sMVo2qIYhILKKNT0YiOdsU24",+ "id": "call_pvBdfaX7NpaFcjrhqNCSyg7S","function": {- "arguments": "{\"message\": \"Test message received successfully. Ready to assist you with any questions or tasks.\", \"request_heartbeat\": false}",+ "arguments": "{\"message\": \"I received your test message successfully. If you need any assistance or want to continue with our project discussions, just let me know!\", \"request_heartbeat\": false}","name": "send_message"},"type": "function"}],- "tool_call_id": "call_sMVo2qIYhILKKNT0YiOdsU24",+ "tool_call_id": "call_pvBdfaX7NpaFcjrhqNCSyg7S","tool_returns": [],- "created_at": "2025-09-08T17:53:06.600989+00:00"+ "created_at": "2025-09-24T22:57:41.948407+00:00"},{"type": "message",@@ -278,7 +279,7 @@"content": [{"type": "text",- "text": "{\n \"status\": \"OK\",\n \"message\": \"Sent message successfully.\",\n \"time\": \"2025-09-08 05:53:06 PM UTC+0000\"\n}"+ "text": "{\n \"status\": \"OK\",\n \"message\": \"Sent message successfully.\",\n \"time\": \"2025-09-24 10:57:41 PM UTC+0000\"\n}"}],"name": "send_message",@@ -290,7 +291,7 @@"model": "gpt-4.1-mini","agent_id": "agent-0","tool_calls": null,- "tool_call_id": "call_sMVo2qIYhILKKNT0YiOdsU24",+ "tool_call_id": "call_pvBdfaX7NpaFcjrhqNCSyg7S","tool_returns": [{"status": "success",@@ -298,7 +299,7 @@"stderr": null}],- "created_at": "2025-09-08T17:53:06.601180+00:00"+ "created_at": "2025-09-24T22:57:41.948586+00:00"}],"files_agents": [],@@ -366,7 +367,7 @@"sources": [],"tools": [{- "id": "tool-6",+ "id": "tool-5","tool_type": "custom","description": "Analyze data and provide insights.","source_type": "json",@@ -410,7 +411,7 @@"metadata_": {}},{- "id": "tool-2",+ "id": "tool-1","tool_type": "custom","description": "Calculate the nth Fibonacci number.","source_type": "json",@@ -446,7 +447,7 @@"metadata_": {}},{- "id": "tool-3",+ "id": "tool-0","tool_type": "letta_core","description": "Search prior conversation history using hybrid search (text + semantic similarity).\n\nExamples:\n # Search all messages\n conversation_search(query=\"project updates\")\n\n # Search only assistant messages\n conversation_search(query=\"error handling\", roles=[\"assistant\"])\n\n # Search with date range (inclusive of both dates)\n conversation_search(query=\"meetings\", start_date=\"2024-01-15\", end_date=\"2024-01-20\")\n # This includes all messages from Jan 15 00:00:00 through Jan 20 23:59:59\n\n # Search messages from a specific day (inclusive)\n conversation_search(query=\"bug reports\", start_date=\"2024-09-04\", end_date=\"2024-09-04\")\n # This includes ALL messages from September 4, 2024\n\n # Search with specific time boundaries\n conversation_search(query=\"deployment\", start_date=\"2024-01-15T09:00\", end_date=\"2024-01-15T17:30\")\n # This includes messages from 9 AM to 5:30 PM on Jan 15\n\n # Search with limit\n conversation_search(query=\"debugging\", limit=10)\n\n Returns:\n str: Query result string containing matching messages with timestamps and content.","source_type": "python",@@ -505,7 +506,7 @@"metadata_": {}},{- "id": "tool-1",+ "id": "tool-2","tool_type": "custom","description": "Get user preferences for a specific category.","source_type": "json",@@ -541,7 +542,7 @@"metadata_": {}},{- "id": "tool-0",+ "id": "tool-3","tool_type": "letta_sleeptime_core","description": "The memory_insert command allows you to insert text at a specific location in a memory block.\n\nExamples:\n # Update a block containing information about the user (append to the end of the block)\n memory_insert(label=\"customer\", new_str=\"The customer's ticket number is 12345\")\n\n # Update a block containing information about the user (insert at the beginning of the block)\n memory_insert(label=\"customer\", new_str=\"The customer's ticket number is 12345\", insert_line=0)\n\n Returns:\n Optional[str]: None is always returned as this function does not produce a response.","source_type": "python",@@ -630,7 +631,7 @@"metadata_": {}},{- "id": "tool-5",+ "id": "tool-6","tool_type": "letta_core","description": "Sends a message to the human user.","source_type": "python",@@ -667,7 +668,7 @@],"mcp_servers": [],"metadata": {- "revision_id": "5b804970e6a0"+ "revision_id": "3d2e9fb40a3c"},- "created_at": "2025-09-08T17:53:06.749694+00:00"+ "created_at": "2025-09-24T22:57:42.392726+00:00"}diff --git a/tests/test_sdk_client.py b/tests/test_sdk_client.pyindex a1136e2e2..a8629670c 100644--- a/tests/test_sdk_client.py+++ b/tests/test_sdk_client.py@@ -13,6 +13,7 @@ from letta_client import (ContinueToolRule,CreateBlock,Letta as LettaSDKClient,+ LettaBatchRequest,LettaRequest,MaxCountPerStepToolRule,MessageCreate,@@ -24,6 +25,10 @@ from letta_client.core import ApiErrorfrom letta_client.types import AgentState, ToolReturnMessagefrom pydantic import BaseModel, Field+from letta.config import LettaConfig+from letta.jobs.llm_batch_job_polling import poll_running_llm_batches+from letta.schemas.enums import JobStatus+from letta.server.server import SyncServerfrom tests.helpers.utils import upload_file_and_wait# Constants@@ -60,6 +65,18 @@ def client() -> LettaSDKClient:yield client[email protected](scope="module")+def server():+ """+ Creates a SyncServer instance for testing.++ Loads and saves config to ensure proper initialization.+ """+ config = LettaConfig.load()+ config.save()+ return SyncServer()++@pytest.fixture(scope="function")def agent(client: LettaSDKClient):agent_state = client.agents.create(@@ -2190,3 +2207,74 @@ def test_upsert_tools(client: LettaSDKClient):# Clean upclient.tools.delete(tool.id)+++async def test_create_batch(client: LettaSDKClient, server: SyncServer):+ # create agents+ agent1 = client.agents.create(+ name="agent1_batch",+ memory_blocks=[{"label": "persona", "value": "you are agent 1"}],+ model="anthropic/claude-3-7-sonnet-20250219",+ embedding="letta/letta-free",+ )+ agent2 = client.agents.create(+ name="agent2_batch",+ memory_blocks=[{"label": "persona", "value": "you are agent 2"}],+ model="anthropic/claude-3-7-sonnet-20250219",+ embedding="letta/letta-free",+ )++ # create a run+ run = client.batches.create(+ requests=[+ LettaBatchRequest(+ messages=[+ MessageCreate(+ role="user",+ content=[+ TextContent(+ text="hi",+ )+ ],+ )+ ],+ agent_id=agent1.id,+ ),+ LettaBatchRequest(+ messages=[+ MessageCreate(+ role="user",+ content=[+ TextContent(+ text="hi",+ )+ ],+ )+ ],+ agent_id=agent2.id,+ ),+ ]+ )+ assert run is not None++ # list batches+ batches = client.batches.list()+ assert len(batches) >= 1, f"Expected 1 or more batches, got {len(batches)}"+ assert batches[0].status == JobStatus.running++ # Poll it once+ await poll_running_llm_batches(server)++ # get the batch results+ results = client.batches.retrieve(+ batch_id=run.id,+ )+ assert results is not None++ # cancel+ client.batches.cancel(batch_id=run.id)+ batch_job = client.batches.retrieve(+ batch_id=run.id,+ )+ assert batch_job.status == JobStatus.cancelled