Platform for stateful agents: AI with advanced memory that can learn and self-improve over time.
feat: Add more tests for batch agent loop (#1819)
Matthew Zhoud1c3906f19e5
diff --git a/examples/sleeptime/sleeptime_source_example.py b/examples/sleeptime/sleeptime_source_example.pyindex e6a3ad75d..efc9e8d3f 100644--- a/examples/sleeptime/sleeptime_source_example.py+++ b/examples/sleeptime/sleeptime_source_example.py@@ -1,11 +1,10 @@import time-from letta_client import AgentType, GroupUpdateManagerConfig, Letta, ManagerType, SleeptimeManagerUpdate-from letta_client.types import agent_type+from letta_client import Lettaclient = Letta(base_url="http://localhost:8283")-# delete all sources+# delete all sourcesfor source in client.sources.list():print(f"Deleting source {source.name}")client.sources.delete(source.id)@@ -21,19 +20,19 @@ agent = client.agents.create()print(f"Created agent id {agent.id}")-# get the group+# get the groupgroup_id = agent.multi_agent_group.idcurrent_frequence = agent.multi_agent_group.sleeptime_agent_frequencyprint(f"Group id: {group_id}, frequency: {current_frequence}")-# create a source+# create a sourcesource_name = "employee_handbook"source = client.sources.create(name=source_name,- description="Provides reference information for the employee handbook",+ description="Provides reference information for the employee handbook",embedding="openai/text-embedding-ada-002" # must match agent)-# attach the source to the agent+# attach the source to the agentclient.agents.sources.attach(source_id=source.id,agent_id=agent.id@@ -52,7 +51,7 @@ print("Agent blocks", [b.label for b in client.agents.blocks.list(agent_id=agentblock = client.agents.blocks.retrieve(agent_id=agent.id, block_label="employee_handbook")-# get attached agents+# get attached agentsagents = client.blocks.agents.list(block_id=block.id)for agent in agents:print(f"Agent id {agent.id}", agent.agent_type)@@ -63,14 +62,10 @@ for agent in agents:while job.status != "completed":job = client.jobs.retrieve(job.id)- # count passages+ # count passagespassages = client.agents.passages.list(agent_id=agent.id)print(f"Passages {len(passages)}")for passage in passages:print(passage.text)time.sleep(2)----diff --git a/letta/agents/letta_agent_batch.py b/letta/agents/letta_agent_batch.pyindex d2648944d..7280c43c6 100644--- a/letta/agents/letta_agent_batch.py+++ b/letta/agents/letta_agent_batch.py@@ -243,7 +243,8 @@ class LettaAgentBatch:)async def _collect_resume_context(self, llm_batch_id: str) -> _ResumeContext:- batch_items = self.batch_manager.list_llm_batch_items(llm_batch_id=llm_batch_id)+ # NOTE: We only continue for items with successful results+ batch_items = self.batch_manager.list_llm_batch_items(llm_batch_id=llm_batch_id, request_status=JobStatus.completed)agent_ids, agent_state_map = [], {}provider_results, name_map, args_map, cont_map = {}, {}, {}, {}diff --git a/letta/server/server.py b/letta/server/server.pyindex 70e63520b..d8b9fddd6 100644--- a/letta/server/server.py+++ b/letta/server/server.py@@ -704,7 +704,6 @@ class SyncServer(Server):def get_cached_llm_config(self, **kwargs):key = make_key(**kwargs)print(self._llm_config_cache)- print("KEY", key)if key not in self._llm_config_cache:self._llm_config_cache[key] = self.get_llm_config_from_handle(**kwargs)return self._llm_config_cache[key]@@ -1182,7 +1181,6 @@ class SyncServer(Server):provider = self.get_provider_from_name(provider_name)llm_configs = [config for config in provider.list_llm_models() if config.handle == handle]- print("LLM CONFIGS", llm_configs)if not llm_configs:llm_configs = [config for config in provider.list_llm_models() if config.model == model_name]if not llm_configs:@@ -1195,8 +1193,6 @@ class SyncServer(Server):if not llm_configs:raise e- print("CONFIGS", llm_configs)-if len(llm_configs) == 1:llm_config = llm_configs[0]elif len(llm_configs) > 1:diff --git a/tests/test_letta_agent_batch.py b/tests/test_letta_agent_batch.pyindex 0f3885a35..a67cc48ce 100644--- a/tests/test_letta_agent_batch.py+++ b/tests/test_letta_agent_batch.py@@ -146,7 +146,7 @@ def step_state_map(agents):Returns:Dict[str, AgentStepState]: Mapping of agent IDs to step states"""- solver = ToolRulesSolver(tool_rules=[InitToolRule(tool_name="send_message")])+ solver = ToolRulesSolver(tool_rules=[InitToolRule(tool_name="get_weather")])return {agent.id: AgentStepState(step_number=0, tool_rules_solver=solver) for agent in agents}@@ -173,25 +173,6 @@ def create_batch_response(batch_id: str, processing_status: str = "in_progress"))-def create_successful_response(custom_id: str) -> BetaMessageBatchIndividualResponse:- """Create a dummy successful batch response."""- return BetaMessageBatchIndividualResponse(- custom_id=custom_id,- result=BetaMessageBatchSucceededResult(- type="succeeded",- message=BetaMessage(- id="msg_abc123",- role="assistant",- type="message",- model="claude-3-5-sonnet-20240620",- content=[{"type": "text", "text": "hi!"}],- usage={"input_tokens": 5, "output_tokens": 7},- stop_reason="end_turn",- ),- ),- )--def create_complete_tool_response(custom_id: str, model: str, request_heartbeat: bool) -> BetaMessageBatchIndividualResponse:"""Create a dummy successful batch response with a tool call after user asks about weather."""return BetaMessageBatchIndividualResponse(@@ -340,6 +321,275 @@ class MockAsyncIterable:# --------------------------------------------------------------------------- #+async def test_error_from_anthropic(+ disable_e2b_api_key, server, default_user, agents: Tuple[AgentState], batch_requests, step_state_map, batch_job+):+ anthropic_batch_id = "msgbatch_test_12345"+ dummy_batch_response = create_batch_response(+ batch_id=anthropic_batch_id,+ )++ # 1. Invoke `step_until_request`+ with patch("letta.llm_api.anthropic_client.AnthropicClient.send_llm_batch_request_async", return_value=dummy_batch_response):+ # Create batch runner+ batch_runner = LettaAgentBatch(+ message_manager=server.message_manager,+ agent_manager=server.agent_manager,+ block_manager=server.block_manager,+ passage_manager=server.passage_manager,+ batch_manager=server.batch_manager,+ sandbox_config_manager=server.sandbox_config_manager,+ job_manager=server.job_manager,+ actor=default_user,+ )++ # Run the method under test+ pre_resume_response = await batch_runner.step_until_request(+ batch_requests=batch_requests,+ agent_step_state_mapping=step_state_map,+ letta_batch_job_id=batch_job.id,+ )++ llm_batch_jobs = server.batch_manager.list_llm_batch_jobs(letta_batch_id=pre_resume_response.letta_batch_id, actor=default_user)+ llm_batch_job = llm_batch_jobs[0]++ # 2. Invoke the polling job and mock responses from Anthropic+ mock_retrieve = AsyncMock(return_value=create_batch_response(batch_id=pre_resume_response.letta_batch_id, processing_status="ended"))++ with patch.object(server.anthropic_async_client.beta.messages.batches, "retrieve", mock_retrieve):+ agents_failed = agents[:1]+ agents_continue = agents[1:]+ # Create failed response for one agent+ mock_items = [create_failed_response(custom_id=agent.id) for agent in agents_failed]+ mock_items.extend(+ [+ create_complete_tool_response(custom_id=agent.id, model=agent.llm_config.model, request_heartbeat=True)+ for agent in agents_continue+ ]+ )++ # Create the mock for results+ mock_results = Mock()+ mock_results.return_value = MockAsyncIterable(mock_items.copy()) # Using copy to preserve the original list++ with patch.object(server.anthropic_async_client.beta.messages.batches, "results", mock_results):+ with patch("letta.llm_api.anthropic_client.AnthropicClient.send_llm_batch_request_async", return_value=dummy_batch_response):+ msg_counts_before = {agent.id: server.message_manager.size(actor=default_user, agent_id=agent.id) for agent in agents}++ new_batch_responses = await poll_running_llm_batches(server)++ # Verify database records were updated correctly+ llm_batch_job = server.batch_manager.get_llm_batch_job_by_id(llm_batch_job.id, actor=default_user)++ # Verify job properties+ assert llm_batch_job.status == JobStatus.completed, "Job status should be 'completed'"++ # Verify batch items+ items = server.batch_manager.list_llm_batch_items(llm_batch_id=llm_batch_job.id, actor=default_user)+ assert len(items) == 3, f"Expected 3 batch items, got {len(items)}"++ # Verify only one new batch response+ assert len(new_batch_responses) == 1+ post_resume_response = new_batch_responses[0]++ assert (+ post_resume_response.letta_batch_id == pre_resume_response.letta_batch_id+ ), "resume_step_after_request is expected to have the same letta_batch_id"+ assert (+ post_resume_response.last_llm_batch_id != pre_resume_response.last_llm_batch_id+ ), "resume_step_after_request is expected to have different llm_batch_id."+ assert post_resume_response.status == JobStatus.running+ # NOTE: We only expect 2 agents to continue (succeeded ones)+ assert post_resume_response.agent_count == 2++ # New batch‑items should exist, initialised in (created, paused) state+ new_items = server.batch_manager.list_llm_batch_items(+ llm_batch_id=post_resume_response.last_llm_batch_id, actor=default_user+ )+ assert len(new_items) == 2, f"Expected 2 new batch item, got {len(new_items)}"+ # Assert that the continuing agent is in the only item+ assert {i.agent_id for i in new_items} == {a.id for a in agents_continue}+ assert {i.request_status for i in new_items} == {JobStatus.created}+ assert {i.step_status for i in new_items} == {AgentStepStatus.paused}++ # Confirm that tool_rules_solver state was preserved correctly+ # Assert every new item's step_state's tool_rules_solver has "get_weather" in the tool_call_history+ assert all(+ "get_weather" in item.step_state.tool_rules_solver.tool_call_history for item in new_items+ ), "Expected 'get_weather' in tool_call_history for all new_items"+ # Assert that each new item's step_number was incremented to 1+ assert all(+ item.step_state.step_number == 1 for item in new_items+ ), "Expected step_number to be incremented to 1 for all new_items"++ # Old items must have been flipped to completed / finished earlier+ # (sanity – we already asserted this above, but we keep it close for clarity)+ old_items = server.batch_manager.list_llm_batch_items(+ llm_batch_id=pre_resume_response.last_llm_batch_id, actor=default_user+ )+ for item in old_items:+ if item.agent_id == agents_failed[0].id:+ assert item.request_status == JobStatus.failed+ assert item.step_status == AgentStepStatus.paused+ else:+ assert item.request_status == JobStatus.completed+ assert item.step_status == AgentStepStatus.completed++ # Tool‑call side‑effects – each agent gets at least 2 extra messages+ for agent in agents:+ before = msg_counts_before[agent.id] # captured just before resume+ after = server.message_manager.size(actor=default_user, agent_id=agent.id)++ if agent.id == agents_failed[0].id:+ assert after == before, f"Agent {agent.id} should not have extra messages persisted due to Anthropic failure"+ else:+ assert after - before >= 2, (+ f"Agent {agent.id} should have an assistant tool‑call " f"and tool‑response message persisted."+ )++ # Check that agent states have been properly modified to have extended in-context messages+ for agent in agents:+ refreshed_agent = server.agent_manager.get_agent_by_id(agent_id=agent.id, actor=default_user)+ if refreshed_agent.id == agents_failed[0].id:+ assert (+ len(refreshed_agent.message_ids) == 4+ ), f"Agent's in-context messages have not been extended, are length: {len(refreshed_agent.message_ids)}"+ else:+ assert (+ len(refreshed_agent.message_ids) == 6+ ), f"Agent's in-context messages have been extended, are length: {len(refreshed_agent.message_ids)}"+++async def test_resume_step_some_stop(+ disable_e2b_api_key, server, default_user, agents: Tuple[AgentState], batch_requests, step_state_map, batch_job+):+ anthropic_batch_id = "msgbatch_test_12345"+ dummy_batch_response = create_batch_response(+ batch_id=anthropic_batch_id,+ )++ # 1. Invoke `step_until_request`+ with patch("letta.llm_api.anthropic_client.AnthropicClient.send_llm_batch_request_async", return_value=dummy_batch_response):+ # Create batch runner+ batch_runner = LettaAgentBatch(+ message_manager=server.message_manager,+ agent_manager=server.agent_manager,+ block_manager=server.block_manager,+ passage_manager=server.passage_manager,+ batch_manager=server.batch_manager,+ sandbox_config_manager=server.sandbox_config_manager,+ job_manager=server.job_manager,+ actor=default_user,+ )++ # Run the method under test+ pre_resume_response = await batch_runner.step_until_request(+ batch_requests=batch_requests,+ agent_step_state_mapping=step_state_map,+ letta_batch_job_id=batch_job.id,+ )++ llm_batch_jobs = server.batch_manager.list_llm_batch_jobs(letta_batch_id=pre_resume_response.letta_batch_id, actor=default_user)+ llm_batch_job = llm_batch_jobs[0]++ # 2. Invoke the polling job and mock responses from Anthropic+ mock_retrieve = AsyncMock(return_value=create_batch_response(batch_id=pre_resume_response.letta_batch_id, processing_status="ended"))++ with patch.object(server.anthropic_async_client.beta.messages.batches, "retrieve", mock_retrieve):+ agents_continue = agents[:1]+ agents_finish = agents[1:]+ mock_items = [+ create_complete_tool_response(custom_id=agent.id, model=agent.llm_config.model, request_heartbeat=True)+ for agent in agents_continue+ ]+ mock_items.extend(+ [+ create_complete_tool_response(custom_id=agent.id, model=agent.llm_config.model, request_heartbeat=False)+ for agent in agents_finish+ ]+ )++ # Create the mock for results+ mock_results = Mock()+ mock_results.return_value = MockAsyncIterable(mock_items.copy()) # Using copy to preserve the original list++ with patch.object(server.anthropic_async_client.beta.messages.batches, "results", mock_results):+ with patch("letta.llm_api.anthropic_client.AnthropicClient.send_llm_batch_request_async", return_value=dummy_batch_response):+ msg_counts_before = {agent.id: server.message_manager.size(actor=default_user, agent_id=agent.id) for agent in agents}++ new_batch_responses = await poll_running_llm_batches(server)++ # Verify database records were updated correctly+ llm_batch_job = server.batch_manager.get_llm_batch_job_by_id(llm_batch_job.id, actor=default_user)++ # Verify job properties+ assert llm_batch_job.status == JobStatus.completed, "Job status should be 'completed'"++ # Verify batch items+ items = server.batch_manager.list_llm_batch_items(llm_batch_id=llm_batch_job.id, actor=default_user)+ assert len(items) == 3, f"Expected 3 batch items, got {len(items)}"+ assert all([item.request_status == JobStatus.completed for item in items])++ # Verify only one new batch response+ assert len(new_batch_responses) == 1+ post_resume_response = new_batch_responses[0]++ assert (+ post_resume_response.letta_batch_id == pre_resume_response.letta_batch_id+ ), "resume_step_after_request is expected to have the same letta_batch_id"+ assert (+ post_resume_response.last_llm_batch_id != pre_resume_response.last_llm_batch_id+ ), "resume_step_after_request is expected to have different llm_batch_id."+ assert post_resume_response.status == JobStatus.running+ # NOTE: We only expect 1 agent to continue+ assert post_resume_response.agent_count == 1++ # New batch‑items should exist, initialised in (created, paused) state+ new_items = server.batch_manager.list_llm_batch_items(+ llm_batch_id=post_resume_response.last_llm_batch_id, actor=default_user+ )+ assert len(new_items) == 1, f"Expected 1 new batch item, got {len(new_items)}"+ # Assert that the continuing agent is in the only item+ assert new_items[0].agent_id == agents_continue[0].id+ assert {i.request_status for i in new_items} == {JobStatus.created}+ assert {i.step_status for i in new_items} == {AgentStepStatus.paused}++ # Confirm that tool_rules_solver state was preserved correctly+ # Assert every new item's step_state's tool_rules_solver has "get_weather" in the tool_call_history+ assert all(+ "get_weather" in item.step_state.tool_rules_solver.tool_call_history for item in new_items+ ), "Expected 'get_weather' in tool_call_history for all new_items"+ # Assert that each new item's step_number was incremented to 1+ assert all(+ item.step_state.step_number == 1 for item in new_items+ ), "Expected step_number to be incremented to 1 for all new_items"++ # Old items must have been flipped to completed / finished earlier+ # (sanity – we already asserted this above, but we keep it close for clarity)+ old_items = server.batch_manager.list_llm_batch_items(+ llm_batch_id=pre_resume_response.last_llm_batch_id, actor=default_user+ )+ assert {i.request_status for i in old_items} == {JobStatus.completed}+ assert {i.step_status for i in old_items} == {AgentStepStatus.completed}++ # Tool‑call side‑effects – each agent gets at least 2 extra messages+ for agent in agents:+ before = msg_counts_before[agent.id] # captured just before resume+ after = server.message_manager.size(actor=default_user, agent_id=agent.id)+ assert after - before >= 2, (+ f"Agent {agent.id} should have an assistant tool‑call " f"and tool‑response message persisted."+ )++ # Check that agent states have been properly modified to have extended in-context messages+ for agent in agents:+ refreshed_agent = server.agent_manager.get_agent_by_id(agent_id=agent.id, actor=default_user)+ assert (+ len(refreshed_agent.message_ids) == 6+ ), f"Agent's in-context messages have been extended, are length: {len(refreshed_agent.message_ids)}"++@pytest.mark.asyncioasync def test_resume_step_after_request_all_continue(disable_e2b_api_key, server, default_user, agents: Tuple[AgentState], batch_requests, step_state_map, batch_job@@ -460,7 +710,7 @@ async def test_resume_step_after_request_all_continue(refreshed_agent = server.agent_manager.get_agent_by_id(agent_id=agent.id, actor=default_user)assert (len(refreshed_agent.message_ids) == 6- ), f"Agent's in-context messages have not been extended, are length: {len(refreshed_agent.message_ids)}"+ ), f"Agent's in-context messages have been extended, are length: {len(refreshed_agent.message_ids)}"@pytest.mark.asyncio@@ -518,7 +768,7 @@ async def test_step_until_request_prepares_and_submits_batch_correctly(# Verify tool configurationfor agent_id, tools in agent_tools_mapping.items():available_tools = {tool["name"] for tool in tools}- assert available_tools == {"send_message"}, f"Expected only send_message tool, got {available_tools}"+ assert available_tools == {"get_weather"}, f"Expected only send_message tool, got {available_tools}"# Verify model assignmentsfor agent_id, expected_model in expected_models.items():