Platform for stateful agents: AI with advanced memory that can learn and self-improve over time.
feat: Create POST v1/agents/messages/batches (#1722)
Matthew Zhoubbd4b087d30f
diff --git a/letta/agents/letta_agent_batch.py b/letta/agents/letta_agent_batch.pyindex 7d2f3c305..b792ae3f8 100644--- a/letta/agents/letta_agent_batch.py+++ b/letta/agents/letta_agent_batch.py@@ -117,11 +117,13 @@ class LettaAgentBatch:self.max_steps = max_stepsasync def step_until_request(- self, batch_requests: List[LettaBatchRequest], agent_step_state_mapping: Dict[str, AgentStepState]+ self, batch_requests: List[LettaBatchRequest], agent_step_state_mapping: Optional[Dict[str, AgentStepState]] = None) -> LettaBatchResponse:# Basic checksif not batch_requests:raise ValueError("Empty list of batch_requests passed in!")+ if agent_step_state_mapping is None:+ agent_step_state_mapping = {}agent_messages_mapping: Dict[str, List[Message]] = {}agent_tools_mapping: Dict[str, List[dict]] = {}@@ -134,6 +136,13 @@ class LettaAgentBatch:agent_messages_mapping[agent_id] = self._get_in_context_messages_per_agent(agent_state=agent_state, input_messages=batch_request.messages)++ # TODO: Think about a cleaner way to do this?+ if agent_id not in agent_step_state_mapping:+ agent_step_state_mapping[agent_id] = AgentStepState(+ step_number=0, tool_rules_solver=ToolRulesSolver(tool_rules=agent_state.tool_rules)+ )+agent_tools_mapping[agent_id] = self._prepare_tools_per_agent(agent_state, agent_step_state_mapping.get(agent_id).tool_rules_solver)diff --git a/letta/server/rest_api/routers/v1/agents.py b/letta/server/rest_api/routers/v1/agents.pyindex 35dcd703b..c5ec3533e 100644--- a/letta/server/rest_api/routers/v1/agents.py+++ b/letta/server/rest_api/routers/v1/agents.py@@ -12,6 +12,7 @@ from sqlalchemy.exc import IntegrityError, OperationalErrorfrom starlette.responses import Response, StreamingResponsefrom letta.agents.letta_agent import LettaAgent+from letta.agents.letta_agent_batch import LettaAgentBatchfrom letta.constants import DEFAULT_MESSAGE_TOOL, DEFAULT_MESSAGE_TOOL_KWARGfrom letta.log import get_loggerfrom letta.orm.errors import NoResultFound@@ -20,8 +21,8 @@ from letta.schemas.block import Block, BlockUpdatefrom letta.schemas.group import Groupfrom letta.schemas.job import JobStatus, JobUpdate, LettaRequestConfigfrom letta.schemas.letta_message import LettaMessageUnion, LettaMessageUpdateUnion-from letta.schemas.letta_request import LettaRequest, LettaStreamingRequest-from letta.schemas.letta_response import LettaResponse+from letta.schemas.letta_request import LettaBatchRequest, LettaRequest, LettaStreamingRequest+from letta.schemas.letta_response import LettaBatchResponse, LettaResponsefrom letta.schemas.memory import ContextWindowOverview, CreateArchivalMemory, Memoryfrom letta.schemas.message import MessageCreatefrom letta.schemas.passage import Passage, PassageUpdate@@ -818,3 +819,31 @@ async def list_agent_groups(actor = server.user_manager.get_user_or_default(user_id=actor_id)print("in list agents with manager_type", manager_type)return server.agent_manager.list_groups(agent_id=agent_id, manager_type=manager_type, actor=actor)+++# Batch APIs++[email protected]("/messages/batches", response_model=LettaBatchResponse, operation_id="create_batch_message_request")+async def send_batch_messages(+ batch_requests: List[LettaBatchRequest] = Body(..., description="Messages and config for all agents"),+ server: SyncServer = Depends(get_letta_server),+ actor_id: Optional[str] = Header(None, alias="user_id"),+):+ """+ Submit a batch of agent messages for asynchronous processing.+ Creates a job that will fan out messages to all listed agents and process them in parallel.+ """+ actor = server.user_manager.get_user_or_default(user_id=actor_id)++ batch_runner = LettaAgentBatch(+ message_manager=server.message_manager,+ agent_manager=server.agent_manager,+ block_manager=server.block_manager,+ passage_manager=server.passage_manager,+ batch_manager=server.batch_manager,+ sandbox_config_manager=server.sandbox_config_manager,+ actor=actor,+ )++ return await batch_runner.step_until_request(batch_requests=batch_requests)