Platform for stateful agents: AI with advanced memory that can learn and self-improve over time.
feat: refactor streaming route logic (#4369)
cthomas11d2507877d6
diff --git a/letta/server/rest_api/routers/v1/agents.py b/letta/server/rest_api/routers/v1/agents.pyindex 794ffc4d9..3962e7874 100644--- a/letta/server/rest_api/routers/v1/agents.py+++ b/letta/server/rest_api/routers/v1/agents.py@@ -1223,6 +1223,9 @@ async def send_message_streaming(request_start_timestamp_ns = get_utc_timestamp_ns()MetricRegistry().user_message_counter.add(1, get_ctx_attributes())+ # TODO (cliandy): clean this up+ redis_client = await get_redis_client()+actor = await server.user_manager.get_actor_or_default_async(actor_id=actor_id)# TODO: This is redundant, remove soonagent = await server.agent_manager.get_agent_by_id_async(agent_id, actor, include_relationships=["multi_agent_group"])@@ -1263,14 +1266,11 @@ async def send_message_streaming(),actor=actor,)+ job_update_metadata = None+ await redis_client.set(f"{REDIS_RUN_ID_PREFIX}:{agent_id}", run.id if run else None)else:run = None- job_update_metadata = None- # TODO (cliandy): clean this up- redis_client = await get_redis_client()- await redis_client.set(f"{REDIS_RUN_ID_PREFIX}:{agent_id}", run.id if run else None)-try:if agent_eligible and model_compatible:if agent.enable_sleeptime and agent.agent_type != AgentType.voice_convo_agent:@@ -1308,6 +1308,23 @@ async def send_message_streaming(),)+ if request.stream_tokens and model_compatible_token_streaming:+ raw_stream = agent_loop.step_stream(+ input_messages=request.messages,+ max_steps=request.max_steps,+ use_assistant_message=request.use_assistant_message,+ request_start_timestamp_ns=request_start_timestamp_ns,+ include_return_message_types=request.include_return_message_types,+ )+ else:+ raw_stream = agent_loop.step_stream_no_tokens(+ request.messages,+ max_steps=request.max_steps,+ use_assistant_message=request.use_assistant_message,+ request_start_timestamp_ns=request_start_timestamp_ns,+ include_return_message_types=request.include_return_message_types,+ )+from letta.server.rest_api.streaming_response import StreamingResponseWithStatusCode, add_keepalive_to_streamif request.background and settings.track_agent_run:@@ -1321,23 +1338,6 @@ async def send_message_streaming(),)- if request.stream_tokens and model_compatible_token_streaming:- raw_stream = agent_loop.step_stream(- input_messages=request.messages,- max_steps=request.max_steps,- use_assistant_message=request.use_assistant_message,- request_start_timestamp_ns=request_start_timestamp_ns,- include_return_message_types=request.include_return_message_types,- )- else:- raw_stream = agent_loop.step_stream_no_tokens(- request.messages,- max_steps=request.max_steps,- use_assistant_message=request.use_assistant_message,- request_start_timestamp_ns=request_start_timestamp_ns,- include_return_message_types=request.include_return_message_types,- )-asyncio.create_task(create_background_stream_processor(stream_generator=raw_stream,@@ -1346,55 +1346,21 @@ async def send_message_streaming())- stream = redis_sse_stream_generator(+ raw_stream = redis_sse_stream_generator(redis_client=redis_client,run_id=run.id,)- if request.include_pings and settings.enable_keepalive:- stream = add_keepalive_to_stream(stream, keepalive_interval=settings.keepalive_interval)-- return StreamingResponseWithStatusCode(- stream,- media_type="text/event-stream",- )-- if request.stream_tokens and model_compatible_token_streaming:- raw_stream = agent_loop.step_stream(- input_messages=request.messages,- max_steps=request.max_steps,- use_assistant_message=request.use_assistant_message,- request_start_timestamp_ns=request_start_timestamp_ns,- include_return_message_types=request.include_return_message_types,- )- # Conditionally wrap with keepalive based on request parameter- if request.include_pings and settings.enable_keepalive:- stream = add_keepalive_to_stream(raw_stream, keepalive_interval=settings.keepalive_interval)- else:- stream = raw_stream-- result = StreamingResponseWithStatusCode(- stream,- media_type="text/event-stream",- )+ # Conditionally wrap with keepalive based on request parameter+ if request.include_pings and settings.enable_keepalive:+ stream = add_keepalive_to_stream(raw_stream, keepalive_interval=settings.keepalive_interval)else:- raw_stream = agent_loop.step_stream_no_tokens(- request.messages,- max_steps=request.max_steps,- use_assistant_message=request.use_assistant_message,- request_start_timestamp_ns=request_start_timestamp_ns,- include_return_message_types=request.include_return_message_types,- )- # Conditionally wrap with keepalive based on request parameter- if request.include_pings and settings.enable_keepalive:- stream = add_keepalive_to_stream(raw_stream, keepalive_interval=settings.keepalive_interval)- else:- stream = raw_stream-- result = StreamingResponseWithStatusCode(- stream,- media_type="text/event-stream",- )+ stream = raw_stream++ result = StreamingResponseWithStatusCode(+ stream,+ media_type="text/event-stream",+ )else:result = await server.send_message_to_agent(agent_id=agent_id,@@ -1409,11 +1375,13 @@ async def send_message_streaming(request_start_timestamp_ns=request_start_timestamp_ns,include_return_message_types=request.include_return_message_types,)- job_status = JobStatus.running+ if settings.track_agent_run:+ job_status = JobStatus.runningreturn resultexcept Exception as e:- job_update_metadata = {"error": str(e)}- job_status = JobStatus.failed+ if settings.track_agent_run:+ job_update_metadata = {"error": str(e)}+ job_status = JobStatus.failedraisefinally:if settings.track_agent_run: