Platform for stateful agents: AI with advanced memory that can learn and self-improve over time.
feat: poll cancellation status in client.runs.stream endpoint [LET-4209] (#4439)
cthomas7a79ea6b43f4
diff --git a/letta/server/rest_api/routers/v1/runs.py b/letta/server/rest_api/routers/v1/runs.pyindex 3ac34c3db..c5e15a40b 100644--- a/letta/server/rest_api/routers/v1/runs.py+++ b/letta/server/rest_api/routers/v1/runs.py@@ -14,7 +14,11 @@ from letta.schemas.openai.chat_completion_response import UsageStatisticsfrom letta.schemas.run import Runfrom letta.schemas.step import Stepfrom letta.server.rest_api.redis_stream_manager import redis_sse_stream_generator-from letta.server.rest_api.streaming_response import StreamingResponseWithStatusCode, add_keepalive_to_stream+from letta.server.rest_api.streaming_response import (+ StreamingResponseWithStatusCode,+ add_keepalive_to_stream,+ cancellation_aware_stream_wrapper,+)from letta.server.rest_api.utils import get_letta_serverfrom letta.server.server import SyncServerfrom letta.settings import settings@@ -296,6 +300,14 @@ async def retrieve_stream(batch_size=request.batch_size,)+ if settings.enable_cancellation_aware_streaming:+ stream = cancellation_aware_stream_wrapper(+ stream_generator=stream,+ job_manager=server.job_manager,+ job_id=run_id,+ actor=actor,+ )+if request.include_pings and settings.enable_keepalive:stream = add_keepalive_to_stream(stream, keepalive_interval=settings.keepalive_interval)diff --git a/letta/settings.py b/letta/settings.pyindex d491241cb..b2e549185 100644--- a/letta/settings.py+++ b/letta/settings.py@@ -211,6 +211,9 @@ class Settings(BaseSettings):enable_keepalive: bool = Field(True, description="Enable keepalive messages in SSE streams to prevent timeouts")keepalive_interval: float = Field(50.0, description="Seconds between keepalive messages (default: 50)")+ # SSE Streaming cancellation settings+ enable_cancellation_aware_streaming: bool = Field(True, description="Enable cancellation aware streaming")+# default handlesdefault_llm_handle: Optional[str] = Nonedefault_embedding_handle: Optional[str] = None