Skip to content

armgit / letta

Public Imported from GitHub
0

Platform for stateful agents: AI with advanced memory that can learn and self-improve over time.

feat: poll cancellation status in client.runs.stream endpoint [LET-4209] (#4439)

cthomas7a79ea6b43f4
diff --git a/letta/server/rest_api/routers/v1/runs.py b/letta/server/rest_api/routers/v1/runs.py
index 3ac34c3db..c5e15a40b 100644
--- a/letta/server/rest_api/routers/v1/runs.py
+++ b/letta/server/rest_api/routers/v1/runs.py
@@ -14,7 +14,11 @@ from letta.schemas.openai.chat_completion_response import UsageStatistics
from letta.schemas.run import Run
from letta.schemas.step import Step
from letta.server.rest_api.redis_stream_manager import redis_sse_stream_generator
-from letta.server.rest_api.streaming_response import StreamingResponseWithStatusCode, add_keepalive_to_stream
+from letta.server.rest_api.streaming_response import (
+ StreamingResponseWithStatusCode,
+ add_keepalive_to_stream,
+ cancellation_aware_stream_wrapper,
+)
from letta.server.rest_api.utils import get_letta_server
from letta.server.server import SyncServer
from letta.settings import settings
@@ -296,6 +300,14 @@ async def retrieve_stream(
batch_size=request.batch_size,
)
+ if settings.enable_cancellation_aware_streaming:
+ stream = cancellation_aware_stream_wrapper(
+ stream_generator=stream,
+ job_manager=server.job_manager,
+ job_id=run_id,
+ actor=actor,
+ )
+
if request.include_pings and settings.enable_keepalive:
stream = add_keepalive_to_stream(stream, keepalive_interval=settings.keepalive_interval)
diff --git a/letta/settings.py b/letta/settings.py
index d491241cb..b2e549185 100644
--- a/letta/settings.py
+++ b/letta/settings.py
@@ -211,6 +211,9 @@ class Settings(BaseSettings):
enable_keepalive: bool = Field(True, description="Enable keepalive messages in SSE streams to prevent timeouts")
keepalive_interval: float = Field(50.0, description="Seconds between keepalive messages (default: 50)")
+ # SSE Streaming cancellation settings
+ enable_cancellation_aware_streaming: bool = Field(True, description="Enable cancellation aware streaming")
+
# default handles
default_llm_handle: Optional[str] = None
default_embedding_handle: Optional[str] = None