Platform for stateful agents: AI with advanced memory that can learn and self-improve over time.
feat: introduce asyncio shield to stream response (#3992)
cthomaseb472dc1e043
diff --git a/letta/server/rest_api/streaming_response.py b/letta/server/rest_api/streaming_response.pyindex 59aa4bc42..87c8d60ab 100644--- a/letta/server/rest_api/streaming_response.py+++ b/letta/server/rest_api/streaming_response.py@@ -15,6 +15,7 @@ from letta.schemas.letta_ping import LettaPingfrom letta.schemas.user import Userfrom letta.server.rest_api.utils import capture_sentry_exceptionfrom letta.services.job_manager import JobManager+from letta.settings import settingslogger = get_logger(__name__)@@ -177,6 +178,18 @@ class StreamingResponseWithStatusCode(StreamingResponse):response_started: bool = Falseasync def stream_response(self, send: Send) -> None:+ if settings.use_asyncio_shield:+ try:+ await asyncio.shield(self._protected_stream_response(send))+ except asyncio.CancelledError:+ logger.info(f"Stream response was cancelled, but shielded task should continue")+ except Exception as e:+ logger.error(f"Error in protected stream response: {e}")+ raise+ else:+ await self._protected_stream_response(send)++ async def _protected_stream_response(self, send: Send) -> None:more_body = Truetry:first_chunk = await self.body_iterator.__anext__()diff --git a/letta/settings.py b/letta/settings.pyindex 0aeae09c0..f497e6fd4 100644--- a/letta/settings.py+++ b/letta/settings.py@@ -268,6 +268,7 @@ class Settings(BaseSettings):# experimental toggleuse_experimental: bool = Falseuse_vertex_structured_outputs_experimental: bool = False+ use_asyncio_shield: bool = True# Database pool monitoringenable_db_pool_monitoring: bool = True # Enable connection pool monitoring