Skip to content

armgit / letta

Public Imported from GitHub
0

Platform for stateful agents: AI with advanced memory that can learn and self-improve over time.

feat: remove ssl allocation from startup (#6127)

cthomas0c06dbf047b6
diff --git a/letta/server/rest_api/routers/v1/messages.py b/letta/server/rest_api/routers/v1/messages.py
index 1b4a2ca9f..e4fd7f29a 100644
--- a/letta/server/rest_api/routers/v1/messages.py
+++ b/letta/server/rest_api/routers/v1/messages.py
@@ -191,6 +191,8 @@ async def cancel_batch(
# TODO: Extend to providers beyond anthropic
# TODO: For now, we only support anthropic
# Cancel the job
+ if server.anthropic_async_client is None:
+ raise HTTPException(status_code=501, detail="Batch job cancellation is not enabled")
anthropic_batch_id = llm_batch_job.create_batch_response.id
await server.anthropic_async_client.messages.batches.cancel(anthropic_batch_id)
diff --git a/letta/server/server.py b/letta/server/server.py
index 79968f5e9..a834f4c99 100644
--- a/letta/server/server.py
+++ b/letta/server/server.py
@@ -184,10 +184,18 @@ class SyncServer(object):
message_manager=self.message_manager,
)
- # A resusable httpx client
- timeout = httpx.Timeout(connect=10.0, read=20.0, write=10.0, pool=10.0)
- limits = httpx.Limits(max_connections=100, max_keepalive_connections=80, keepalive_expiry=300)
- self.httpx_client = httpx.AsyncClient(timeout=timeout, follow_redirects=True, limits=limits)
+ if settings.enable_batch_job_polling:
+ # A resusable httpx client
+ timeout = httpx.Timeout(connect=10.0, read=20.0, write=10.0, pool=10.0)
+ limits = httpx.Limits(max_connections=100, max_keepalive_connections=80, keepalive_expiry=300)
+ self.httpx_client = httpx.AsyncClient(timeout=timeout, follow_redirects=True, limits=limits)
+
+ # TODO: Replace this with the Anthropic client we have in house
+ # Reuse the shared httpx client to prevent duplicate SSL contexts and connection pools
+ self.anthropic_async_client = AsyncAnthropic(http_client=self.httpx_client)
+ else:
+ self.httpx_client = None
+ self.anthropic_async_client = None
# For MCP
# TODO: remove this
@@ -198,9 +206,6 @@ class SyncServer(object):
self._llm_config_cache = {}
self._embedding_config_cache = {}
- # TODO: Replace this with the Anthropic client we have in house
- self.anthropic_async_client = AsyncAnthropic()
-
# collect providers (always has Letta as a default)
self._enabled_providers: List[Provider] = [LettaProvider(name="letta")]
if model_settings.openai_api_key: