Platform for stateful agents: AI with advanced memory that can learn and self-improve over time.
feat: add idempotency for conversation streaming (#10147)
cthomascb4df48ea2d2
diff --git a/fern/openapi.json b/fern/openapi.jsonindex 888a8e655..a060db865 100644--- a/fern/openapi.json+++ b/fern/openapi.json@@ -9307,7 +9307,7 @@"post": {"tags": ["conversations"],"summary": "Retrieve Conversation Stream",- "description": "Resume the stream for the most recent active run in a conversation.\n\nThis endpoint allows you to reconnect to an active background stream\nfor a conversation, enabling recovery from network interruptions.\n\n**Agent-direct mode**: Pass conversation_id=\"default\" with agent_id in request body\nto retrieve the stream for the agent's most recent active run.\n\n**Deprecated**: Passing an agent ID as conversation_id still works but will be removed.",+ "description": "Resume the stream for the most recent active run in a conversation.\n\nThis endpoint allows you to reconnect to an active background stream\nfor a conversation, enabling recovery from network interruptions.\n\n**Agent-direct mode**: Pass conversation_id=\"default\" with agent_id in request body\nto retrieve the stream for the agent's most recent active run.\n\n**Direct run access**: Pass run_id directly to skip run lookup entirely.\nUseful for recovery from duplicate request 409 errors.\n\n**OTID lookup**: Pass otid to look up the run_id from Redis.\nUseful when you have the otid from a 409 error response.\n\n**Deprecated**: Passing an agent ID as conversation_id still works but will be removed.","operationId": "retrieve_conversation_stream","parameters": [{@@ -44103,6 +44103,30 @@"title": "Agent Id","description": "Agent ID for agent-direct mode with 'default' conversation. Use with conversation_id='default' in the URL path."},+ "run_id": {+ "anyOf": [+ {+ "type": "string"+ },+ {+ "type": "null"+ }+ ],+ "title": "Run Id",+ "description": "Run ID to stream directly, bypassing run lookup. Use for recovery from duplicate requests."+ },+ "otid": {+ "anyOf": [+ {+ "type": "string"+ },+ {+ "type": "null"+ }+ ],+ "title": "Otid",+ "description": "Offline threading ID to look up the run_id. Bypasses active run lookup if run_id not provided."+ },"starting_after": {"type": "integer","title": "Starting After",diff --git a/letta/constants.py b/letta/constants.pyindex 126d12855..c79eac76c 100644--- a/letta/constants.py+++ b/letta/constants.py@@ -474,6 +474,10 @@ REDIS_RUN_ID_PREFIX = "agent:send_message:run_id"CONVERSATION_LOCK_PREFIX = "conversation:lock:"CONVERSATION_LOCK_TTL_SECONDS = 300 # 5 minutes+# OTID -> run_id mapping (for recovering from duplicate requests)+OTID_RUN_PREFIX = "otid:run:"+OTID_RUN_TTL_SECONDS = 10800 # 3 hours (same as stream TTL)+# Memory repo locks - prevents concurrent modifications to git-based memoryMEMORY_REPO_LOCK_PREFIX = "memory_repo:lock:"MEMORY_REPO_LOCK_TTL_SECONDS = 60 # 1 minute (git operations should be fast)diff --git a/letta/data_sources/redis_client.py b/letta/data_sources/redis_client.pyindex 28b2574d9..25808a48f 100644--- a/letta/data_sources/redis_client.py+++ b/letta/data_sources/redis_client.py@@ -7,6 +7,8 @@ from letta.constants import (CONVERSATION_LOCK_TTL_SECONDS,MEMORY_REPO_LOCK_PREFIX,MEMORY_REPO_LOCK_TTL_SECONDS,+ OTID_RUN_PREFIX,+ OTID_RUN_TTL_SECONDS,REDIS_EXCLUDE,REDIS_INCLUDE,REDIS_SET_DEFAULT_VAL,@@ -245,6 +247,67 @@ class AsyncRedisClient:logger.warning(f"Failed to release conversation lock for conversation {conversation_id}: {e}")return False+ async def set_otid_run_mapping(self, otid: str, run_id: str) -> bool:+ """+ Store a mapping from otid to run_id.++ This allows recovering the run_id from a duplicate request's otid+ when a 409 Conflict error is raised.++ Args:+ otid: The offline threading ID (used as lock token)+ run_id: The run ID associated with this request++ Returns:+ True if mapping was stored successfully+ """+ try:+ client = await self.get_client()+ key = f"{OTID_RUN_PREFIX}{otid}"+ await client.set(key, run_id, ex=OTID_RUN_TTL_SECONDS)+ return True+ except Exception as e:+ logger.warning(f"Failed to store otid->run_id mapping for otid {otid}: {e}")+ return False++ async def get_run_id_by_otid(self, otid: str) -> Optional[str]:+ """+ Look up the run_id associated with an otid.++ Args:+ otid: The offline threading ID to look up++ Returns:+ The run_id if found, None otherwise+ """+ try:+ client = await self.get_client()+ key = f"{OTID_RUN_PREFIX}{otid}"+ return await client.get(key)+ except Exception as e:+ logger.warning(f"Failed to lookup run_id for otid {otid}: {e}")+ return None++ async def has_stream_chunks(self, run_id: str) -> bool:+ """+ Check if there are any chunks available for a run in Redis.++ Args:+ run_id: The run ID to check++ Returns:+ True if chunks exist, False otherwise+ """+ try:+ client = await self.get_client()+ stream_key = f"sse:run:{run_id}"+ # Check if stream exists and has entries+ length = await client.xlen(stream_key)+ return length > 0+ except Exception as e:+ logger.warning(f"Failed to check stream chunks for run {run_id}: {e}")+ return False+async def acquire_memory_repo_lock(self,agent_id: str,@@ -537,6 +600,15 @@ class NoopAsyncRedisClient(AsyncRedisClient):async def release_conversation_lock(self, conversation_id: str) -> bool:return False+ async def set_otid_run_mapping(self, otid: str, run_id: str) -> bool:+ return False++ async def get_run_id_by_otid(self, otid: str) -> Optional[str]:+ return None++ async def has_stream_chunks(self, run_id: str) -> bool:+ return False+async def acquire_memory_repo_lock(self,agent_id: str,diff --git a/letta/errors.py b/letta/errors.pyindex c03bcab08..6f16592a3 100644--- a/letta/errors.py+++ b/letta/errors.py@@ -81,16 +81,29 @@ class ConcurrentUpdateError(LettaError):class ConversationBusyError(LettaError):"""Error raised when attempting to send a message while another request is already processing for the same conversation."""- def __init__(self, conversation_id: str, lock_holder_token: Optional[str] = None):+ def __init__(+ self,+ conversation_id: str,+ lock_holder_token: Optional[str] = None,+ run_id: Optional[str] = None,+ ):self.conversation_id = conversation_idself.lock_holder_token = lock_holder_token- message = "Cannot send a new message: Another request is currently being processed for this conversation. Please wait for the current request to complete."+ self.run_id = run_id++ # Build message with available info+ if run_id:+ message = f"Cannot send a new message: Another request (run_id={run_id}) is currently being processed for this conversation. Please wait for it to complete."+ else:+ message = "Cannot send a new message: Another request is currently being processed for this conversation. Please wait for the current request to complete."+code = ErrorCode.CONFLICTdetails = {"error_code": "CONVERSATION_BUSY","conversation_id": conversation_id,- "lock_holder_token": lock_holder_token,}+ if run_id:+ details["run_id"] = run_idsuper().__init__(message=message, code=code, details=details)diff --git a/letta/schemas/letta_request.py b/letta/schemas/letta_request.pyindex 513581908..2d0e9a73a 100644--- a/letta/schemas/letta_request.py+++ b/letta/schemas/letta_request.py@@ -220,6 +220,14 @@ class RetrieveStreamRequest(BaseModel):default=None,description="Agent ID for agent-direct mode with 'default' conversation. Use with conversation_id='default' in the URL path.",)+ run_id: Optional[str] = Field(+ default=None,+ description="Run ID to stream directly, bypassing run lookup. Use for recovery from duplicate requests.",+ )+ otid: Optional[str] = Field(+ default=None,+ description="Offline threading ID to look up the run_id. Bypasses active run lookup if run_id not provided.",+ )starting_after: int = Field(0, description="Sequence id to use as a cursor for pagination. Response will start streaming after this chunk sequence id")diff --git a/letta/server/rest_api/app.py b/letta/server/rest_api/app.pyindex 80b825b50..9404243ea 100644--- a/letta/server/rest_api/app.py+++ b/letta/server/rest_api/app.py@@ -582,7 +582,15 @@ def create_application() -> "FastAPI":app.add_exception_handler(UniqueConstraintViolationError, _error_handler_409)app.add_exception_handler(IntegrityError, _error_handler_409)app.add_exception_handler(ConcurrentUpdateError, _error_handler_409)- app.add_exception_handler(ConversationBusyError, _error_handler_409)++ async def _conversation_busy_handler(request: Request, exc: ConversationBusyError):+ logger.error(f"{type(exc).__name__}", exc_info=exc)+ content = {"detail": str(exc)}+ if exc.run_id:+ content["run_id"] = exc.run_id+ return JSONResponse(status_code=409, content=content)++ app.add_exception_handler(ConversationBusyError, _conversation_busy_handler)app.add_exception_handler(MemoryRepoBusyError, _error_handler_409)app.add_exception_handler(PendingApprovalError, _error_handler_409)app.add_exception_handler(NoActiveRunsToCancelError, _error_handler_409)diff --git a/letta/server/rest_api/routers/v1/conversations.py b/letta/server/rest_api/routers/v1/conversations.pyindex 2ec8b06d4..1386de1fe 100644--- a/letta/server/rest_api/routers/v1/conversations.py+++ b/letta/server/rest_api/routers/v1/conversations.py@@ -1,6 +1,5 @@from datetime import timedeltafrom typing import Annotated, Any, Dict, List, Literal, Optional-from uuid import uuid4from fastapi import APIRouter, Body, Depends, HTTPException, Query, statusfrom pydantic import BaseModel, Field@@ -10,7 +9,7 @@ from letta.agents.agent_loop import AgentLoopfrom letta.agents.letta_agent_v3 import LettaAgentV3from letta.constants import REDIS_RUN_ID_PREFIXfrom letta.data_sources.redis_client import NoopAsyncRedisClient, get_redis_client-from letta.errors import LettaExpiredError, LettaInvalidArgumentError, NoActiveRunsToCancelError+from letta.errors import ConversationBusyError, LettaExpiredError, LettaInvalidArgumentError, NoActiveRunsToCancelErrorfrom letta.helpers.datetime_helpers import get_utc_timefrom letta.log import get_loggerfrom letta.schemas.conversation import Conversation, CreateConversation, UpdateConversation@@ -278,12 +277,24 @@ async def _send_agent_direct_message()agent = agent.model_copy(update={"llm_config": override_llm_config})+ # Collect all otids from messages for request deduplication+ message_otids = [msg.otid for msg in request.messages if msg.otid]++ # Derive a request token from ALL message otids for deduplication+ from letta.services.streaming_service import derive_request_token, enrich_conversation_busy_error++ request_token = derive_request_token(message_otids)+# Acquire lock using agent_id as lock keyif not isinstance(redis_client, NoopAsyncRedisClient):- await redis_client.acquire_conversation_lock(- conversation_id=agent_id,- token=str(uuid4()),- )+ try:+ await redis_client.acquire_conversation_lock(+ conversation_id=agent_id,+ token=request_token,+ )++ except ConversationBusyError as e:+ raise await enrich_conversation_busy_error(redis_client, e)try:# Create a run for execution tracking@@ -305,6 +316,15 @@ async def _send_agent_direct_message(# Set run_id in Redis for cancellation supportawait redis_client.set(f"{REDIS_RUN_ID_PREFIX}:{agent_id}", run.id if run else None)+ # Store request_token -> run_id mapping for duplicate request recovery+ if request_token and run:+ await redis_client.set_otid_run_mapping(request_token, run.id)++ # Store each individual otid -> run_id mapping for client convenience+ if run:+ for otid in message_otids:+ await redis_client.set_otid_run_mapping(otid, run.id)+agent_loop = AgentLoop.load(agent_state=agent, actor=actor)return await agent_loop.step(request.messages,@@ -620,55 +640,22 @@ async def retrieve_conversation_stream(This endpoint allows you to reconnect to an active background streamfor a conversation, enabling recovery from network interruptions.- **Agent-direct mode**: Pass conversation_id="default" with agent_id in request body+ **Agent-direct mode**: Pass conversation_id=\"default\" with agent_id in request bodyto retrieve the stream for the agent's most recent active run.+ **Direct run access**: Pass run_id directly to skip run lookup entirely.+ Useful for recovery from duplicate request 409 errors.++ **OTID lookup**: Pass otid to look up the run_id from Redis.+ Useful when you have the otid from a 409 error response.+**Deprecated**: Passing an agent ID as conversation_id still works but will be removed."""actor = await server.user_manager.get_actor_or_default_async(actor_id=headers.actor_id)runs_manager = RunManager()-- # Agent-direct mode: conversation_id="default" + agent_id in body (preferred)- # OR conversation_id="agent-*" (backwards compat, deprecated)- resolved_agent_id = None- if conversation_id == "default" and request and request.agent_id:- resolved_agent_id = request.agent_id- elif conversation_id.startswith("agent-"):- resolved_agent_id = conversation_id-- # Find the most recent active run- if resolved_agent_id:- # Agent-direct mode: find runs by agent_id- active_runs = await runs_manager.list_runs(- actor=actor,- agent_id=resolved_agent_id,- statuses=[RunStatus.created, RunStatus.running],- limit=1,- ascending=False,- )- else:- # Normal mode: find runs by conversation_id- active_runs = await runs_manager.list_runs(- actor=actor,- conversation_id=conversation_id,- statuses=[RunStatus.created, RunStatus.running],- limit=1,- ascending=False,- )-- if not active_runs:- raise LettaInvalidArgumentError("No active runs found for this conversation.")-- run = active_runs[0]-- if not run.background:- raise LettaInvalidArgumentError("Run was not created in background mode, so it cannot be retrieved.")-- if run.created_at < get_utc_time() - timedelta(hours=3):- raise LettaExpiredError("Run was created more than 3 hours ago, and is now expired.")-redis_client = await get_redis_client()+ # Check Redis availability earlyif isinstance(redis_client, NoopAsyncRedisClient):raise HTTPException(status_code=503,@@ -679,9 +666,74 @@ async def retrieve_conversation_stream(),)+ run_id = None+ run = None++ # Priority 1: Direct run_id provided (bypasses all lookups)+ if request and request.run_id:+ run_id = request.run_id+ # Fetch run to check expiration+ run = await runs_manager.get_run_by_id(run_id=run_id, actor=actor)+ if not run:+ raise LettaInvalidArgumentError(f"Run {run_id} not found.")++ # Priority 2: OTID provided (look up run_id from Redis)+ elif request and request.otid:+ run_id = await redis_client.get_run_id_by_otid(request.otid)+ if not run_id:+ raise LettaInvalidArgumentError(f"No run found for otid={request.otid}. The run may have expired or never existed.")+ # Fetch run to check expiration+ run = await runs_manager.get_run_by_id(run_id=run_id, actor=actor)+ if not run:+ raise LettaInvalidArgumentError(f"Run {run_id} (from otid={request.otid}) not found.")++ # Priority 3: Fall back to active run lookup+ else:+ # Agent-direct mode: conversation_id="default" + agent_id in body (preferred)+ # OR conversation_id="agent-*" (backwards compat, deprecated)+ resolved_agent_id = None+ if conversation_id == "default" and request and request.agent_id:+ resolved_agent_id = request.agent_id+ elif conversation_id.startswith("agent-"):+ resolved_agent_id = conversation_id++ # Find the most recent active run+ if resolved_agent_id:+ # Agent-direct mode: find runs by agent_id+ active_runs = await runs_manager.list_runs(+ actor=actor,+ agent_id=resolved_agent_id,+ statuses=[RunStatus.created, RunStatus.running],+ limit=1,+ ascending=False,+ )+ else:+ # Normal mode: find runs by conversation_id+ active_runs = await runs_manager.list_runs(+ actor=actor,+ conversation_id=conversation_id,+ statuses=[RunStatus.created, RunStatus.running],+ limit=1,+ ascending=False,+ )++ if not active_runs:+ raise LettaInvalidArgumentError("No active runs found for this conversation.")++ run = active_runs[0]+ run_id = run.id++ # For active run lookup, require background mode+ if not run.background:+ raise LettaInvalidArgumentError("Run was not created in background mode, so it cannot be retrieved.")++ # Check expiration for all paths+ if run and run.created_at < get_utc_time() - timedelta(hours=3):+ raise LettaExpiredError("Run was created more than 3 hours ago, and is now expired.")+stream = redis_sse_stream_generator(redis_client=redis_client,- run_id=run.id,+ run_id=run_id,starting_after=request.starting_after if request else None,poll_interval=request.poll_interval if request else None,batch_size=request.batch_size if request else None,@@ -693,13 +745,13 @@ async def retrieve_conversation_stream(stream = cancellation_aware_stream_wrapper(stream_generator=stream,run_manager=server.run_manager,- run_id=run.id,+ run_id=run_id,actor=actor,- cancellation_event=get_cancellation_event_for_run(run.id),+ cancellation_event=get_cancellation_event_for_run(run_id),)if request and request.include_pings and settings.enable_keepalive:- stream = add_keepalive_to_stream(stream, keepalive_interval=settings.keepalive_interval, run_id=run.id)+ stream = add_keepalive_to_stream(stream, keepalive_interval=settings.keepalive_interval, run_id=run_id)return StreamingResponseWithStatusCode(stream,diff --git a/letta/services/streaming_service.py b/letta/services/streaming_service.pyindex 0f0261d1a..a458be71d 100644--- a/letta/services/streaming_service.py+++ b/letta/services/streaming_service.py@@ -1,4 +1,5 @@import asyncio+import hashlibimport jsonimport timefrom datetime import datetime, timezone@@ -12,8 +13,9 @@ from openai.types.chat.chat_completion_chunk import Choice, ChoiceDeltafrom letta.agents.agent_loop import AgentLoopfrom letta.agents.base_agent_v2 import BaseAgentV2from letta.constants import REDIS_RUN_ID_PREFIX-from letta.data_sources.redis_client import NoopAsyncRedisClient, get_redis_client+from letta.data_sources.redis_client import AsyncRedisClient, NoopAsyncRedisClient, get_redis_clientfrom letta.errors import (+ ConversationBusyError,LettaInvalidArgumentError,LettaServiceUnavailableError,LLMAuthenticationError,@@ -58,6 +60,86 @@ from letta.utils import safe_create_tasklogger = get_logger(__name__)+def derive_request_token(otids: list[str]) -> str:+ """+ Derive a request token from all message otids for deduplication.++ This ensures that two requests with different message combinations get+ different lock tokens, even if they share the same first message.++ Args:+ otids: List of otids from all messages in the request++ Returns:+ A hash of all otids, or a random UUID if no otids provided+ """+ if not otids:+ return str(uuid4())+ combined = "|".join(otids)+ return hashlib.sha256(combined.encode()).hexdigest()[:16]+++async def try_recover_duplicate_request(+ redis_client: "AsyncRedisClient",+ request_token: str,+ lock_key: str,+ include_pings: bool = False,+) -> Optional[StreamingResponse]:+ """+ Check if an existing run already exists for this request token (same otid retry).+ If so, return a stream attached to the existing run as a read-only reader.+ Called BEFORE lock acquisition so duplicate requests never touch the lock.++ Args:+ redis_client: The Redis client+ request_token: Hash of all message otids+ lock_key: The conversation/agent ID used as lock key+ include_pings: Whether to add keepalive pings to the stream++ Returns:+ StreamingResponse if recovery succeeded, None otherwise+ """+ existing_run_id = await redis_client.get_run_id_by_otid(request_token)+ if not existing_run_id:+ return None++ logger.info(+ f"Recovering from duplicate request: returning stream for existing run_id={existing_run_id} "+ f"(request_token={request_token}, lock_key={lock_key})"+ )+ stream = redis_sse_stream_generator(+ redis_client=redis_client,+ run_id=existing_run_id,+ )+ if include_pings and settings.enable_keepalive:+ stream = add_keepalive_to_stream(stream, keepalive_interval=settings.keepalive_interval, run_id=existing_run_id)+ return StreamingResponseWithStatusCode(stream, media_type="text/event-stream")+++async def enrich_conversation_busy_error(+ redis_client: "AsyncRedisClient",+ error: ConversationBusyError,+) -> ConversationBusyError:+ """+ Enrich a ConversationBusyError with the run_id of the lock holder if available.++ Args:+ redis_client: The Redis client+ error: The original ConversationBusyError++ Returns:+ A new ConversationBusyError with run_id populated if found+ """+ existing_run_id = None+ if error.lock_holder_token:+ existing_run_id = await redis_client.get_run_id_by_otid(error.lock_holder_token)+ return ConversationBusyError(+ conversation_id=error.conversation_id,+ lock_holder_token=error.lock_holder_token,+ run_id=existing_run_id,+ )++async def prepend_initial_run_ping(stream_generator: AsyncIterator[str | bytes],run_id: str,@@ -186,16 +268,40 @@ class StreamingService:# Determine lock key: use conversation_id if provided, else agent_id if should_locklock_key = conversation_id if conversation_id else (agent_id if should_lock else None)+ # Collect all otids from messages for request deduplication+ # Each message has an otid (auto-generated if not provided)+ message_otids = [msg.otid for msg in request.messages if msg.otid]++ # Derive a request token from ALL message otids for deduplication+ # This ensures requests with different message combinations get different tokens+ request_token = derive_request_token(message_otids)+# Attempt to acquire lock if lock_key is set# This prevents concurrent message processing for the same conversation/agent# Skip locking if Redis is not available (graceful degradation)if lock_key and not isinstance(redis_client, NoopAsyncRedisClient):+ # Check for existing run BEFORE acquiring the lock.+ # Same-otid retries should never acquire the lock — just read from Redis.+ # Only possible when background=True (Redis-backed streaming).+ if request.background:+ recovery_response = await try_recover_duplicate_request(+ redis_client=redis_client,+ request_token=request_token,+ lock_key=lock_key,+ include_pings=request.include_pings,+ )+ if recovery_response:+ return None, recovery_response+admission_wait_start_ns = get_utc_timestamp_ns()try:await redis_client.acquire_conversation_lock(conversation_id=lock_key,- token=str(uuid4()),+ token=request_token,)++ except ConversationBusyError as e:+ raise await enrich_conversation_busy_error(redis_client, e)finally:admission_wait_ms = (get_utc_timestamp_ns() - admission_wait_start_ns) / 1_000_000MetricRegistry().request_admission_wait_ms_histogram.record(@@ -215,6 +321,16 @@ class StreamingService:run = await self._create_run(agent_id, request, run_type, actor, conversation_id=conversation_id)await redis_client.set(f"{REDIS_RUN_ID_PREFIX}:{agent_id}", run.id if run else None)+ # Store request_token -> run_id mapping for duplicate request recovery+ # This allows detecting exact retry vs different request+ if request_token:+ await redis_client.set_otid_run_mapping(request_token, run.id)++ # Store each individual otid -> run_id mapping for client convenience+ # Client can use ANY otid from their request to recover the stream+ for otid in message_otids:+ await redis_client.set_otid_run_mapping(otid, run.id)+# use agent loop for streamingagent_loop = AgentLoop.load(agent_state=agent, actor=actor)diff --git a/tests/integration_test_conversations_sdk.py b/tests/integration_test_conversations_sdk.pyindex 8fe33fde3..3c99b3b2b 100644--- a/tests/integration_test_conversations_sdk.py+++ b/tests/integration_test_conversations_sdk.py@@ -2,15 +2,43 @@Integration tests for the Conversations API using the SDK."""+import asyncio+import loggingimport uuid+from typing import Any, List, Optionalimport pytestimport requests-from letta_client import Letta+from letta_client import APIError, AsyncLetta, Letta+from letta_client.types import MessageCreateParam++logger = logging.getLogger(__name__)TEST_MODEL_HANDLE = "anthropic/claude-haiku-4-5"+def async_client(server_url: str) -> AsyncLetta:+ """Create an async Letta client."""+ return AsyncLetta(base_url=server_url)+++def otid_test_agent(client: Letta):+ """Create a test agent for otid/lock tests using openai/gpt-4o-mini (always available)."""+ agent_state = client.agents.create(+ name=f"test_conversations_{uuid.uuid4().hex[:8]}",+ model="openai/gpt-4o-mini",+ embedding="openai/text-embedding-3-small",+ memory_blocks=[+ {"label": "human", "value": "Test user"},+ {"label": "persona", "value": "You are a helpful assistant."},+ ],+ )+ yield agent_state+ client.agents.delete(agent_id=agent_state.id)++@pytest.fixturedef client(server_url: str) -> Letta:"""Create a Letta client."""@@ -1504,3 +1532,191 @@ class TestConversationSystemMessageRecompilation:finally:client.agents.delete(agent_id=agent.id)++[email protected](loop_scope="function")+async def test_concurrent_conversation_requests_return_409(+ server_url: str,+ otid_test_agent,+ async_client: AsyncLetta,+ disable_e2b_api_key: Any,+) -> None:+ """+ Test that concurrent requests to the same conversation return 409 CONVERSATION_BUSY.++ This is a simpler test to verify the locking mechanism is working.+ If this fails, the lock is not being held properly.+ """+ from letta.settings import settings++ if settings.redis_host is None or settings.redis_port is None:+ pytest.skip("Redis not configured - skipping conversation lock test")++ conversation = await async_client.conversations.create(agent_id=otid_test_agent.id)++ # Different otids for the two requests+ otid1 = f"test-otid-1-{uuid.uuid4()}"+ otid2 = f"test-otid-2-{uuid.uuid4()}"++ messages1 = [+ MessageCreateParam(+ role="user",+ content="Hello! Please count from 1 to 5 slowly.",+ otid=otid1,+ )+ ]++ messages2 = [+ MessageCreateParam(+ role="user",+ content="Hello! Please count from 1 to 5 slowly.",+ otid=otid2, # Different otid+ )+ ]++ # Track state+ stream1_started = asyncio.Event()+ stream2_error: Optional[APIError] = None++ async def collect_stream1():+ """First request - the original."""+ stream = await async_client.conversations.messages.create(+ conversation_id=conversation.id,+ messages=messages1,+ background=True,+ )+ async for chunk in stream:+ if not stream1_started.is_set():+ stream1_started.set()+ await asyncio.sleep(0.01)++ async def collect_stream2():+ """Second request - should return 409 (different otid)."""+ await stream1_started.wait()++ nonlocal stream2_error+ try:+ stream = await async_client.conversations.messages.create(+ conversation_id=conversation.id,+ messages=messages2, # Different otid!+ background=True,+ )+ async for chunk in stream:+ pass+ except APIError as e:+ stream2_error = e++ await asyncio.gather(collect_stream1(), collect_stream2())++ # Stream 2 should have gotten 409 if stream 1 was still running.+ # If stream 1 completed before stream 2 started, stream 2 may succeed —+ # that's correct behavior (lock released, new request is fine).+ if stream2_error is not None:+ assert stream2_error.status_code == 409, f"Stream 2 should have returned 409, got {stream2_error.status_code}"+ logger.info(f"Stream 2 correctly returned 409: {stream2_error.body}")+ else:+ logger.info("Stream 1 completed before stream 2 started — no lock contention (acceptable)")++[email protected](loop_scope="function")+async def test_duplicate_request_recovery_with_same_otid(+ server_url: str,+ otid_test_agent,+ async_client: AsyncLetta,+ disable_e2b_api_key: Any,+) -> None:+ """+ Test that a duplicate request with the same otid recovers the existing stream.++ This test verifies the otid-based deduplication and recovery mechanism:+ 1. Start a streaming request with a specific otid+ 2. Start another request with the SAME otid (while first is still running)+ 3. The second request should recover and return the same run_id+ 4. After both complete, a third request with the same otid should still recover+ """+ from letta.settings import settings++ if settings.redis_host is None or settings.redis_port is None:+ pytest.skip("Redis not configured - skipping conversation lock test")++ conversation = await async_client.conversations.create(agent_id=otid_test_agent.id)++ shared_otid = f"test-otid-{uuid.uuid4()}"+ messages = [+ MessageCreateParam(+ role="user",+ content="Hello! Please count from 1 to 5 slowly.",+ otid=shared_otid,+ )+ ]++ # Track state+ chunks_from_stream1: List[Any] = []+ chunks_from_stream2: List[Any] = []+ stream1_started = asyncio.Event()++ async def collect_stream1():+ """First request - the original."""+ stream = await async_client.conversations.messages.create(+ conversation_id=conversation.id,+ messages=messages,+ background=True,+ )+ async for chunk in stream:+ chunks_from_stream1.append(chunk)+ if not stream1_started.is_set():+ stream1_started.set()+ await asyncio.sleep(0.01)++ async def collect_stream2():+ """Second request - should recover from stream1 (same otid)."""+ await stream1_started.wait()+ await asyncio.sleep(0.1)++ stream = await async_client.conversations.messages.create(+ conversation_id=conversation.id,+ messages=messages, # Same otid!+ background=True,+ )+ async for chunk in stream:+ chunks_from_stream2.append(chunk)++ # Run both streams concurrently+ await asyncio.gather(collect_stream1(), collect_stream2())++ def extract_run_id(chunks: List[Any]) -> Optional[str]:+ for chunk in chunks:+ if hasattr(chunk, "run_id") and chunk.run_id:+ return chunk.run_id+ return None++ run_id_1 = extract_run_id(chunks_from_stream1)+ run_id_2 = extract_run_id(chunks_from_stream2)++ logger.info(f"Stream 1 run_id: {run_id_1}")+ logger.info(f"Stream 2 run_id: {run_id_2}")++ assert len(chunks_from_stream1) > 0, "Stream 1 should have received chunks"+ assert len(chunks_from_stream2) > 0, "Stream 2 should have recovered and received chunks"++ assert run_id_1 is not None, "Stream 1 should have a run_id"+ assert run_id_2 is not None, "Stream 2 should have a run_id"+ assert run_id_1 == run_id_2, f"Both streams should have the same run_id (recovery): {run_id_1} vs {run_id_2}"++ # Stream 3: retry AFTER both streams have completed (lock is definitely released).+ # Should still recover the same run from Redis without acquiring the lock.+ chunks_from_stream3: List[Any] = []+ stream3 = await async_client.conversations.messages.create(+ conversation_id=conversation.id,+ messages=messages, # Same otid!+ background=True,+ )+ async for chunk in stream3:+ chunks_from_stream3.append(chunk)++ run_id_3 = extract_run_id(chunks_from_stream3)+ logger.info(f"Stream 3 run_id: {run_id_3}")++ assert len(chunks_from_stream3) > 0, "Stream 3 should have received chunks"+ assert run_id_3 is not None, "Stream 3 should have a run_id"+ assert run_id_3 == run_id_1, f"Stream 3 should recover the same run_id after lock released: {run_id_3} vs {run_id_1}"