π OpenHands: AI-Driven Development
fix: prevent conversation export overload (#14899)
Graham Neubig9c299e62708f
diff --git a/enterprise/storage/redis.py b/enterprise/storage/redis.pyindex cc1869effc..38c703556d 100644--- a/enterprise/storage/redis.py+++ b/enterprise/storage/redis.py@@ -1,72 +1,24 @@-import os-import threading-from redis import Redisfrom redis import asyncio as aioredisfrom redis import exceptions as redis_exceptions-# Redis configuration-REDIS_HOST = os.environ.get('REDIS_HOST', 'localhost')-REDIS_PORT = int(os.environ.get('REDIS_PORT', '6379'))-REDIS_PASSWORD = os.environ.get('REDIS_PASSWORD', '')-REDIS_DB = int(os.environ.get('REDIS_DB', '0'))-REDIS_SOCKET_TIMEOUT = 2--_redis_client: Redis | None = None-_redis_client_async: aioredis.Redis | None = None-_redis_lock = threading.Lock()---def _get_redis_kwargs():- """Return common kwargs for Redis client creation."""- return {- 'host': REDIS_HOST,- 'port': REDIS_PORT,- 'password': REDIS_PASSWORD,- 'db': REDIS_DB,- 'socket_timeout': REDIS_SOCKET_TIMEOUT,- }---def get_redis_client() -> Redis:- """Get a shared synchronous Redis client, lazily initialized.-- Thread-safe with double-checked locking pattern.-- Returns:- A Redis client for synchronous operations.- """- global _redis_client- if _redis_client is None:- with _redis_lock:- if _redis_client is None:- _redis_client = Redis(**_get_redis_kwargs())- return _redis_client---def get_redis_client_async() -> aioredis.Redis:- """Get a shared asynchronous Redis client, lazily initialized.-- Note: This function is synchronous but returns an async client.- Thread-safe initialization is handled via a threading lock since- asyncio.Lock cannot be used in a sync context.-- Returns:- An aioredis client for asynchronous operations.- """- global _redis_client_async- if _redis_client_async is None:- with _redis_lock:- if _redis_client_async is None:- _redis_client_async = aioredis.Redis(**_get_redis_kwargs())- return _redis_client_async---def get_redis_authed_url():- return f'redis://:{REDIS_PASSWORD}@{REDIS_HOST}:{REDIS_PORT}/{REDIS_DB}'-+from openhands.app_server.utils.redis import (+ REDIS_DB,+ REDIS_HOST,+ REDIS_PASSWORD,+ REDIS_PORT,+ REDIS_SOCKET_TIMEOUT,+ get_redis_authed_url,+ get_redis_client,+ get_redis_client_async,+)__all__ = [+ 'REDIS_DB',+ 'REDIS_HOST',+ 'REDIS_PASSWORD',+ 'REDIS_PORT',+ 'REDIS_SOCKET_TIMEOUT','Redis','aioredis','get_redis_client',diff --git a/openhands/app_server/app_conversation/app_conversation_router.py b/openhands/app_server/app_conversation/app_conversation_router.pyindex d4f7dfbf07..83e26e7fca 100644--- a/openhands/app_server/app_conversation/app_conversation_router.py+++ b/openhands/app_server/app_conversation/app_conversation_router.py@@ -13,7 +13,7 @@ from typing import Annotated, Any, AsyncGenerator, Literalfrom uuid import UUIDimport httpx-from fastapi import APIRouter, Depends, HTTPException, Query, Request, Response, status+from fastapi import APIRouter, Depends, HTTPException, Query, Request, statusfrom fastapi.responses import JSONResponse, StreamingResponsefrom sqlalchemy.ext.asyncio import AsyncSession@@ -43,6 +43,9 @@ from openhands.app_server.app_conversation.app_conversation_models import ()from openhands.app_server.app_conversation.app_conversation_service import (AppConversationService,+ ConversationExportAlreadyRunning,+ ConversationExportLockUnavailable,+ ConversationExportTooLarge,)from openhands.app_server.app_conversation.app_conversation_service_base import (AppConversationServiceBase,@@ -1563,8 +1566,9 @@ async def export_conversation(A zip file containing the conversation trajectory"""try:- # Get the zip file content- zip_content = await app_conversation_service.export_conversation(+ # Prepare the zip stream before sending headers so lock and validation+ # errors can still be returned as HTTP status codes.+ zip_stream = await app_conversation_service.open_conversation_export(conversation_id)@@ -1591,9 +1595,8 @@ async def export_conversation(except Exception:logger.exception('analytics:trajectory_downloaded:failed')- # Return as a downloadable zip file- return Response(- content=zip_content,+ return StreamingResponse(+ zip_stream,media_type='application/zip',headers={'Content-Disposition': f'attachment; filename="conversation_{conversation_id}.zip"'@@ -1601,6 +1604,12 @@ async def export_conversation()except ValueError as e:raise HTTPException(status_code=404, detail=str(e))+ except ConversationExportAlreadyRunning as e:+ raise HTTPException(status_code=409, detail=str(e))+ except ConversationExportLockUnavailable as e:+ raise HTTPException(status_code=503, detail=str(e))+ except ConversationExportTooLarge as e:+ raise HTTPException(status_code=413, detail=str(e))except Exception as e:raise HTTPException(status_code=500, detail=f'Failed to download trajectory: {str(e)}'diff --git a/openhands/app_server/app_conversation/app_conversation_service.py b/openhands/app_server/app_conversation/app_conversation_service.pyindex 6be1d32ddf..b2aa155eb7 100644--- a/openhands/app_server/app_conversation/app_conversation_service.py+++ b/openhands/app_server/app_conversation/app_conversation_service.py@@ -18,6 +18,18 @@ from openhands.sdk.utils.models import DiscriminatedUnionMixinfrom openhands.sdk.workspace.remote.async_remote_workspace import AsyncRemoteWorkspace+class ConversationExportAlreadyRunning(Exception):+ """Raised when another worker is already exporting the same conversation."""+++class ConversationExportLockUnavailable(Exception):+ """Raised when the export lock cannot be checked."""+++class ConversationExportTooLarge(Exception):+ """Raised when a conversation exceeds the configured export event limit."""++class AppConversationService(ABC):"""Service for managing conversations running in sandboxes."""@@ -160,6 +172,21 @@ class AppConversationService(ABC):Returns the zip file as bytes."""+ async def open_conversation_export(+ self, conversation_id: UUID+ ) -> AsyncGenerator[bytes, None]:+ """Prepare a streaming conversation trajectory export.++ Implementations may override this to acquire locks or stream zip data.+ The default preserves compatibility with byte-returning exporters.+ """+ content = await self.export_conversation(conversation_id)++ async def stream():+ yield content++ return stream()+class AppConversationServiceInjector(DiscriminatedUnionMixin, Injector[AppConversationService], ABCdiff --git a/openhands/app_server/app_conversation/live_status_app_conversation_service.py b/openhands/app_server/app_conversation/live_status_app_conversation_service.pyindex e3473c4769..eccc2b1b8b 100644--- a/openhands/app_server/app_conversation/live_status_app_conversation_service.py+++ b/openhands/app_server/app_conversation/live_status_app_conversation_service.py@@ -1,15 +1,15 @@import asyncioimport importlib.metadata+import ioimport jsonimport loggingimport os-import tempfileimport zipfilefrom collections import defaultdictfrom collections.abc import Mappingfrom dataclasses import dataclass, fieldfrom datetime import datetime, timedelta-from typing import Any, AsyncGenerator, Sequence, cast+from typing import Any, AsyncGenerator, BinaryIO, Sequence, castfrom uuid import UUID, uuid4import httpx@@ -44,6 +44,9 @@ from openhands.app_server.app_conversation.app_conversation_models import (from openhands.app_server.app_conversation.app_conversation_service import (AppConversationService,AppConversationServiceInjector,+ ConversationExportAlreadyRunning,+ ConversationExportLockUnavailable,+ ConversationExportTooLarge,)from openhands.app_server.app_conversation.app_conversation_service_base import (AppConversationServiceBase,@@ -105,6 +108,12 @@ from openhands.app_server.utils.llm_metadata import (get_llm_metadata,should_set_litellm_extra_body,)+from openhands.app_server.utils.redis_lock import (+ LockError,+ RedisLockUnavailable,+ refresh_lock_periodically,+ try_acquire_redis_lock,+)from openhands.sdk import Agent, AgentContext, LocalWorkspacefrom openhands.sdk.hooks import HookConfigfrom openhands.sdk.llm import LLM@@ -114,7 +123,6 @@ from openhands.sdk.secret import LookupSecret, StaticSecretfrom openhands.sdk.settings import ACPAgentSettingsfrom openhands.sdk.subagent import get_registered_agent_definitionsfrom openhands.sdk.tool.builtins import SwitchLLMTool-from openhands.sdk.utils.paging import page_iteratorfrom openhands.sdk.utils.redact import (redact_api_key_literals,redact_text_secrets,@@ -133,6 +141,37 @@ from openhands.tools.preset.planning import (_conversation_info_type_adapter = TypeAdapter(list[ConversationInfo | None])_logger = logging.getLogger(__name__)+_EXPORT_LOCK_KEY_PREFIX = 'app_conversation_export'+++class _StreamingZipBuffer(io.RawIOBase):+ """Small non-seekable writer used by zipfile to emit chunks incrementally.++ zipfile.ZipFile only needs write() and tell() from its underlying file+ object when writing to a non-seekable stream. Everything else+ (flush, writable, seekable) is either unused by zipfile or already+ handled by the io.RawIOBase defaults.+ """++ def __init__(self):+ self._chunks: list[bytes] = []+ self._position = 0++ def write(self, data) -> int:+ chunk = bytes(data)+ if chunk:+ self._chunks.append(chunk)+ self._position += len(chunk)+ return len(data)++ def tell(self) -> int:+ return self._position++ def drain(self) -> list[bytes]:+ chunks = self._chunks+ self._chunks = []+ return chunks+def _expected_sdk_version() -> str | None:"""App's pinned openhands-sdk version, or None if its metadata is unresolvable."""@@ -181,6 +220,10 @@ class LiveStatusAppConversationService(AppConversationServiceBase):default_factory=ConversationSecretEnricher)app_mode: str | None = None+ export_max_events: int = 10000+ export_lock_ttl_seconds: int = 3600+ export_lock_refresh_interval_seconds: int = 30+ export_lock_required: bool | None = Noneasync def _get_sandbox_grouping_strategy(self) -> SandboxGroupingStrategy:"""Get the sandbox grouping strategy from user settings."""@@ -2246,15 +2289,9 @@ class LiveStatusAppConversationService(AppConversationServiceBase):return deleted_info or deleted_tasks- async def export_conversation(self, conversation_id: UUID) -> bytes:- """Download a conversation trajectory as a zip file.-- Args:- conversation_id: The UUID of the conversation to download.-- Returns the zip file as bytes.- """- # Get the conversation info to verify it exists and user has access+ async def _get_conversation_export_info(+ self, conversation_id: UUID+ ) -> AppConversationInfo:conversation_info = (await self.app_conversation_info_service.get_app_conversation_info(conversation_id@@ -2262,44 +2299,145 @@ class LiveStatusAppConversationService(AppConversationServiceBase):)if not conversation_info:raise ValueError(f'Conversation not found: {conversation_id}')+ return conversation_info++ async def _validate_conversation_export_size(self, conversation_id: UUID):+ if self.export_max_events <= 0:+ return++ event_count = await self.event_service.count_events(conversation_id)+ if event_count > self.export_max_events:+ raise ConversationExportTooLarge(+ f'Conversation export contains {event_count} events, '+ f'exceeding the limit of {self.export_max_events}'+ )++ def _conversation_export_lock_key(self, conversation_id: UUID) -> str:+ return f'{_EXPORT_LOCK_KEY_PREFIX}:{conversation_id.hex}'++ def _conversation_export_lock_refresh_interval(self) -> int:+ ttl_seconds = max(1, self.export_lock_ttl_seconds)+ configured_interval = max(1, self.export_lock_refresh_interval_seconds)+ return min(configured_interval, max(1, ttl_seconds // 2))++ def _conversation_export_lock_required(self) -> bool:+ if self.export_lock_required is not None:+ return self.export_lock_required+ return (self.app_mode or '').lower() == 'saas'++ async def _stream_conversation_zip(+ self, conversation_id: UUID, conversation_info: AppConversationInfo+ ) -> AsyncGenerator[bytes, None]:+ zip_buffer = _StreamingZipBuffer()+ with zipfile.ZipFile(+ cast(BinaryIO, zip_buffer), 'w', zipfile.ZIP_DEFLATED+ ) as zipf:+ zipf.writestr('meta.json', conversation_info.model_dump_json(indent=2))+ for chunk in zip_buffer.drain():+ yield chunk- # Create a temporary directory to store files- with tempfile.TemporaryDirectory() as temp_dir:- # Get all events for this conversationi = 0- async for event in page_iterator(- self.event_service.search_events, conversation_id=conversation_id+ async for event in self.event_service.iter_events_for_export(+ conversation_id):event_filename = f'event_{i:06d}_{event.id}.json'- event_path = os.path.join(temp_dir, event_filename)-- with open(event_path, 'w', encoding='utf-8') as f:- # Use model_dump with mode='json' to handle UUID serialization- event_data = event.model_dump(mode='json')- json.dump(event_data, f, indent=2)+ event_data = event.model_dump(mode='json')+ event_json = json.dumps(event_data, indent=2)+ zipf.writestr(event_filename, event_json)+ for chunk in zip_buffer.drain():+ yield chunki += 1- # Create meta.json with conversation info- meta_path = os.path.join(temp_dir, 'meta.json')- with open(meta_path, 'w', encoding='utf-8') as f:- f.write(conversation_info.model_dump_json(indent=2))+ for chunk in zip_buffer.drain():+ yield chunk++ async def open_conversation_export(+ self, conversation_id: UUID+ ) -> AsyncGenerator[bytes, None]:+ """Prepare a locked streaming conversation trajectory export."""+ conversation_info = await self._get_conversation_export_info(conversation_id)+ lock_key = self._conversation_export_lock_key(conversation_id)+ lock_unavailable = False+ try:+ export_lock = await try_acquire_redis_lock(+ lock_key, max(1, self.export_lock_ttl_seconds)+ )+ except RedisLockUnavailable as e:+ if self._conversation_export_lock_required():+ raise ConversationExportLockUnavailable(+ f'Could not acquire export lock for conversation {conversation_id}'+ ) from e+ _logger.warning(+ 'conversation_export:lock_unavailable_proceeding_without_lock',+ extra={'conversation_id': str(conversation_id)},+ )+ export_lock = None+ lock_unavailable = True++ if export_lock is None and not lock_unavailable:+ raise ConversationExportAlreadyRunning(+ f'Conversation export already running: {conversation_id}'+ )++ try:+ await self._validate_conversation_export_size(conversation_id)+ except Exception:+ if export_lock:+ try:+ await export_lock.release()+ except LockError:+ pass+ raise++ refresh_interval = self._conversation_export_lock_refresh_interval()++ async def stream():+ # Refresh the lock in a background task so the streaming loop stays+ # simple and lock maintenance doesn't block chunk generation.+ refresh_task = (+ asyncio.create_task(+ refresh_lock_periodically(export_lock, refresh_interval)+ )+ if export_lock+ else None+ )+ try:+ async for chunk in self._stream_conversation_zip(+ conversation_id, conversation_info+ ):+ yield chunk+ finally:+ if refresh_task is not None:+ refresh_task.cancel()+ if export_lock:+ try:+ await export_lock.release()+ except LockError:+ _logger.warning(+ 'conversation_export:lock_release_failed',+ extra={'conversation_id': str(conversation_id)},+ )++ return stream()++ async def export_conversation(self, conversation_id: UUID) -> bytes:+ """Download a conversation trajectory as a zip file.++ Args:+ conversation_id: The UUID of the conversation to download.- # Create zip file in memory- zip_buffer = tempfile.NamedTemporaryFile()- with zipfile.ZipFile(zip_buffer, 'w', zipfile.ZIP_DEFLATED) as zipf:- # Add all files from temp directory to zip- for root, dirs, files in os.walk(temp_dir):- for file in files:- file_path = os.path.join(root, file)- arcname = os.path.relpath(file_path, temp_dir)- zipf.write(file_path, arcname)+ Returns the zip file as bytes.+ """+ conversation_info = await self._get_conversation_export_info(conversation_id)+ await self._validate_conversation_export_size(conversation_id)- # Read the zip file content- zip_buffer.seek(0)- zip_content = zip_buffer.read()- zip_buffer.close()+ chunks = []+ async for chunk in self._stream_conversation_zip(+ conversation_id, conversation_info+ ):+ chunks.append(chunk)- return zip_content+ return b''.join(chunks)class LiveStatusAppConversationServiceInjector(AppConversationServiceInjector):@@ -2324,6 +2462,25 @@ class LiveStatusAppConversationServiceInjector(AppConversationServiceInjector):'be retrieved by a sandboxed conversation.'),)+ export_max_events: int = Field(+ default=10000,+ description='The maximum number of events allowed in a conversation export',+ )+ export_lock_ttl_seconds: int = Field(+ default=3600,+ description='Redis lock TTL for a single conversation export',+ )+ export_lock_refresh_interval_seconds: int = Field(+ default=30,+ description='How often to refresh the Redis lock during a conversation export',+ )+ export_lock_required: bool | None = Field(+ default=None,+ description=(+ 'Whether Redis export locking is required. Defaults to required in '+ 'SAAS mode and best-effort elsewhere.'+ ),+ )async def inject(self, state: InjectorState, request: Request | None = None@@ -2408,4 +2565,10 @@ class LiveStatusAppConversationServiceInjector(AppConversationServiceInjector):access_token_hard_timeout=access_token_hard_timeout,conversation_secret_enricher=conversation_secret_enricher,app_mode=app_mode,+ export_max_events=self.export_max_events,+ export_lock_ttl_seconds=self.export_lock_ttl_seconds,+ export_lock_refresh_interval_seconds=(+ self.export_lock_refresh_interval_seconds+ ),+ export_lock_required=self.export_lock_required,)diff --git a/openhands/app_server/event/event_service.py b/openhands/app_server/event/event_service.pyindex fdfd85c216..3e11464874 100644--- a/openhands/app_server/event/event_service.py+++ b/openhands/app_server/event/event_service.py@@ -2,6 +2,7 @@ import asyncioimport loggingfrom abc import ABC, abstractmethodfrom datetime import datetime+from typing import AsyncGeneratorfrom uuid import UUIDfrom openhands.agent_server.models import EventPage, EventSortOrder@@ -9,6 +10,7 @@ from openhands.app_server.event_callback.event_callback_models import EventKindfrom openhands.app_server.services.injector import Injectorfrom openhands.sdk import Eventfrom openhands.sdk.utils.models import DiscriminatedUnionMixin+from openhands.sdk.utils.paging import page_iterator_logger = logging.getLogger(__name__)@@ -43,6 +45,18 @@ class EventService(ABC):) -> int:"""Count events matching the given filters."""+ async def iter_events_for_export(+ self, conversation_id: UUID+ ) -> AsyncGenerator[Event, None]:+ """Iterate all events for a conversation in export order.++ Implementations can override this to avoid paginated searches that reload the+ full event history for each page.+ """+ events = page_iterator(self.search_events, conversation_id=conversation_id)+ async for event in events:+ yield event+@abstractmethodasync def save_event(self, conversation_id: UUID, event: Event):"""Save an event. Internal method intended not be part of the REST api."""diff --git a/openhands/app_server/event/event_service_base.py b/openhands/app_server/event/event_service_base.pyindex 381b4d06f6..fa44293651 100644--- a/openhands/app_server/event/event_service_base.py+++ b/openhands/app_server/event/event_service_base.py@@ -1,8 +1,10 @@import asyncio+import osfrom abc import ABC, abstractmethodfrom dataclasses import dataclassfrom datetime import datetimefrom pathlib import Path+from typing import AsyncGeneratorfrom uuid import UUIDfrom openhands.agent_server.models import EventPage, EventSortOrder@@ -19,6 +21,13 @@ from openhands.sdk import Eventfrom openhands.sdk.utils.paging import page_iterator+def _event_load_concurrency() -> int:+ try:+ return max(1, int(os.getenv('EVENT_SERVICE_LOAD_EVENT_CONCURRENCY', '10')))+ except ValueError:+ return 10++@dataclassclass EventServiceBase(EventService, ABC):"""Event Service for getting events - the only check on permissions for events is@@ -44,6 +53,16 @@ class EventServiceBase(EventService, ABC):def _search_paths(self, prefix: Path) -> list[Path]:"""Search paths."""+ async def _load_events_from_paths(self, paths: list[Path]) -> list[Event | None]:+ loop = asyncio.get_running_loop()+ semaphore = asyncio.Semaphore(_event_load_concurrency())++ async def load_event(path: Path) -> Event | None:+ async with semaphore:+ return await loop.run_in_executor(None, self._load_event, path)++ return await asyncio.gather(*(load_event(path) for path in paths))+async def get_conversation_path(self, conversation_id: UUID) -> Path:"""Get a path for a conversation. Ensure user_id is included if possible."""path = self.prefix@@ -87,10 +106,7 @@ class EventServiceBase(EventService, ABC):prefix = await self.get_conversation_path(conversation_id)paths = await loop.run_in_executor(None, self._search_paths, prefix)- # Type error: run_in_executor expects a return value, but self._load_event is typed return Event | None.- events = await asyncio.gather(- *[loop.run_in_executor(None, self._load_event, path) for path in paths] # type: ignore[arg-type]- )+ events = await self._load_events_from_paths(paths)# Convert datetime filters to ISO strings so they can be compared# against event.timestamp (which is stored as an ISO 8601 string).timestamp_gte_str = timestamp__gte.isoformat() if timestamp__gte else None@@ -126,6 +142,19 @@ class EventServiceBase(EventService, ABC):return EventPage(items=items, next_page_id=next_page_id)+ async def iter_events_for_export(+ self, conversation_id: UUID+ ) -> AsyncGenerator[Event, None]:+ """Iterate all events once in timestamp order for trajectory export."""+ loop = asyncio.get_running_loop()+ prefix = await self.get_conversation_path(conversation_id)+ paths = await loop.run_in_executor(None, self._search_paths, prefix)+ events = await self._load_events_from_paths(paths)+ items = [event for event in events if event]+ items.sort(key=lambda event: event.timestamp)+ for event in items:+ yield event+async def count_events(self,conversation_id: UUID,diff --git a/openhands/app_server/utils/redis.py b/openhands/app_server/utils/redis.pynew file mode 100644index 0000000000..72d0acfe7c--- /dev/null+++ b/openhands/app_server/utils/redis.py@@ -0,0 +1,81 @@+import os+import threading++from redis import Redis+from redis import asyncio as aioredis+from redis import exceptions as redis_exceptions++# Redis configuration+REDIS_HOST = os.environ.get('REDIS_HOST', 'localhost')+REDIS_PORT = int(os.environ.get('REDIS_PORT', '6379'))+REDIS_PASSWORD = os.environ.get('REDIS_PASSWORD', '')+REDIS_DB = int(os.environ.get('REDIS_DB', '0'))+REDIS_SOCKET_TIMEOUT = 2++_redis_client: Redis | None = None+_redis_client_async: aioredis.Redis | None = None+_redis_lock = threading.Lock()+++def _get_redis_kwargs():+ """Return common kwargs for Redis client creation."""+ return {+ 'host': REDIS_HOST,+ 'port': REDIS_PORT,+ 'password': REDIS_PASSWORD,+ 'db': REDIS_DB,+ 'socket_timeout': REDIS_SOCKET_TIMEOUT,+ }+++def get_redis_client() -> Redis:+ """Get a shared synchronous Redis client, lazily initialized.++ Thread-safe with double-checked locking pattern.++ Returns:+ A Redis client for synchronous operations.+ """+ global _redis_client+ if _redis_client is None:+ with _redis_lock:+ if _redis_client is None:+ _redis_client = Redis(**_get_redis_kwargs())+ return _redis_client+++def get_redis_client_async() -> aioredis.Redis:+ """Get a shared asynchronous Redis client, lazily initialized.++ Note: This function is synchronous but returns an async client.+ Thread-safe initialization is handled via a threading lock since+ asyncio.Lock cannot be used in a sync context.++ Returns:+ An aioredis client for asynchronous operations.+ """+ global _redis_client_async+ if _redis_client_async is None:+ with _redis_lock:+ if _redis_client_async is None:+ _redis_client_async = aioredis.Redis(**_get_redis_kwargs())+ return _redis_client_async+++def get_redis_authed_url():+ return f'redis://:{REDIS_PASSWORD}@{REDIS_HOST}:{REDIS_PORT}/{REDIS_DB}'+++__all__ = [+ 'REDIS_DB',+ 'REDIS_HOST',+ 'REDIS_PASSWORD',+ 'REDIS_PORT',+ 'REDIS_SOCKET_TIMEOUT',+ 'Redis',+ 'aioredis',+ 'get_redis_client',+ 'get_redis_client_async',+ 'get_redis_authed_url',+ 'redis_exceptions',+]diff --git a/openhands/app_server/utils/redis_lock.py b/openhands/app_server/utils/redis_lock.pynew file mode 100644index 0000000000..11e2a07e42--- /dev/null+++ b/openhands/app_server/utils/redis_lock.py@@ -0,0 +1,53 @@+import asyncio+import logging++from redis.asyncio.lock import Lock+from redis.exceptions import LockError++from openhands.app_server.utils.redis import get_redis_client_async, redis_exceptions++_logger = logging.getLogger(__name__)++# Re-export so callers can catch lock errors without importing redis directly.+__all__ = [+ 'Lock',+ 'LockError',+ 'RedisLockUnavailable',+ 'try_acquire_redis_lock',+ 'refresh_lock_periodically',+]+++class RedisLockUnavailable(Exception):+ """Raised when Redis cannot be used to evaluate a lock."""+++async def try_acquire_redis_lock(key: str, ttl_seconds: int) -> Lock | None:+ """Try to acquire a Redis lock; return None if already held by another caller."""+ redis = get_redis_client_async()+ lock = redis.lock(key, timeout=ttl_seconds)+ try:+ acquired = await lock.acquire(blocking=False)+ except redis_exceptions.RedisError as e:+ raise RedisLockUnavailable from e+ return lock if acquired else None+++async def refresh_lock_periodically(lock: Lock, interval: int) -> None:+ """Keep a Redis lock alive by refreshing its TTL every *interval* seconds.++ Intended to run as a background task (via ``asyncio.create_task``) alongside+ a long-running operation. Cancel the task when the operation finishes; the+ caller is responsible for releasing the lock afterwards.+ """+ try:+ while True:+ await asyncio.sleep(interval)+ try:+ await lock.reacquire()+ except LockError:+ _logger.warning(+ 'redis_lock:periodic_refresh_failed', extra={'key': lock.name}+ )+ except asyncio.CancelledError:+ passdiff --git a/tests/unit/app_server/test_filesystem_event_service.py b/tests/unit/app_server/test_filesystem_event_service.pyindex 606e9afcca..85b72a776e 100644--- a/tests/unit/app_server/test_filesystem_event_service.py+++ b/tests/unit/app_server/test_filesystem_event_service.py@@ -144,6 +144,29 @@ class TestFilesystemEventServiceSearchEvents:timestamps = [item.timestamp for item in result.items]assert timestamps == sorted(timestamps, reverse=True)+ @pytest.mark.asyncio+ async def test_iter_events_for_export_returns_all_events_in_timestamp_order(+ self, service: FilesystemEventService+ ):+ """Test export iterator returns all events once in timestamp order."""+ conversation_id = uuid4()+ events = []++ for _ in range(3):+ event = create_token_event()+ events.append(event)+ await service.save_event(conversation_id, event)+ time.sleep(0.01)++ result = [+ event async for event in service.iter_events_for_export(conversation_id)+ ]++ assert [event.id for event in result] == [event.id for event in events]+ assert [event.timestamp for event in result] == sorted(+ event.timestamp for event in result+ )+@pytest.mark.asyncioasync def test_search_events_returns_event_page(self, service: FilesystemEventServicediff --git a/tests/unit/app_server/test_live_status_app_conversation_service.py b/tests/unit/app_server/test_live_status_app_conversation_service.pyindex 884d909b6c..bf22e1e6a2 100644--- a/tests/unit/app_server/test_live_status_app_conversation_service.py+++ b/tests/unit/app_server/test_live_status_app_conversation_service.py@@ -1,6 +1,5 @@"""Unit tests for the methods in LiveStatusAppConversationService."""-import builtinsimport ioimport jsonimport os@@ -24,6 +23,11 @@ from openhands.app_server.app_conversation.app_conversation_models import (AppConversationStartRequest,ConversationTrigger,)+from openhands.app_server.app_conversation.app_conversation_service import (+ ConversationExportAlreadyRunning,+ ConversationExportLockUnavailable,+ ConversationExportTooLarge,+)from openhands.app_server.app_conversation.live_status_app_conversation_service import (LiveStatusAppConversationService,)@@ -43,6 +47,7 @@ from openhands.app_server.settings.settings_models import (Settings,)from openhands.app_server.user.user_context import UserContext+from openhands.app_server.utils.redis_lock import RedisLockUnavailablefrom openhands.sdk import Agent, AgentContext, Eventfrom openhands.sdk.llm import LLMfrom openhands.sdk.secret import LookupSecret, StaticSecret@@ -50,6 +55,27 @@ from openhands.sdk.settings import ConversationSettings, OpenHandsAgentSettingsfrom openhands.sdk.workspace.remote.async_remote_workspace import AsyncRemoteWorkspace+def _async_iter(items):+ async def iter_items():+ for item in items:+ yield item++ return iter_items()+++class _FakeExportLock:+ def __init__(self):+ self.name = 'fake-lock'+ self.released = False+ self.reacquired = False++ async def reacquire(self):+ self.reacquired = True++ async def release(self):+ self.released = True++def _build_test_user_agent_settings(user: SimpleNamespace) -> OpenHandsAgentSettings:llm_vals: dict = {}model = getattr(user, 'llm_model', '') or ''@@ -1378,10 +1404,7 @@ class TestLiveStatusAppConversationService:async def test_export_conversation_success(self):"""Test successful download of conversation trajectory."""- # Arrangeconversation_id = uuid4()-- # Mock conversation infomock_conversation_info = Mock(spec=AppConversationInfo)mock_conversation_info.id = conversation_idmock_conversation_info.title = 'Test Conversation'@@ -1398,7 +1421,6 @@ class TestLiveStatusAppConversationService:return_value=mock_conversation_info)- # Mock eventsmock_event1 = Mock(spec=Event)mock_event1.id = uuid4()mock_event1.model_dump = Mock(@@ -1411,64 +1433,52 @@ class TestLiveStatusAppConversationService:return_value={'id': str(mock_event2.id), 'type': 'observation'})- # Mock event service search_events to return paginated results- mock_event_page1 = Mock()- mock_event_page1.items = [mock_event1]- mock_event_page1.next_page_id = 'page2'-- mock_event_page2 = Mock()- mock_event_page2.items = [mock_event2]- mock_event_page2.next_page_id = None-- self.mock_event_service.search_events = AsyncMock(- side_effect=[mock_event_page1, mock_event_page2]+ self.mock_event_service.count_events = AsyncMock(return_value=2)+ self.mock_event_service.iter_events_for_export = Mock(+ return_value=_async_iter([mock_event1, mock_event2]))+ self.mock_event_service.search_events = AsyncMock()- # Actresult = await self.service.export_conversation(conversation_id)- # Assertassert result is not None- assert isinstance(result, bytes) # Should be bytes+ assert isinstance(result, bytes)- # Verify the zip file contentswith zipfile.ZipFile(io.BytesIO(result), 'r') as zipf:file_list = zipf.namelist()- # Should contain meta.json and event filesassert 'meta.json' in file_listassert any(f.startswith('event_') and f.endswith('.json') for f in file_list)- # Check meta.json contentwith zipf.open('meta.json') as meta_file:meta_content = meta_file.read().decode('utf-8')assert '"id": "test"' in meta_contentassert '"title": "Test Conversation"' in meta_content- # Check event filesevent_files = [f for f in file_list if f.startswith('event_')]- assert len(event_files) == 2 # Should have 2 event files+ assert len(event_files) == 2- # Verify event file contentwith zipf.open(event_files[0]) as event_file:event_content = json.loads(event_file.read().decode('utf-8'))assert 'id' in event_contentassert 'type' in event_content- # Verify service callsself.mock_app_conversation_info_service.get_app_conversation_info.assert_called_once_with(conversation_id)- assert self.mock_event_service.search_events.call_count == 2+ self.mock_event_service.count_events.assert_awaited_once_with(conversation_id)+ self.mock_event_service.iter_events_for_export.assert_called_once_with(+ conversation_id+ )+ self.mock_event_service.search_events.assert_not_called()mock_conversation_info.model_dump_json.assert_called_once_with(indent=2)async def test_export_conversation_writes_utf8_when_platform_default_cannot_encode(self,):"""Test trajectory export writes JSON as UTF-8 regardless of platform default."""- # Arrangeconversation_id = uuid4()mock_conversation_info = Mock(spec=AppConversationInfo)@@ -1492,27 +1502,13 @@ class TestLiveStatusAppConversationService:})- mock_event_page = Mock()- mock_event_page.items = [mock_event]- mock_event_page.next_page_id = None- self.mock_event_service.search_events = AsyncMock(return_value=mock_event_page)-- real_open = builtins.open-- def cp1252_default_open(file, mode='r', *args, **kwargs):- if 'w' in mode and 'encoding' not in kwargs:- kwargs['encoding'] = 'cp1252'- return real_open(file, mode, *args, **kwargs)+ self.mock_event_service.count_events = AsyncMock(return_value=1)+ self.mock_event_service.iter_events_for_export = Mock(+ return_value=_async_iter([mock_event])+ )- # Act- with patch(- 'openhands.app_server.app_conversation.live_status_app_conversation_service.open',- cp1252_default_open,- create=True,- ):- result = await self.service.export_conversation(conversation_id)+ result = await self.service.export_conversation(conversation_id)- # Assertwith zipfile.ZipFile(io.BytesIO(result), 'r') as zipf:with zipf.open('meta.json') as meta_file:meta_content = meta_file.read().decode('utf-8')@@ -1529,31 +1525,29 @@ class TestLiveStatusAppConversationService:@pytest.mark.asyncioasync def test_export_conversation_conversation_not_found(self):"""Test download when conversation is not found."""- # Arrangeconversation_id = uuid4()self.mock_app_conversation_info_service.get_app_conversation_info = AsyncMock(return_value=None)+ self.mock_event_service.count_events = AsyncMock()+ self.mock_event_service.iter_events_for_export = Mock()- # Act & Assertwith pytest.raises(ValueError, match=f'Conversation not found: {conversation_id}'):await self.service.export_conversation(conversation_id)- # Verify service callsself.mock_app_conversation_info_service.get_app_conversation_info.assert_called_once_with(conversation_id)- self.mock_event_service.search_events.assert_not_called()+ self.mock_event_service.count_events.assert_not_called()+ self.mock_event_service.iter_events_for_export.assert_not_called()@pytest.mark.asyncioasync def test_export_conversation_empty_events(self):"""Test download with conversation that has no events."""- # Arrangeconversation_id = uuid4()- # Mock conversation infomock_conversation_info = Mock(spec=AppConversationInfo)mock_conversation_info.id = conversation_idmock_conversation_info.title = 'Empty Conversation'@@ -1565,48 +1559,36 @@ class TestLiveStatusAppConversationService:return_value=mock_conversation_info)- # Mock empty event page- mock_event_page = Mock()- mock_event_page.items = []- mock_event_page.next_page_id = None-- self.mock_event_service.search_events = AsyncMock(return_value=mock_event_page)+ self.mock_event_service.count_events = AsyncMock(return_value=0)+ self.mock_event_service.iter_events_for_export = Mock(+ return_value=_async_iter([])+ )- # Actresult = await self.service.export_conversation(conversation_id)- # Assertassert result is not None- assert isinstance(result, bytes) # Should be bytes+ assert isinstance(result, bytes)- # Verify the zip file contentswith zipfile.ZipFile(io.BytesIO(result), 'r') as zipf:file_list = zipf.namelist()- # Should only contain meta.json (no event files)assert 'meta.json' in file_listassert len([f for f in file_list if f.startswith('event_')]) == 0- # Verify service callsself.mock_app_conversation_info_service.get_app_conversation_info.assert_called_once_with(conversation_id)- self.mock_event_service.search_events.assert_called_once()+ self.mock_event_service.count_events.assert_awaited_once_with(conversation_id)+ self.mock_event_service.iter_events_for_export.assert_called_once_with(+ conversation_id+ )@pytest.mark.asyncio- async def test_export_conversation_calls_search_events_with_correct_parameter_name(+ async def test_export_conversation_calls_iterator_with_conversation_id(self,):- """Test that export_conversation calls search_events with 'conversation_id' parameter, not 'conversation_id__eq'.-- This test verifies the fix for a bug where page_iterator was called with- conversation_id__eq instead of conversation_id, causing a TypeError since- the search_events method expects conversation_id as its parameter name.- """- # Arrange+ """Test that export_conversation calls the export iterator with conversation_id."""conversation_id = uuid4()-- # Mock conversation infomock_conversation_info = Mock(spec=AppConversationInfo)mock_conversation_info.id = conversation_idmock_conversation_info.model_dump_json = Mock(return_value='{}')@@ -1615,35 +1597,23 @@ class TestLiveStatusAppConversationService:return_value=mock_conversation_info)- # Mock empty event page to simplify test- mock_event_page = Mock()- mock_event_page.items = []- mock_event_page.next_page_id = None-- self.mock_event_service.search_events = AsyncMock(return_value=mock_event_page)+ self.mock_event_service.count_events = AsyncMock(return_value=0)+ self.mock_event_service.iter_events_for_export = Mock(+ return_value=_async_iter([])+ )+ self.mock_event_service.search_events = AsyncMock()- # Actawait self.service.export_conversation(conversation_id)- # Assert - Verify search_events was called with 'conversation_id', not 'conversation_id__eq'- self.mock_event_service.search_events.assert_called()- call_kwargs = self.mock_event_service.search_events.call_args[1]-- assert 'conversation_id' in call_kwargs, (- "search_events should be called with 'conversation_id' parameter"- )- assert 'conversation_id__eq' not in call_kwargs, (- "search_events should NOT be called with 'conversation_id__eq' parameter"+ self.mock_event_service.iter_events_for_export.assert_called_once_with(+ conversation_id)- assert call_kwargs['conversation_id'] == conversation_id+ self.mock_event_service.search_events.assert_not_called()@pytest.mark.asyncio- async def test_export_conversation_large_pagination(self):- """Test download with multiple pages of events."""- # Arrange+ async def test_export_conversation_large_iterator(self):+ """Test download with many events from one export iterator pass."""conversation_id = uuid4()-- # Mock conversation infomock_conversation_info = Mock(spec=AppConversationInfo)mock_conversation_info.id = conversation_idmock_conversation_info.title = 'Large Conversation'@@ -1655,64 +1625,171 @@ class TestLiveStatusAppConversationService:return_value=mock_conversation_info)- # Create multiple pages of events- events_per_page = 3- total_pages = 4+ total_events = 12all_events = []- for page_num in range(total_pages):- page_events = []- for i in range(events_per_page):- mock_event = Mock(spec=Event)- mock_event.id = uuid4()- mock_event.model_dump = Mock(- return_value={- 'id': str(mock_event.id),- 'type': f'event_page_{page_num}_item_{i}',- }- )- page_events.append(mock_event)- all_events.append(mock_event)-- mock_event_page = Mock()- mock_event_page.items = page_events- mock_event_page.next_page_id = (- f'page{page_num + 1}' if page_num < total_pages - 1 else None+ for i in range(total_events):+ mock_event = Mock(spec=Event)+ mock_event.id = uuid4()+ mock_event.model_dump = Mock(+ return_value={+ 'id': str(mock_event.id),+ 'type': f'event_{i}',+ })+ all_events.append(mock_event)- if page_num == 0:- first_page = mock_event_page- elif page_num == 1:- second_page = mock_event_page- elif page_num == 2:- third_page = mock_event_page- else:- fourth_page = mock_event_page-- self.mock_event_service.search_events = AsyncMock(- side_effect=[first_page, second_page, third_page, fourth_page]+ self.mock_event_service.count_events = AsyncMock(return_value=total_events)+ self.mock_event_service.iter_events_for_export = Mock(+ return_value=_async_iter(all_events))+ self.mock_event_service.search_events = AsyncMock()- # Actresult = await self.service.export_conversation(conversation_id)- # Assertassert result is not None- assert isinstance(result, bytes) # Should be bytes+ assert isinstance(result, bytes)- # Verify the zip file contentswith zipfile.ZipFile(io.BytesIO(result), 'r') as zipf:file_list = zipf.namelist()- # Should contain meta.json and all event filesassert 'meta.json' in file_listevent_files = [f for f in file_list if f.startswith('event_')]- assert (- len(event_files) == total_pages * events_per_page- ) # Should have all events+ assert len(event_files) == total_events++ self.mock_event_service.iter_events_for_export.assert_called_once_with(+ conversation_id+ )+ self.mock_event_service.search_events.assert_not_called()++ @pytest.mark.asyncio+ async def test_open_conversation_export_streams_with_lock(self):+ """Test streaming export acquires and releases the conversation export lock."""+ conversation_id = uuid4()+ mock_conversation_info = Mock(spec=AppConversationInfo)+ mock_conversation_info.id = conversation_id+ mock_conversation_info.model_dump_json = Mock(return_value='{}')++ mock_event = Mock(spec=Event)+ mock_event.id = uuid4()+ mock_event.model_dump = Mock(return_value={'id': str(mock_event.id)})++ self.mock_app_conversation_info_service.get_app_conversation_info = AsyncMock(+ return_value=mock_conversation_info+ )+ self.mock_event_service.count_events = AsyncMock(return_value=1)+ self.mock_event_service.iter_events_for_export = Mock(+ return_value=_async_iter([mock_event])+ )++ fake_lock = _FakeExportLock()+ with patch(+ 'openhands.app_server.app_conversation.live_status_app_conversation_service.try_acquire_redis_lock',+ new=AsyncMock(return_value=fake_lock),+ ) as mock_acquire_lock:+ stream = await self.service.open_conversation_export(conversation_id)+ result = b''.join([chunk async for chunk in stream])++ assert result+ assert fake_lock.released is True+ mock_acquire_lock.assert_awaited_once_with(+ f'app_conversation_export:{conversation_id.hex}',+ self.service.export_lock_ttl_seconds,+ )++ @pytest.mark.asyncio+ async def test_open_conversation_export_rejects_duplicate_export(self):+ """Test streaming export rejects duplicate export attempts."""+ conversation_id = uuid4()+ mock_conversation_info = Mock(spec=AppConversationInfo)+ mock_conversation_info.id = conversation_id+ mock_conversation_info.model_dump_json = Mock(return_value='{}')++ self.mock_app_conversation_info_service.get_app_conversation_info = AsyncMock(+ return_value=mock_conversation_info+ )++ with patch(+ 'openhands.app_server.app_conversation.live_status_app_conversation_service.try_acquire_redis_lock',+ new=AsyncMock(return_value=None),+ ):+ with pytest.raises(ConversationExportAlreadyRunning):+ await self.service.open_conversation_export(conversation_id)++ self.mock_event_service.count_events.assert_not_called()++ @pytest.mark.asyncio+ async def test_open_conversation_export_fails_closed_when_lock_unavailable(self):+ """Test streaming export fails closed when Redis lock cannot be checked."""+ conversation_id = uuid4()+ mock_conversation_info = Mock(spec=AppConversationInfo)+ mock_conversation_info.id = conversation_id+ mock_conversation_info.model_dump_json = Mock(return_value='{}')++ self.service.export_lock_required = True+ self.mock_app_conversation_info_service.get_app_conversation_info = AsyncMock(+ return_value=mock_conversation_info+ )++ with patch(+ 'openhands.app_server.app_conversation.live_status_app_conversation_service.try_acquire_redis_lock',+ new=AsyncMock(side_effect=RedisLockUnavailable()),+ ):+ with pytest.raises(ConversationExportLockUnavailable):+ await self.service.open_conversation_export(conversation_id)++ self.mock_event_service.count_events.assert_not_called()++ @pytest.mark.asyncio+ async def test_open_conversation_export_proceeds_when_lock_not_required(self):+ """Test non-SaaS export proceeds without Redis when locking is best effort."""+ conversation_id = uuid4()+ mock_conversation_info = Mock(spec=AppConversationInfo)+ mock_conversation_info.id = conversation_id+ mock_conversation_info.model_dump_json = Mock(return_value='{}')++ mock_event = Mock(spec=Event)+ mock_event.id = uuid4()+ mock_event.model_dump = Mock(return_value={'id': str(mock_event.id)})++ self.service.export_lock_required = False+ self.mock_app_conversation_info_service.get_app_conversation_info = AsyncMock(+ return_value=mock_conversation_info+ )+ self.mock_event_service.count_events = AsyncMock(return_value=1)+ self.mock_event_service.iter_events_for_export = Mock(+ return_value=_async_iter([mock_event])+ )++ with patch(+ 'openhands.app_server.app_conversation.live_status_app_conversation_service.try_acquire_redis_lock',+ new=AsyncMock(side_effect=RedisLockUnavailable()),+ ):+ stream = await self.service.open_conversation_export(conversation_id)+ result = b''.join([chunk async for chunk in stream])++ assert result+ self.mock_event_service.count_events.assert_awaited_once_with(conversation_id)++ @pytest.mark.asyncio+ async def test_export_conversation_rejects_too_many_events(self):+ """Test export rejects conversations over the configured event limit."""+ conversation_id = uuid4()+ mock_conversation_info = Mock(spec=AppConversationInfo)+ mock_conversation_info.id = conversation_id+ mock_conversation_info.model_dump_json = Mock(return_value='{}')++ self.service.export_max_events = 1+ self.mock_app_conversation_info_service.get_app_conversation_info = AsyncMock(+ return_value=mock_conversation_info+ )+ self.mock_event_service.count_events = AsyncMock(return_value=2)+ self.mock_event_service.iter_events_for_export = Mock()++ with pytest.raises(ConversationExportTooLarge):+ await self.service.export_conversation(conversation_id)- # Verify service calls - should call search_events for each page- assert self.mock_event_service.search_events.call_count == total_pages+ self.mock_event_service.iter_events_for_export.assert_not_called()@patch('openhands.app_server.app_conversation.live_status_app_conversation_service.AsyncRemoteWorkspace'