Skip to content

πŸ™Œ OpenHands: AI-Driven Development

fix: prevent conversation export overload (#14899)

Graham Neubig9c299e62708f
diff --git a/enterprise/storage/redis.py b/enterprise/storage/redis.py
index cc1869effc..38c703556d 100644
--- a/enterprise/storage/redis.py
+++ b/enterprise/storage/redis.py
@@ -1,72 +1,24 @@
-import os
-import threading
-
from redis import Redis
from redis import asyncio as aioredis
from redis import exceptions as redis_exceptions
-# Redis configuration
-REDIS_HOST = os.environ.get('REDIS_HOST', 'localhost')
-REDIS_PORT = int(os.environ.get('REDIS_PORT', '6379'))
-REDIS_PASSWORD = os.environ.get('REDIS_PASSWORD', '')
-REDIS_DB = int(os.environ.get('REDIS_DB', '0'))
-REDIS_SOCKET_TIMEOUT = 2
-
-_redis_client: Redis | None = None
-_redis_client_async: aioredis.Redis | None = None
-_redis_lock = threading.Lock()
-
-
-def _get_redis_kwargs():
- """Return common kwargs for Redis client creation."""
- return {
- 'host': REDIS_HOST,
- 'port': REDIS_PORT,
- 'password': REDIS_PASSWORD,
- 'db': REDIS_DB,
- 'socket_timeout': REDIS_SOCKET_TIMEOUT,
- }
-
-
-def get_redis_client() -> Redis:
- """Get a shared synchronous Redis client, lazily initialized.
-
- Thread-safe with double-checked locking pattern.
-
- Returns:
- A Redis client for synchronous operations.
- """
- global _redis_client
- if _redis_client is None:
- with _redis_lock:
- if _redis_client is None:
- _redis_client = Redis(**_get_redis_kwargs())
- return _redis_client
-
-
-def get_redis_client_async() -> aioredis.Redis:
- """Get a shared asynchronous Redis client, lazily initialized.
-
- Note: This function is synchronous but returns an async client.
- Thread-safe initialization is handled via a threading lock since
- asyncio.Lock cannot be used in a sync context.
-
- Returns:
- An aioredis client for asynchronous operations.
- """
- global _redis_client_async
- if _redis_client_async is None:
- with _redis_lock:
- if _redis_client_async is None:
- _redis_client_async = aioredis.Redis(**_get_redis_kwargs())
- return _redis_client_async
-
-
-def get_redis_authed_url():
- return f'redis://:{REDIS_PASSWORD}@{REDIS_HOST}:{REDIS_PORT}/{REDIS_DB}'
-
+from openhands.app_server.utils.redis import (
+ REDIS_DB,
+ REDIS_HOST,
+ REDIS_PASSWORD,
+ REDIS_PORT,
+ REDIS_SOCKET_TIMEOUT,
+ get_redis_authed_url,
+ get_redis_client,
+ get_redis_client_async,
+)
__all__ = [
+ 'REDIS_DB',
+ 'REDIS_HOST',
+ 'REDIS_PASSWORD',
+ 'REDIS_PORT',
+ 'REDIS_SOCKET_TIMEOUT',
'Redis',
'aioredis',
'get_redis_client',
diff --git a/openhands/app_server/app_conversation/app_conversation_router.py b/openhands/app_server/app_conversation/app_conversation_router.py
index d4f7dfbf07..83e26e7fca 100644
--- a/openhands/app_server/app_conversation/app_conversation_router.py
+++ b/openhands/app_server/app_conversation/app_conversation_router.py
@@ -13,7 +13,7 @@ from typing import Annotated, Any, AsyncGenerator, Literal
from uuid import UUID
import httpx
-from fastapi import APIRouter, Depends, HTTPException, Query, Request, Response, status
+from fastapi import APIRouter, Depends, HTTPException, Query, Request, status
from fastapi.responses import JSONResponse, StreamingResponse
from sqlalchemy.ext.asyncio import AsyncSession
@@ -43,6 +43,9 @@ from openhands.app_server.app_conversation.app_conversation_models import (
)
from openhands.app_server.app_conversation.app_conversation_service import (
AppConversationService,
+ ConversationExportAlreadyRunning,
+ ConversationExportLockUnavailable,
+ ConversationExportTooLarge,
)
from openhands.app_server.app_conversation.app_conversation_service_base import (
AppConversationServiceBase,
@@ -1563,8 +1566,9 @@ async def export_conversation(
A zip file containing the conversation trajectory
"""
try:
- # Get the zip file content
- zip_content = await app_conversation_service.export_conversation(
+ # Prepare the zip stream before sending headers so lock and validation
+ # errors can still be returned as HTTP status codes.
+ zip_stream = await app_conversation_service.open_conversation_export(
conversation_id
)
@@ -1591,9 +1595,8 @@ async def export_conversation(
except Exception:
logger.exception('analytics:trajectory_downloaded:failed')
- # Return as a downloadable zip file
- return Response(
- content=zip_content,
+ return StreamingResponse(
+ zip_stream,
media_type='application/zip',
headers={
'Content-Disposition': f'attachment; filename="conversation_{conversation_id}.zip"'
@@ -1601,6 +1604,12 @@ async def export_conversation(
)
except ValueError as e:
raise HTTPException(status_code=404, detail=str(e))
+ except ConversationExportAlreadyRunning as e:
+ raise HTTPException(status_code=409, detail=str(e))
+ except ConversationExportLockUnavailable as e:
+ raise HTTPException(status_code=503, detail=str(e))
+ except ConversationExportTooLarge as e:
+ raise HTTPException(status_code=413, detail=str(e))
except Exception as e:
raise HTTPException(
status_code=500, detail=f'Failed to download trajectory: {str(e)}'
diff --git a/openhands/app_server/app_conversation/app_conversation_service.py b/openhands/app_server/app_conversation/app_conversation_service.py
index 6be1d32ddf..b2aa155eb7 100644
--- a/openhands/app_server/app_conversation/app_conversation_service.py
+++ b/openhands/app_server/app_conversation/app_conversation_service.py
@@ -18,6 +18,18 @@ from openhands.sdk.utils.models import DiscriminatedUnionMixin
from openhands.sdk.workspace.remote.async_remote_workspace import AsyncRemoteWorkspace
+class ConversationExportAlreadyRunning(Exception):
+ """Raised when another worker is already exporting the same conversation."""
+
+
+class ConversationExportLockUnavailable(Exception):
+ """Raised when the export lock cannot be checked."""
+
+
+class ConversationExportTooLarge(Exception):
+ """Raised when a conversation exceeds the configured export event limit."""
+
+
class AppConversationService(ABC):
"""Service for managing conversations running in sandboxes."""
@@ -160,6 +172,21 @@ class AppConversationService(ABC):
Returns the zip file as bytes.
"""
+ async def open_conversation_export(
+ self, conversation_id: UUID
+ ) -> AsyncGenerator[bytes, None]:
+ """Prepare a streaming conversation trajectory export.
+
+ Implementations may override this to acquire locks or stream zip data.
+ The default preserves compatibility with byte-returning exporters.
+ """
+ content = await self.export_conversation(conversation_id)
+
+ async def stream():
+ yield content
+
+ return stream()
+
class AppConversationServiceInjector(
DiscriminatedUnionMixin, Injector[AppConversationService], ABC
diff --git a/openhands/app_server/app_conversation/live_status_app_conversation_service.py b/openhands/app_server/app_conversation/live_status_app_conversation_service.py
index e3473c4769..eccc2b1b8b 100644
--- a/openhands/app_server/app_conversation/live_status_app_conversation_service.py
+++ b/openhands/app_server/app_conversation/live_status_app_conversation_service.py
@@ -1,15 +1,15 @@
import asyncio
import importlib.metadata
+import io
import json
import logging
import os
-import tempfile
import zipfile
from collections import defaultdict
from collections.abc import Mapping
from dataclasses import dataclass, field
from datetime import datetime, timedelta
-from typing import Any, AsyncGenerator, Sequence, cast
+from typing import Any, AsyncGenerator, BinaryIO, Sequence, cast
from uuid import UUID, uuid4
import httpx
@@ -44,6 +44,9 @@ from openhands.app_server.app_conversation.app_conversation_models import (
from openhands.app_server.app_conversation.app_conversation_service import (
AppConversationService,
AppConversationServiceInjector,
+ ConversationExportAlreadyRunning,
+ ConversationExportLockUnavailable,
+ ConversationExportTooLarge,
)
from openhands.app_server.app_conversation.app_conversation_service_base import (
AppConversationServiceBase,
@@ -105,6 +108,12 @@ from openhands.app_server.utils.llm_metadata import (
get_llm_metadata,
should_set_litellm_extra_body,
)
+from openhands.app_server.utils.redis_lock import (
+ LockError,
+ RedisLockUnavailable,
+ refresh_lock_periodically,
+ try_acquire_redis_lock,
+)
from openhands.sdk import Agent, AgentContext, LocalWorkspace
from openhands.sdk.hooks import HookConfig
from openhands.sdk.llm import LLM
@@ -114,7 +123,6 @@ from openhands.sdk.secret import LookupSecret, StaticSecret
from openhands.sdk.settings import ACPAgentSettings
from openhands.sdk.subagent import get_registered_agent_definitions
from openhands.sdk.tool.builtins import SwitchLLMTool
-from openhands.sdk.utils.paging import page_iterator
from openhands.sdk.utils.redact import (
redact_api_key_literals,
redact_text_secrets,
@@ -133,6 +141,37 @@ from openhands.tools.preset.planning import (
_conversation_info_type_adapter = TypeAdapter(list[ConversationInfo | None])
_logger = logging.getLogger(__name__)
+_EXPORT_LOCK_KEY_PREFIX = 'app_conversation_export'
+
+
+class _StreamingZipBuffer(io.RawIOBase):
+ """Small non-seekable writer used by zipfile to emit chunks incrementally.
+
+ zipfile.ZipFile only needs write() and tell() from its underlying file
+ object when writing to a non-seekable stream. Everything else
+ (flush, writable, seekable) is either unused by zipfile or already
+ handled by the io.RawIOBase defaults.
+ """
+
+ def __init__(self):
+ self._chunks: list[bytes] = []
+ self._position = 0
+
+ def write(self, data) -> int:
+ chunk = bytes(data)
+ if chunk:
+ self._chunks.append(chunk)
+ self._position += len(chunk)
+ return len(data)
+
+ def tell(self) -> int:
+ return self._position
+
+ def drain(self) -> list[bytes]:
+ chunks = self._chunks
+ self._chunks = []
+ return chunks
+
def _expected_sdk_version() -> str | None:
"""App's pinned openhands-sdk version, or None if its metadata is unresolvable."""
@@ -181,6 +220,10 @@ class LiveStatusAppConversationService(AppConversationServiceBase):
default_factory=ConversationSecretEnricher
)
app_mode: str | None = None
+ export_max_events: int = 10000
+ export_lock_ttl_seconds: int = 3600
+ export_lock_refresh_interval_seconds: int = 30
+ export_lock_required: bool | None = None
async def _get_sandbox_grouping_strategy(self) -> SandboxGroupingStrategy:
"""Get the sandbox grouping strategy from user settings."""
@@ -2246,15 +2289,9 @@ class LiveStatusAppConversationService(AppConversationServiceBase):
return deleted_info or deleted_tasks
- async def export_conversation(self, conversation_id: UUID) -> bytes:
- """Download a conversation trajectory as a zip file.
-
- Args:
- conversation_id: The UUID of the conversation to download.
-
- Returns the zip file as bytes.
- """
- # Get the conversation info to verify it exists and user has access
+ async def _get_conversation_export_info(
+ self, conversation_id: UUID
+ ) -> AppConversationInfo:
conversation_info = (
await self.app_conversation_info_service.get_app_conversation_info(
conversation_id
@@ -2262,44 +2299,145 @@ class LiveStatusAppConversationService(AppConversationServiceBase):
)
if not conversation_info:
raise ValueError(f'Conversation not found: {conversation_id}')
+ return conversation_info
+
+ async def _validate_conversation_export_size(self, conversation_id: UUID):
+ if self.export_max_events <= 0:
+ return
+
+ event_count = await self.event_service.count_events(conversation_id)
+ if event_count > self.export_max_events:
+ raise ConversationExportTooLarge(
+ f'Conversation export contains {event_count} events, '
+ f'exceeding the limit of {self.export_max_events}'
+ )
+
+ def _conversation_export_lock_key(self, conversation_id: UUID) -> str:
+ return f'{_EXPORT_LOCK_KEY_PREFIX}:{conversation_id.hex}'
+
+ def _conversation_export_lock_refresh_interval(self) -> int:
+ ttl_seconds = max(1, self.export_lock_ttl_seconds)
+ configured_interval = max(1, self.export_lock_refresh_interval_seconds)
+ return min(configured_interval, max(1, ttl_seconds // 2))
+
+ def _conversation_export_lock_required(self) -> bool:
+ if self.export_lock_required is not None:
+ return self.export_lock_required
+ return (self.app_mode or '').lower() == 'saas'
+
+ async def _stream_conversation_zip(
+ self, conversation_id: UUID, conversation_info: AppConversationInfo
+ ) -> AsyncGenerator[bytes, None]:
+ zip_buffer = _StreamingZipBuffer()
+ with zipfile.ZipFile(
+ cast(BinaryIO, zip_buffer), 'w', zipfile.ZIP_DEFLATED
+ ) as zipf:
+ zipf.writestr('meta.json', conversation_info.model_dump_json(indent=2))
+ for chunk in zip_buffer.drain():
+ yield chunk
- # Create a temporary directory to store files
- with tempfile.TemporaryDirectory() as temp_dir:
- # Get all events for this conversation
i = 0
- async for event in page_iterator(
- self.event_service.search_events, conversation_id=conversation_id
+ async for event in self.event_service.iter_events_for_export(
+ conversation_id
):
event_filename = f'event_{i:06d}_{event.id}.json'
- event_path = os.path.join(temp_dir, event_filename)
-
- with open(event_path, 'w', encoding='utf-8') as f:
- # Use model_dump with mode='json' to handle UUID serialization
- event_data = event.model_dump(mode='json')
- json.dump(event_data, f, indent=2)
+ event_data = event.model_dump(mode='json')
+ event_json = json.dumps(event_data, indent=2)
+ zipf.writestr(event_filename, event_json)
+ for chunk in zip_buffer.drain():
+ yield chunk
i += 1
- # Create meta.json with conversation info
- meta_path = os.path.join(temp_dir, 'meta.json')
- with open(meta_path, 'w', encoding='utf-8') as f:
- f.write(conversation_info.model_dump_json(indent=2))
+ for chunk in zip_buffer.drain():
+ yield chunk
+
+ async def open_conversation_export(
+ self, conversation_id: UUID
+ ) -> AsyncGenerator[bytes, None]:
+ """Prepare a locked streaming conversation trajectory export."""
+ conversation_info = await self._get_conversation_export_info(conversation_id)
+ lock_key = self._conversation_export_lock_key(conversation_id)
+ lock_unavailable = False
+ try:
+ export_lock = await try_acquire_redis_lock(
+ lock_key, max(1, self.export_lock_ttl_seconds)
+ )
+ except RedisLockUnavailable as e:
+ if self._conversation_export_lock_required():
+ raise ConversationExportLockUnavailable(
+ f'Could not acquire export lock for conversation {conversation_id}'
+ ) from e
+ _logger.warning(
+ 'conversation_export:lock_unavailable_proceeding_without_lock',
+ extra={'conversation_id': str(conversation_id)},
+ )
+ export_lock = None
+ lock_unavailable = True
+
+ if export_lock is None and not lock_unavailable:
+ raise ConversationExportAlreadyRunning(
+ f'Conversation export already running: {conversation_id}'
+ )
+
+ try:
+ await self._validate_conversation_export_size(conversation_id)
+ except Exception:
+ if export_lock:
+ try:
+ await export_lock.release()
+ except LockError:
+ pass
+ raise
+
+ refresh_interval = self._conversation_export_lock_refresh_interval()
+
+ async def stream():
+ # Refresh the lock in a background task so the streaming loop stays
+ # simple and lock maintenance doesn't block chunk generation.
+ refresh_task = (
+ asyncio.create_task(
+ refresh_lock_periodically(export_lock, refresh_interval)
+ )
+ if export_lock
+ else None
+ )
+ try:
+ async for chunk in self._stream_conversation_zip(
+ conversation_id, conversation_info
+ ):
+ yield chunk
+ finally:
+ if refresh_task is not None:
+ refresh_task.cancel()
+ if export_lock:
+ try:
+ await export_lock.release()
+ except LockError:
+ _logger.warning(
+ 'conversation_export:lock_release_failed',
+ extra={'conversation_id': str(conversation_id)},
+ )
+
+ return stream()
+
+ async def export_conversation(self, conversation_id: UUID) -> bytes:
+ """Download a conversation trajectory as a zip file.
+
+ Args:
+ conversation_id: The UUID of the conversation to download.
- # Create zip file in memory
- zip_buffer = tempfile.NamedTemporaryFile()
- with zipfile.ZipFile(zip_buffer, 'w', zipfile.ZIP_DEFLATED) as zipf:
- # Add all files from temp directory to zip
- for root, dirs, files in os.walk(temp_dir):
- for file in files:
- file_path = os.path.join(root, file)
- arcname = os.path.relpath(file_path, temp_dir)
- zipf.write(file_path, arcname)
+ Returns the zip file as bytes.
+ """
+ conversation_info = await self._get_conversation_export_info(conversation_id)
+ await self._validate_conversation_export_size(conversation_id)
- # Read the zip file content
- zip_buffer.seek(0)
- zip_content = zip_buffer.read()
- zip_buffer.close()
+ chunks = []
+ async for chunk in self._stream_conversation_zip(
+ conversation_id, conversation_info
+ ):
+ chunks.append(chunk)
- return zip_content
+ return b''.join(chunks)
class LiveStatusAppConversationServiceInjector(AppConversationServiceInjector):
@@ -2324,6 +2462,25 @@ class LiveStatusAppConversationServiceInjector(AppConversationServiceInjector):
'be retrieved by a sandboxed conversation.'
),
)
+ export_max_events: int = Field(
+ default=10000,
+ description='The maximum number of events allowed in a conversation export',
+ )
+ export_lock_ttl_seconds: int = Field(
+ default=3600,
+ description='Redis lock TTL for a single conversation export',
+ )
+ export_lock_refresh_interval_seconds: int = Field(
+ default=30,
+ description='How often to refresh the Redis lock during a conversation export',
+ )
+ export_lock_required: bool | None = Field(
+ default=None,
+ description=(
+ 'Whether Redis export locking is required. Defaults to required in '
+ 'SAAS mode and best-effort elsewhere.'
+ ),
+ )
async def inject(
self, state: InjectorState, request: Request | None = None
@@ -2408,4 +2565,10 @@ class LiveStatusAppConversationServiceInjector(AppConversationServiceInjector):
access_token_hard_timeout=access_token_hard_timeout,
conversation_secret_enricher=conversation_secret_enricher,
app_mode=app_mode,
+ export_max_events=self.export_max_events,
+ export_lock_ttl_seconds=self.export_lock_ttl_seconds,
+ export_lock_refresh_interval_seconds=(
+ self.export_lock_refresh_interval_seconds
+ ),
+ export_lock_required=self.export_lock_required,
)
diff --git a/openhands/app_server/event/event_service.py b/openhands/app_server/event/event_service.py
index fdfd85c216..3e11464874 100644
--- a/openhands/app_server/event/event_service.py
+++ b/openhands/app_server/event/event_service.py
@@ -2,6 +2,7 @@ import asyncio
import logging
from abc import ABC, abstractmethod
from datetime import datetime
+from typing import AsyncGenerator
from uuid import UUID
from openhands.agent_server.models import EventPage, EventSortOrder
@@ -9,6 +10,7 @@ from openhands.app_server.event_callback.event_callback_models import EventKind
from openhands.app_server.services.injector import Injector
from openhands.sdk import Event
from openhands.sdk.utils.models import DiscriminatedUnionMixin
+from openhands.sdk.utils.paging import page_iterator
_logger = logging.getLogger(__name__)
@@ -43,6 +45,18 @@ class EventService(ABC):
) -> int:
"""Count events matching the given filters."""
+ async def iter_events_for_export(
+ self, conversation_id: UUID
+ ) -> AsyncGenerator[Event, None]:
+ """Iterate all events for a conversation in export order.
+
+ Implementations can override this to avoid paginated searches that reload the
+ full event history for each page.
+ """
+ events = page_iterator(self.search_events, conversation_id=conversation_id)
+ async for event in events:
+ yield event
+
@abstractmethod
async def save_event(self, conversation_id: UUID, event: Event):
"""Save an event. Internal method intended not be part of the REST api."""
diff --git a/openhands/app_server/event/event_service_base.py b/openhands/app_server/event/event_service_base.py
index 381b4d06f6..fa44293651 100644
--- a/openhands/app_server/event/event_service_base.py
+++ b/openhands/app_server/event/event_service_base.py
@@ -1,8 +1,10 @@
import asyncio
+import os
from abc import ABC, abstractmethod
from dataclasses import dataclass
from datetime import datetime
from pathlib import Path
+from typing import AsyncGenerator
from uuid import UUID
from openhands.agent_server.models import EventPage, EventSortOrder
@@ -19,6 +21,13 @@ from openhands.sdk import Event
from openhands.sdk.utils.paging import page_iterator
+def _event_load_concurrency() -> int:
+ try:
+ return max(1, int(os.getenv('EVENT_SERVICE_LOAD_EVENT_CONCURRENCY', '10')))
+ except ValueError:
+ return 10
+
+
@dataclass
class EventServiceBase(EventService, ABC):
"""Event Service for getting events - the only check on permissions for events is
@@ -44,6 +53,16 @@ class EventServiceBase(EventService, ABC):
def _search_paths(self, prefix: Path) -> list[Path]:
"""Search paths."""
+ async def _load_events_from_paths(self, paths: list[Path]) -> list[Event | None]:
+ loop = asyncio.get_running_loop()
+ semaphore = asyncio.Semaphore(_event_load_concurrency())
+
+ async def load_event(path: Path) -> Event | None:
+ async with semaphore:
+ return await loop.run_in_executor(None, self._load_event, path)
+
+ return await asyncio.gather(*(load_event(path) for path in paths))
+
async def get_conversation_path(self, conversation_id: UUID) -> Path:
"""Get a path for a conversation. Ensure user_id is included if possible."""
path = self.prefix
@@ -87,10 +106,7 @@ class EventServiceBase(EventService, ABC):
prefix = await self.get_conversation_path(conversation_id)
paths = await loop.run_in_executor(None, self._search_paths, prefix)
- # Type error: run_in_executor expects a return value, but self._load_event is typed return Event | None.
- events = await asyncio.gather(
- *[loop.run_in_executor(None, self._load_event, path) for path in paths] # type: ignore[arg-type]
- )
+ events = await self._load_events_from_paths(paths)
# Convert datetime filters to ISO strings so they can be compared
# against event.timestamp (which is stored as an ISO 8601 string).
timestamp_gte_str = timestamp__gte.isoformat() if timestamp__gte else None
@@ -126,6 +142,19 @@ class EventServiceBase(EventService, ABC):
return EventPage(items=items, next_page_id=next_page_id)
+ async def iter_events_for_export(
+ self, conversation_id: UUID
+ ) -> AsyncGenerator[Event, None]:
+ """Iterate all events once in timestamp order for trajectory export."""
+ loop = asyncio.get_running_loop()
+ prefix = await self.get_conversation_path(conversation_id)
+ paths = await loop.run_in_executor(None, self._search_paths, prefix)
+ events = await self._load_events_from_paths(paths)
+ items = [event for event in events if event]
+ items.sort(key=lambda event: event.timestamp)
+ for event in items:
+ yield event
+
async def count_events(
self,
conversation_id: UUID,
diff --git a/openhands/app_server/utils/redis.py b/openhands/app_server/utils/redis.py
new file mode 100644
index 0000000000..72d0acfe7c
--- /dev/null
+++ b/openhands/app_server/utils/redis.py
@@ -0,0 +1,81 @@
+import os
+import threading
+
+from redis import Redis
+from redis import asyncio as aioredis
+from redis import exceptions as redis_exceptions
+
+# Redis configuration
+REDIS_HOST = os.environ.get('REDIS_HOST', 'localhost')
+REDIS_PORT = int(os.environ.get('REDIS_PORT', '6379'))
+REDIS_PASSWORD = os.environ.get('REDIS_PASSWORD', '')
+REDIS_DB = int(os.environ.get('REDIS_DB', '0'))
+REDIS_SOCKET_TIMEOUT = 2
+
+_redis_client: Redis | None = None
+_redis_client_async: aioredis.Redis | None = None
+_redis_lock = threading.Lock()
+
+
+def _get_redis_kwargs():
+ """Return common kwargs for Redis client creation."""
+ return {
+ 'host': REDIS_HOST,
+ 'port': REDIS_PORT,
+ 'password': REDIS_PASSWORD,
+ 'db': REDIS_DB,
+ 'socket_timeout': REDIS_SOCKET_TIMEOUT,
+ }
+
+
+def get_redis_client() -> Redis:
+ """Get a shared synchronous Redis client, lazily initialized.
+
+ Thread-safe with double-checked locking pattern.
+
+ Returns:
+ A Redis client for synchronous operations.
+ """
+ global _redis_client
+ if _redis_client is None:
+ with _redis_lock:
+ if _redis_client is None:
+ _redis_client = Redis(**_get_redis_kwargs())
+ return _redis_client
+
+
+def get_redis_client_async() -> aioredis.Redis:
+ """Get a shared asynchronous Redis client, lazily initialized.
+
+ Note: This function is synchronous but returns an async client.
+ Thread-safe initialization is handled via a threading lock since
+ asyncio.Lock cannot be used in a sync context.
+
+ Returns:
+ An aioredis client for asynchronous operations.
+ """
+ global _redis_client_async
+ if _redis_client_async is None:
+ with _redis_lock:
+ if _redis_client_async is None:
+ _redis_client_async = aioredis.Redis(**_get_redis_kwargs())
+ return _redis_client_async
+
+
+def get_redis_authed_url():
+ return f'redis://:{REDIS_PASSWORD}@{REDIS_HOST}:{REDIS_PORT}/{REDIS_DB}'
+
+
+__all__ = [
+ 'REDIS_DB',
+ 'REDIS_HOST',
+ 'REDIS_PASSWORD',
+ 'REDIS_PORT',
+ 'REDIS_SOCKET_TIMEOUT',
+ 'Redis',
+ 'aioredis',
+ 'get_redis_client',
+ 'get_redis_client_async',
+ 'get_redis_authed_url',
+ 'redis_exceptions',
+]
diff --git a/openhands/app_server/utils/redis_lock.py b/openhands/app_server/utils/redis_lock.py
new file mode 100644
index 0000000000..11e2a07e42
--- /dev/null
+++ b/openhands/app_server/utils/redis_lock.py
@@ -0,0 +1,53 @@
+import asyncio
+import logging
+
+from redis.asyncio.lock import Lock
+from redis.exceptions import LockError
+
+from openhands.app_server.utils.redis import get_redis_client_async, redis_exceptions
+
+_logger = logging.getLogger(__name__)
+
+# Re-export so callers can catch lock errors without importing redis directly.
+__all__ = [
+ 'Lock',
+ 'LockError',
+ 'RedisLockUnavailable',
+ 'try_acquire_redis_lock',
+ 'refresh_lock_periodically',
+]
+
+
+class RedisLockUnavailable(Exception):
+ """Raised when Redis cannot be used to evaluate a lock."""
+
+
+async def try_acquire_redis_lock(key: str, ttl_seconds: int) -> Lock | None:
+ """Try to acquire a Redis lock; return None if already held by another caller."""
+ redis = get_redis_client_async()
+ lock = redis.lock(key, timeout=ttl_seconds)
+ try:
+ acquired = await lock.acquire(blocking=False)
+ except redis_exceptions.RedisError as e:
+ raise RedisLockUnavailable from e
+ return lock if acquired else None
+
+
+async def refresh_lock_periodically(lock: Lock, interval: int) -> None:
+ """Keep a Redis lock alive by refreshing its TTL every *interval* seconds.
+
+ Intended to run as a background task (via ``asyncio.create_task``) alongside
+ a long-running operation. Cancel the task when the operation finishes; the
+ caller is responsible for releasing the lock afterwards.
+ """
+ try:
+ while True:
+ await asyncio.sleep(interval)
+ try:
+ await lock.reacquire()
+ except LockError:
+ _logger.warning(
+ 'redis_lock:periodic_refresh_failed', extra={'key': lock.name}
+ )
+ except asyncio.CancelledError:
+ pass
diff --git a/tests/unit/app_server/test_filesystem_event_service.py b/tests/unit/app_server/test_filesystem_event_service.py
index 606e9afcca..85b72a776e 100644
--- a/tests/unit/app_server/test_filesystem_event_service.py
+++ b/tests/unit/app_server/test_filesystem_event_service.py
@@ -144,6 +144,29 @@ class TestFilesystemEventServiceSearchEvents:
timestamps = [item.timestamp for item in result.items]
assert timestamps == sorted(timestamps, reverse=True)
+ @pytest.mark.asyncio
+ async def test_iter_events_for_export_returns_all_events_in_timestamp_order(
+ self, service: FilesystemEventService
+ ):
+ """Test export iterator returns all events once in timestamp order."""
+ conversation_id = uuid4()
+ events = []
+
+ for _ in range(3):
+ event = create_token_event()
+ events.append(event)
+ await service.save_event(conversation_id, event)
+ time.sleep(0.01)
+
+ result = [
+ event async for event in service.iter_events_for_export(conversation_id)
+ ]
+
+ assert [event.id for event in result] == [event.id for event in events]
+ assert [event.timestamp for event in result] == sorted(
+ event.timestamp for event in result
+ )
+
@pytest.mark.asyncio
async def test_search_events_returns_event_page(
self, service: FilesystemEventService
diff --git a/tests/unit/app_server/test_live_status_app_conversation_service.py b/tests/unit/app_server/test_live_status_app_conversation_service.py
index 884d909b6c..bf22e1e6a2 100644
--- a/tests/unit/app_server/test_live_status_app_conversation_service.py
+++ b/tests/unit/app_server/test_live_status_app_conversation_service.py
@@ -1,6 +1,5 @@
"""Unit tests for the methods in LiveStatusAppConversationService."""
-import builtins
import io
import json
import os
@@ -24,6 +23,11 @@ from openhands.app_server.app_conversation.app_conversation_models import (
AppConversationStartRequest,
ConversationTrigger,
)
+from openhands.app_server.app_conversation.app_conversation_service import (
+ ConversationExportAlreadyRunning,
+ ConversationExportLockUnavailable,
+ ConversationExportTooLarge,
+)
from openhands.app_server.app_conversation.live_status_app_conversation_service import (
LiveStatusAppConversationService,
)
@@ -43,6 +47,7 @@ from openhands.app_server.settings.settings_models import (
Settings,
)
from openhands.app_server.user.user_context import UserContext
+from openhands.app_server.utils.redis_lock import RedisLockUnavailable
from openhands.sdk import Agent, AgentContext, Event
from openhands.sdk.llm import LLM
from openhands.sdk.secret import LookupSecret, StaticSecret
@@ -50,6 +55,27 @@ from openhands.sdk.settings import ConversationSettings, OpenHandsAgentSettings
from openhands.sdk.workspace.remote.async_remote_workspace import AsyncRemoteWorkspace
+def _async_iter(items):
+ async def iter_items():
+ for item in items:
+ yield item
+
+ return iter_items()
+
+
+class _FakeExportLock:
+ def __init__(self):
+ self.name = 'fake-lock'
+ self.released = False
+ self.reacquired = False
+
+ async def reacquire(self):
+ self.reacquired = True
+
+ async def release(self):
+ self.released = True
+
+
def _build_test_user_agent_settings(user: SimpleNamespace) -> OpenHandsAgentSettings:
llm_vals: dict = {}
model = getattr(user, 'llm_model', '') or ''
@@ -1378,10 +1404,7 @@ class TestLiveStatusAppConversationService:
async def test_export_conversation_success(self):
"""Test successful download of conversation trajectory."""
- # Arrange
conversation_id = uuid4()
-
- # Mock conversation info
mock_conversation_info = Mock(spec=AppConversationInfo)
mock_conversation_info.id = conversation_id
mock_conversation_info.title = 'Test Conversation'
@@ -1398,7 +1421,6 @@ class TestLiveStatusAppConversationService:
return_value=mock_conversation_info
)
- # Mock events
mock_event1 = Mock(spec=Event)
mock_event1.id = uuid4()
mock_event1.model_dump = Mock(
@@ -1411,64 +1433,52 @@ class TestLiveStatusAppConversationService:
return_value={'id': str(mock_event2.id), 'type': 'observation'}
)
- # Mock event service search_events to return paginated results
- mock_event_page1 = Mock()
- mock_event_page1.items = [mock_event1]
- mock_event_page1.next_page_id = 'page2'
-
- mock_event_page2 = Mock()
- mock_event_page2.items = [mock_event2]
- mock_event_page2.next_page_id = None
-
- self.mock_event_service.search_events = AsyncMock(
- side_effect=[mock_event_page1, mock_event_page2]
+ self.mock_event_service.count_events = AsyncMock(return_value=2)
+ self.mock_event_service.iter_events_for_export = Mock(
+ return_value=_async_iter([mock_event1, mock_event2])
)
+ self.mock_event_service.search_events = AsyncMock()
- # Act
result = await self.service.export_conversation(conversation_id)
- # Assert
assert result is not None
- assert isinstance(result, bytes) # Should be bytes
+ assert isinstance(result, bytes)
- # Verify the zip file contents
with zipfile.ZipFile(io.BytesIO(result), 'r') as zipf:
file_list = zipf.namelist()
- # Should contain meta.json and event files
assert 'meta.json' in file_list
assert any(
f.startswith('event_') and f.endswith('.json') for f in file_list
)
- # Check meta.json content
with zipf.open('meta.json') as meta_file:
meta_content = meta_file.read().decode('utf-8')
assert '"id": "test"' in meta_content
assert '"title": "Test Conversation"' in meta_content
- # Check event files
event_files = [f for f in file_list if f.startswith('event_')]
- assert len(event_files) == 2 # Should have 2 event files
+ assert len(event_files) == 2
- # Verify event file content
with zipf.open(event_files[0]) as event_file:
event_content = json.loads(event_file.read().decode('utf-8'))
assert 'id' in event_content
assert 'type' in event_content
- # Verify service calls
self.mock_app_conversation_info_service.get_app_conversation_info.assert_called_once_with(
conversation_id
)
- assert self.mock_event_service.search_events.call_count == 2
+ self.mock_event_service.count_events.assert_awaited_once_with(conversation_id)
+ self.mock_event_service.iter_events_for_export.assert_called_once_with(
+ conversation_id
+ )
+ self.mock_event_service.search_events.assert_not_called()
mock_conversation_info.model_dump_json.assert_called_once_with(indent=2)
async def test_export_conversation_writes_utf8_when_platform_default_cannot_encode(
self,
):
"""Test trajectory export writes JSON as UTF-8 regardless of platform default."""
- # Arrange
conversation_id = uuid4()
mock_conversation_info = Mock(spec=AppConversationInfo)
@@ -1492,27 +1502,13 @@ class TestLiveStatusAppConversationService:
}
)
- mock_event_page = Mock()
- mock_event_page.items = [mock_event]
- mock_event_page.next_page_id = None
- self.mock_event_service.search_events = AsyncMock(return_value=mock_event_page)
-
- real_open = builtins.open
-
- def cp1252_default_open(file, mode='r', *args, **kwargs):
- if 'w' in mode and 'encoding' not in kwargs:
- kwargs['encoding'] = 'cp1252'
- return real_open(file, mode, *args, **kwargs)
+ self.mock_event_service.count_events = AsyncMock(return_value=1)
+ self.mock_event_service.iter_events_for_export = Mock(
+ return_value=_async_iter([mock_event])
+ )
- # Act
- with patch(
- 'openhands.app_server.app_conversation.live_status_app_conversation_service.open',
- cp1252_default_open,
- create=True,
- ):
- result = await self.service.export_conversation(conversation_id)
+ result = await self.service.export_conversation(conversation_id)
- # Assert
with zipfile.ZipFile(io.BytesIO(result), 'r') as zipf:
with zipf.open('meta.json') as meta_file:
meta_content = meta_file.read().decode('utf-8')
@@ -1529,31 +1525,29 @@ class TestLiveStatusAppConversationService:
@pytest.mark.asyncio
async def test_export_conversation_conversation_not_found(self):
"""Test download when conversation is not found."""
- # Arrange
conversation_id = uuid4()
self.mock_app_conversation_info_service.get_app_conversation_info = AsyncMock(
return_value=None
)
+ self.mock_event_service.count_events = AsyncMock()
+ self.mock_event_service.iter_events_for_export = Mock()
- # Act & Assert
with pytest.raises(
ValueError, match=f'Conversation not found: {conversation_id}'
):
await self.service.export_conversation(conversation_id)
- # Verify service calls
self.mock_app_conversation_info_service.get_app_conversation_info.assert_called_once_with(
conversation_id
)
- self.mock_event_service.search_events.assert_not_called()
+ self.mock_event_service.count_events.assert_not_called()
+ self.mock_event_service.iter_events_for_export.assert_not_called()
@pytest.mark.asyncio
async def test_export_conversation_empty_events(self):
"""Test download with conversation that has no events."""
- # Arrange
conversation_id = uuid4()
- # Mock conversation info
mock_conversation_info = Mock(spec=AppConversationInfo)
mock_conversation_info.id = conversation_id
mock_conversation_info.title = 'Empty Conversation'
@@ -1565,48 +1559,36 @@ class TestLiveStatusAppConversationService:
return_value=mock_conversation_info
)
- # Mock empty event page
- mock_event_page = Mock()
- mock_event_page.items = []
- mock_event_page.next_page_id = None
-
- self.mock_event_service.search_events = AsyncMock(return_value=mock_event_page)
+ self.mock_event_service.count_events = AsyncMock(return_value=0)
+ self.mock_event_service.iter_events_for_export = Mock(
+ return_value=_async_iter([])
+ )
- # Act
result = await self.service.export_conversation(conversation_id)
- # Assert
assert result is not None
- assert isinstance(result, bytes) # Should be bytes
+ assert isinstance(result, bytes)
- # Verify the zip file contents
with zipfile.ZipFile(io.BytesIO(result), 'r') as zipf:
file_list = zipf.namelist()
- # Should only contain meta.json (no event files)
assert 'meta.json' in file_list
assert len([f for f in file_list if f.startswith('event_')]) == 0
- # Verify service calls
self.mock_app_conversation_info_service.get_app_conversation_info.assert_called_once_with(
conversation_id
)
- self.mock_event_service.search_events.assert_called_once()
+ self.mock_event_service.count_events.assert_awaited_once_with(conversation_id)
+ self.mock_event_service.iter_events_for_export.assert_called_once_with(
+ conversation_id
+ )
@pytest.mark.asyncio
- async def test_export_conversation_calls_search_events_with_correct_parameter_name(
+ async def test_export_conversation_calls_iterator_with_conversation_id(
self,
):
- """Test that export_conversation calls search_events with 'conversation_id' parameter, not 'conversation_id__eq'.
-
- This test verifies the fix for a bug where page_iterator was called with
- conversation_id__eq instead of conversation_id, causing a TypeError since
- the search_events method expects conversation_id as its parameter name.
- """
- # Arrange
+ """Test that export_conversation calls the export iterator with conversation_id."""
conversation_id = uuid4()
-
- # Mock conversation info
mock_conversation_info = Mock(spec=AppConversationInfo)
mock_conversation_info.id = conversation_id
mock_conversation_info.model_dump_json = Mock(return_value='{}')
@@ -1615,35 +1597,23 @@ class TestLiveStatusAppConversationService:
return_value=mock_conversation_info
)
- # Mock empty event page to simplify test
- mock_event_page = Mock()
- mock_event_page.items = []
- mock_event_page.next_page_id = None
-
- self.mock_event_service.search_events = AsyncMock(return_value=mock_event_page)
+ self.mock_event_service.count_events = AsyncMock(return_value=0)
+ self.mock_event_service.iter_events_for_export = Mock(
+ return_value=_async_iter([])
+ )
+ self.mock_event_service.search_events = AsyncMock()
- # Act
await self.service.export_conversation(conversation_id)
- # Assert - Verify search_events was called with 'conversation_id', not 'conversation_id__eq'
- self.mock_event_service.search_events.assert_called()
- call_kwargs = self.mock_event_service.search_events.call_args[1]
-
- assert 'conversation_id' in call_kwargs, (
- "search_events should be called with 'conversation_id' parameter"
- )
- assert 'conversation_id__eq' not in call_kwargs, (
- "search_events should NOT be called with 'conversation_id__eq' parameter"
+ self.mock_event_service.iter_events_for_export.assert_called_once_with(
+ conversation_id
)
- assert call_kwargs['conversation_id'] == conversation_id
+ self.mock_event_service.search_events.assert_not_called()
@pytest.mark.asyncio
- async def test_export_conversation_large_pagination(self):
- """Test download with multiple pages of events."""
- # Arrange
+ async def test_export_conversation_large_iterator(self):
+ """Test download with many events from one export iterator pass."""
conversation_id = uuid4()
-
- # Mock conversation info
mock_conversation_info = Mock(spec=AppConversationInfo)
mock_conversation_info.id = conversation_id
mock_conversation_info.title = 'Large Conversation'
@@ -1655,64 +1625,171 @@ class TestLiveStatusAppConversationService:
return_value=mock_conversation_info
)
- # Create multiple pages of events
- events_per_page = 3
- total_pages = 4
+ total_events = 12
all_events = []
- for page_num in range(total_pages):
- page_events = []
- for i in range(events_per_page):
- mock_event = Mock(spec=Event)
- mock_event.id = uuid4()
- mock_event.model_dump = Mock(
- return_value={
- 'id': str(mock_event.id),
- 'type': f'event_page_{page_num}_item_{i}',
- }
- )
- page_events.append(mock_event)
- all_events.append(mock_event)
-
- mock_event_page = Mock()
- mock_event_page.items = page_events
- mock_event_page.next_page_id = (
- f'page{page_num + 1}' if page_num < total_pages - 1 else None
+ for i in range(total_events):
+ mock_event = Mock(spec=Event)
+ mock_event.id = uuid4()
+ mock_event.model_dump = Mock(
+ return_value={
+ 'id': str(mock_event.id),
+ 'type': f'event_{i}',
+ }
)
+ all_events.append(mock_event)
- if page_num == 0:
- first_page = mock_event_page
- elif page_num == 1:
- second_page = mock_event_page
- elif page_num == 2:
- third_page = mock_event_page
- else:
- fourth_page = mock_event_page
-
- self.mock_event_service.search_events = AsyncMock(
- side_effect=[first_page, second_page, third_page, fourth_page]
+ self.mock_event_service.count_events = AsyncMock(return_value=total_events)
+ self.mock_event_service.iter_events_for_export = Mock(
+ return_value=_async_iter(all_events)
)
+ self.mock_event_service.search_events = AsyncMock()
- # Act
result = await self.service.export_conversation(conversation_id)
- # Assert
assert result is not None
- assert isinstance(result, bytes) # Should be bytes
+ assert isinstance(result, bytes)
- # Verify the zip file contents
with zipfile.ZipFile(io.BytesIO(result), 'r') as zipf:
file_list = zipf.namelist()
- # Should contain meta.json and all event files
assert 'meta.json' in file_list
event_files = [f for f in file_list if f.startswith('event_')]
- assert (
- len(event_files) == total_pages * events_per_page
- ) # Should have all events
+ assert len(event_files) == total_events
+
+ self.mock_event_service.iter_events_for_export.assert_called_once_with(
+ conversation_id
+ )
+ self.mock_event_service.search_events.assert_not_called()
+
+ @pytest.mark.asyncio
+ async def test_open_conversation_export_streams_with_lock(self):
+ """Test streaming export acquires and releases the conversation export lock."""
+ conversation_id = uuid4()
+ mock_conversation_info = Mock(spec=AppConversationInfo)
+ mock_conversation_info.id = conversation_id
+ mock_conversation_info.model_dump_json = Mock(return_value='{}')
+
+ mock_event = Mock(spec=Event)
+ mock_event.id = uuid4()
+ mock_event.model_dump = Mock(return_value={'id': str(mock_event.id)})
+
+ self.mock_app_conversation_info_service.get_app_conversation_info = AsyncMock(
+ return_value=mock_conversation_info
+ )
+ self.mock_event_service.count_events = AsyncMock(return_value=1)
+ self.mock_event_service.iter_events_for_export = Mock(
+ return_value=_async_iter([mock_event])
+ )
+
+ fake_lock = _FakeExportLock()
+ with patch(
+ 'openhands.app_server.app_conversation.live_status_app_conversation_service.try_acquire_redis_lock',
+ new=AsyncMock(return_value=fake_lock),
+ ) as mock_acquire_lock:
+ stream = await self.service.open_conversation_export(conversation_id)
+ result = b''.join([chunk async for chunk in stream])
+
+ assert result
+ assert fake_lock.released is True
+ mock_acquire_lock.assert_awaited_once_with(
+ f'app_conversation_export:{conversation_id.hex}',
+ self.service.export_lock_ttl_seconds,
+ )
+
+ @pytest.mark.asyncio
+ async def test_open_conversation_export_rejects_duplicate_export(self):
+ """Test streaming export rejects duplicate export attempts."""
+ conversation_id = uuid4()
+ mock_conversation_info = Mock(spec=AppConversationInfo)
+ mock_conversation_info.id = conversation_id
+ mock_conversation_info.model_dump_json = Mock(return_value='{}')
+
+ self.mock_app_conversation_info_service.get_app_conversation_info = AsyncMock(
+ return_value=mock_conversation_info
+ )
+
+ with patch(
+ 'openhands.app_server.app_conversation.live_status_app_conversation_service.try_acquire_redis_lock',
+ new=AsyncMock(return_value=None),
+ ):
+ with pytest.raises(ConversationExportAlreadyRunning):
+ await self.service.open_conversation_export(conversation_id)
+
+ self.mock_event_service.count_events.assert_not_called()
+
+ @pytest.mark.asyncio
+ async def test_open_conversation_export_fails_closed_when_lock_unavailable(self):
+ """Test streaming export fails closed when Redis lock cannot be checked."""
+ conversation_id = uuid4()
+ mock_conversation_info = Mock(spec=AppConversationInfo)
+ mock_conversation_info.id = conversation_id
+ mock_conversation_info.model_dump_json = Mock(return_value='{}')
+
+ self.service.export_lock_required = True
+ self.mock_app_conversation_info_service.get_app_conversation_info = AsyncMock(
+ return_value=mock_conversation_info
+ )
+
+ with patch(
+ 'openhands.app_server.app_conversation.live_status_app_conversation_service.try_acquire_redis_lock',
+ new=AsyncMock(side_effect=RedisLockUnavailable()),
+ ):
+ with pytest.raises(ConversationExportLockUnavailable):
+ await self.service.open_conversation_export(conversation_id)
+
+ self.mock_event_service.count_events.assert_not_called()
+
+ @pytest.mark.asyncio
+ async def test_open_conversation_export_proceeds_when_lock_not_required(self):
+ """Test non-SaaS export proceeds without Redis when locking is best effort."""
+ conversation_id = uuid4()
+ mock_conversation_info = Mock(spec=AppConversationInfo)
+ mock_conversation_info.id = conversation_id
+ mock_conversation_info.model_dump_json = Mock(return_value='{}')
+
+ mock_event = Mock(spec=Event)
+ mock_event.id = uuid4()
+ mock_event.model_dump = Mock(return_value={'id': str(mock_event.id)})
+
+ self.service.export_lock_required = False
+ self.mock_app_conversation_info_service.get_app_conversation_info = AsyncMock(
+ return_value=mock_conversation_info
+ )
+ self.mock_event_service.count_events = AsyncMock(return_value=1)
+ self.mock_event_service.iter_events_for_export = Mock(
+ return_value=_async_iter([mock_event])
+ )
+
+ with patch(
+ 'openhands.app_server.app_conversation.live_status_app_conversation_service.try_acquire_redis_lock',
+ new=AsyncMock(side_effect=RedisLockUnavailable()),
+ ):
+ stream = await self.service.open_conversation_export(conversation_id)
+ result = b''.join([chunk async for chunk in stream])
+
+ assert result
+ self.mock_event_service.count_events.assert_awaited_once_with(conversation_id)
+
+ @pytest.mark.asyncio
+ async def test_export_conversation_rejects_too_many_events(self):
+ """Test export rejects conversations over the configured event limit."""
+ conversation_id = uuid4()
+ mock_conversation_info = Mock(spec=AppConversationInfo)
+ mock_conversation_info.id = conversation_id
+ mock_conversation_info.model_dump_json = Mock(return_value='{}')
+
+ self.service.export_max_events = 1
+ self.mock_app_conversation_info_service.get_app_conversation_info = AsyncMock(
+ return_value=mock_conversation_info
+ )
+ self.mock_event_service.count_events = AsyncMock(return_value=2)
+ self.mock_event_service.iter_events_for_export = Mock()
+
+ with pytest.raises(ConversationExportTooLarge):
+ await self.service.export_conversation(conversation_id)
- # Verify service calls - should call search_events for each page
- assert self.mock_event_service.search_events.call_count == total_pages
+ self.mock_event_service.iter_events_for_export.assert_not_called()
@patch(
'openhands.app_server.app_conversation.live_status_app_conversation_service.AsyncRemoteWorkspace'