Skip to content

πŸ™Œ OpenHands: AI-Driven Development

feat(app-server): capture production workspace state β€” initial snapshot at start + archive before delete (APP-2403) (#14953)

simonrosenberg77bd80d3a44c
diff --git a/openhands/app_server/app_conversation/app_conversation_models.py b/openhands/app_server/app_conversation/app_conversation_models.py
index 5fc97bed68..fc4c928399 100644
--- a/openhands/app_server/app_conversation/app_conversation_models.py
+++ b/openhands/app_server/app_conversation/app_conversation_models.py
@@ -33,6 +33,12 @@ __all__ = ['SandboxGroupingStrategy']
# The typed ``AppConversationInfo.acp_server`` field is a projection of this tag.
ACP_SERVER_TAG_KEY = 'acpserver'
+# Conversation-tag key pinning the resolved (grouped) workspace path at creation
+# so the delete-time archive captures the right directory without re-deriving it
+# from settings that may have changed. Must satisfy the SDK ^[a-z0-9]+$ tag-key
+# rule β€” no underscores.
+ARCHIVE_WORKSPACE_PATH_TAG_KEY = 'archiveworkspacepath'
+
class ConversationTrigger(Enum):
RESOLVER = 'resolver'
diff --git a/openhands/app_server/app_conversation/app_conversation_router.py b/openhands/app_server/app_conversation/app_conversation_router.py
index 83e26e7fca..fed3db1f1e 100644
--- a/openhands/app_server/app_conversation/app_conversation_router.py
+++ b/openhands/app_server/app_conversation/app_conversation_router.py
@@ -23,6 +23,7 @@ from openhands.app_server.app_conversation.app_conversation_info_service import
AppConversationInfoService,
)
from openhands.app_server.app_conversation.app_conversation_models import (
+ ARCHIVE_WORKSPACE_PATH_TAG_KEY,
AppConversation,
AppConversationInfo,
AppConversationPage,
@@ -871,17 +872,52 @@ async def _finalize_sandbox_delete(
sandbox_id: str,
db_session: AsyncSession,
httpx_client: httpx.AsyncClient,
+ conversation_id: UUID | None = None,
+ workspace_path: str | None = None,
) -> None:
- """Delete sandbox if no other conversations reference it, then close connections."""
+ """Archive the conversation's workspace, then delete the sandbox if unreferenced.
+
+ Runs detached (background task) AFTER the delete response was already returned.
+ The workspace is captured FIRST β€” a conversation-scoped step, while the runtime
+ is still up β€” and only if that succeeds (or archiving is not REQUIRED) is the
+ sandbox torn down, and only when no other conversation still references it
+ (under grouping a sibling conversation keeps it alive). When archiving is
+ REQUIRED and fails, the sandbox + running runtime are kept so the runtime-api
+ idle reap captures the workspace later (the durability backstop). delete_sandbox
+ is sandbox-scoped (stop + delete) and knows nothing about conversations.
+ """
try:
- conversation_count = (
- await app_conversation_info_service.count_conversations_by_sandbox_id(
- sandbox_id
+ archived = await sandbox_service.archive_conversation_workspace(
+ sandbox_id,
+ conversation_id=conversation_id.hex if conversation_id else None,
+ workspace_path=workspace_path,
+ )
+ if not archived:
+ # REQUIRED archive failed: keep the sandbox + running runtime for the
+ # runtime-api idle reap to capture (the durability backstop).
+ logger.warning(
+ 'Workspace archive required but failed for %s; leaving the '
+ 'sandbox + runtime for the idle reap',
+ sandbox_id,
)
- )
- if conversation_count == 0:
- await sandbox_service.delete_sandbox(sandbox_id)
+ else:
+ conversation_count = (
+ await app_conversation_info_service.count_conversations_by_sandbox_id(
+ sandbox_id
+ )
+ )
+ if conversation_count == 0:
+ await sandbox_service.delete_sandbox(sandbox_id)
await db_session.commit()
+ except Exception:
+ # Any failure in the finalizer (a transient stop/lookup error, the count
+ # query, the commit itself): do NOT commit a half-done delete, so no
+ # orphaned row is left; the row + running runtime stay for the runtime-api
+ # idle reap to capture + reap.
+ logger.exception(
+ 'Deferred sandbox cleanup failed for %s; kept for retry', sandbox_id
+ )
+ await db_session.rollback()
finally:
await asyncio.gather(
db_session.aclose(),
@@ -965,6 +1001,8 @@ async def delete_app_conversation(
# Delete the sandbox in the background if no other conversations reference it
if sandbox_id:
+ # Path pinned at creation; the finalizer archives exactly this directory.
+ workspace_path = app_conversation_info.tags.get(ARCHIVE_WORKSPACE_PATH_TAG_KEY)
asyncio.create_task(
_finalize_sandbox_delete(
sandbox_service,
@@ -972,6 +1010,8 @@ async def delete_app_conversation(
sandbox_id,
db_session,
httpx_client,
+ conversation_id=conversation_uuid,
+ workspace_path=workspace_path,
)
)
diff --git a/openhands/app_server/app_conversation/app_conversation_service.py b/openhands/app_server/app_conversation/app_conversation_service.py
index b2aa155eb7..484c615bd8 100644
--- a/openhands/app_server/app_conversation/app_conversation_service.py
+++ b/openhands/app_server/app_conversation/app_conversation_service.py
@@ -122,6 +122,7 @@ class AppConversationService(ABC):
sandbox: SandboxInfo,
workspace: AsyncRemoteWorkspace,
agent_server_url: str,
+ conversation_id: UUID,
) -> AsyncGenerator[AppConversationStartTask, None]:
"""Run the setup scripts for the project and yield status updates"""
yield task
diff --git a/openhands/app_server/app_conversation/app_conversation_service_base.py b/openhands/app_server/app_conversation/app_conversation_service_base.py
index 174f7085d6..06e69a14fe 100644
--- a/openhands/app_server/app_conversation/app_conversation_service_base.py
+++ b/openhands/app_server/app_conversation/app_conversation_service_base.py
@@ -1,3 +1,4 @@
+import asyncio
import logging
import os
import shlex
@@ -27,6 +28,7 @@ from openhands.app_server.app_conversation.skill_loader import (
load_skills_from_agent_server,
)
from openhands.app_server.integrations.service_types import ProviderType
+from openhands.app_server.sandbox import workspace_archive
from openhands.app_server.sandbox.sandbox_models import SandboxInfo
from openhands.app_server.user.user_context import UserContext
from openhands.app_server.utils.auth import looks_like_jwt
@@ -46,6 +48,7 @@ from openhands.sdk.skills import Skill
from openhands.sdk.workspace.remote.async_remote_workspace import AsyncRemoteWorkspace
_logger = logging.getLogger(__name__)
+
PRE_COMMIT_HOOK = '.git/hooks/pre-commit'
PRE_COMMIT_LOCAL = '.git/hooks/pre-commit.local'
@@ -252,6 +255,7 @@ class AppConversationServiceBase(AppConversationService, ABC):
sandbox: SandboxInfo,
workspace: AsyncRemoteWorkspace,
agent_server_url: str,
+ conversation_id: UUID,
) -> AsyncGenerator[AppConversationStartTask, None]:
task.status = AppConversationStartTaskStatus.PREPARING_REPOSITORY
yield task
@@ -264,6 +268,28 @@ class AppConversationServiceBase(AppConversationService, ABC):
workspace.working_dir, task.request.selected_repository
)
+ # Snapshot the INITIAL workspace (repo exactly as cloned, before setup.sh)
+ # for downstream dataset/eval creation. Awaited HERE β€” before any
+ # mutating step β€” so
+ # the capture is deterministic and can't race setup.sh writing the same
+ # tree. Best-effort and bounded so it never fails or unduly delays startup.
+ try:
+ await asyncio.wait_for(
+ self._maybe_archive_initial_state(
+ task,
+ sandbox,
+ workspace,
+ agent_server_url,
+ project_dir,
+ conversation_id,
+ ),
+ timeout=workspace_archive.initial_archive_deadline(),
+ )
+ except Exception as e:
+ _logger.warning(
+ 'Initial workspace archive did not complete in time (ignored): %s', e
+ )
+
task.status = AppConversationStartTaskStatus.RUNNING_SETUP_SCRIPT
yield task
await self.maybe_run_setup_script(workspace, project_dir)
@@ -281,6 +307,55 @@ class AppConversationServiceBase(AppConversationService, ABC):
agent_server_url,
)
+ async def _maybe_archive_initial_state(
+ self,
+ task: AppConversationStartTask,
+ sandbox: SandboxInfo,
+ workspace: AsyncRemoteWorkspace,
+ agent_server_url: str,
+ project_dir: str,
+ conversation_id: UUID,
+ ) -> None:
+ """Snapshot the initial (as-cloned) workspace state β€” best-effort.
+
+ Awaited before setup.sh, so it captures the repo exactly as cloned with no
+ concurrent mutation, keyed to the real conversation id. No-op unless
+ ``RUNTIME_FILE_ARCHIVE_INITIAL_ENABLED`` is set and a repository was
+ actually cloned (an empty workspace is not a useful starting state).
+ Every failure is swallowed so this can never break conversation startup.
+ """
+ try:
+ if not workspace_archive.initial_archive_enabled():
+ return
+ if not task.request.selected_repository:
+ # Only repo-backed conversations have a meaningful initial state.
+ return
+
+ # Record the commit the snapshot came from β€” tar.gz carries no
+ # base-commit header. Best-effort: a miss just leaves base_commit ''.
+ base_commit = ''
+ try:
+ result = await workspace.execute_command(
+ 'git rev-parse HEAD', project_dir
+ )
+ if not result.exit_code:
+ base_commit = result.stdout.strip()
+ except Exception as e:
+ _logger.debug('Initial-state base commit lookup failed: %s', e)
+
+ conv_hex = conversation_id.hex if conversation_id else None
+ await workspace_archive.archive_initial_workspace(
+ workspace.client,
+ agent_server_url=agent_server_url,
+ session_api_key=sandbox.session_api_key,
+ project_dir=project_dir,
+ sandbox_id=sandbox.id,
+ conversation_id=conv_hex,
+ base_commit=base_commit,
+ )
+ except Exception as e:
+ _logger.warning('Initial workspace archive step failed (ignored): %s', e)
+
async def _configure_git_user_settings(
self,
workspace: AsyncRemoteWorkspace,
diff --git a/openhands/app_server/app_conversation/live_status_app_conversation_service.py b/openhands/app_server/app_conversation/live_status_app_conversation_service.py
index 2d79f95234..de4f59f79c 100644
--- a/openhands/app_server/app_conversation/live_status_app_conversation_service.py
+++ b/openhands/app_server/app_conversation/live_status_app_conversation_service.py
@@ -28,6 +28,7 @@ from openhands.app_server.app_conversation.app_conversation_info_service import
)
from openhands.app_server.app_conversation.app_conversation_models import (
ACP_SERVER_TAG_KEY,
+ ARCHIVE_WORKSPACE_PATH_TAG_KEY,
AgentType,
AppConversation,
AppConversationInfo,
@@ -97,6 +98,7 @@ from openhands.app_server.sandbox.sandbox_spec_service import (
from openhands.app_server.services.injector import InjectorState
from openhands.app_server.services.jwt_service import JwtService
from openhands.app_server.settings.llm_profiles import resolve_profile_llm
+from openhands.app_server.settings.settings_models import grouped_workspace_dir
from openhands.app_server.user.user_context import UserContext
from openhands.app_server.user.user_models import UserInfo
from openhands.app_server.utils.docker_utils import (
@@ -395,10 +397,12 @@ class LiveStatusAppConversationService(AppConversationServiceBase):
conversation_id = request.conversation_id or uuid4()
# Setup working dir based on grouping
- working_dir = sandbox_spec.working_dir
sandbox_grouping_strategy = await self._get_sandbox_grouping_strategy()
- if sandbox_grouping_strategy != SandboxGroupingStrategy.NO_GROUPING:
- working_dir = f'{working_dir}/{conversation_id.hex}'
+ working_dir = grouped_workspace_dir(
+ sandbox_spec.working_dir,
+ sandbox_grouping_strategy,
+ conversation_id.hex,
+ )
# Run setup scripts
remote_workspace = AsyncRemoteWorkspace(
@@ -407,7 +411,7 @@ class LiveStatusAppConversationService(AppConversationServiceBase):
working_dir=working_dir,
)
async for updated_task in self.run_setup_scripts(
- task, sandbox, remote_workspace, agent_server_url
+ task, sandbox, remote_workspace, agent_server_url, conversation_id
):
yield updated_task
@@ -486,6 +490,10 @@ class LiveStatusAppConversationService(AppConversationServiceBase):
# the same agent back through the AgentBase discriminator.
request_agent = start_conversation_request.agent
tags: dict[str, str] = {}
+ # Pin where the workspace was actually created so the delete-time
+ # archive captures the right directory without re-deriving the path
+ # from settings (e.g. grouping) that may change before delete.
+ tags[ARCHIVE_WORKSPACE_PATH_TAG_KEY] = working_dir
if request_agent.agent_kind == 'acp':
llm_model = request_agent.acp_model
agent_kind = 'acp'
diff --git a/openhands/app_server/errors.py b/openhands/app_server/errors.py
index c7cf9614c0..4a8548fdf9 100644
--- a/openhands/app_server/errors.py
+++ b/openhands/app_server/errors.py
@@ -41,3 +41,22 @@ class PermissionsError(OpenHandsError):
class SandboxError(OpenHandsError):
"""Error in Sandbox."""
+
+
+class SandboxDeleteRetryError(OpenHandsError):
+ """The sandbox exists but its delete could not complete and was kept for retry.
+
+ Raised by ``delete_sandbox`` when the runtime /stop or lookup hits a transient
+ failure. (Archiving never raises β€” it returns False from
+ ``archive_conversation_workspace`` to signal a REQUIRED capture should block.)
+ 503 (vs 404) so a client distinguishes "still here, try again" from "not
+ found" and keeps retrying.
+ """
+
+ def __init__(
+ self,
+ detail: Any = None,
+ headers: dict[str, str] | None = None,
+ status_code: int = status.HTTP_503_SERVICE_UNAVAILABLE,
+ ):
+ super().__init__(status_code=status_code, detail=detail, headers=headers)
diff --git a/openhands/app_server/file_store/files.py b/openhands/app_server/file_store/files.py
index 0beb8ebfe9..ce5eedf518 100644
--- a/openhands/app_server/file_store/files.py
+++ b/openhands/app_server/file_store/files.py
@@ -17,6 +17,18 @@ class FileStore(DiscriminatedUnionMixin, ABC):
def write(self, path: str, contents: str | bytes) -> None:
pass
+ def write_from_path(self, path: str, source_path: str) -> None:
+ """Store the object at ``path`` from a local file at ``source_path``.
+
+ The default reads the whole file into memory and delegates to ``write``;
+ backends that can stream from disk override this to bound peak memory.
+ Callers uploading large blobs (e.g. multi-GB workspace archives, where
+ buffering one whole copy in RAM under concurrent deletes risks
+ OOM-killing the pod) should prefer this over ``write(path, f.read())``.
+ """
+ with open(source_path, 'rb') as f:
+ self.write(path, f.read())
+
@abstractmethod
def read(self, path: str) -> str:
pass
diff --git a/openhands/app_server/file_store/google_cloud.py b/openhands/app_server/file_store/google_cloud.py
index 271768e28f..882a0ee5f5 100644
--- a/openhands/app_server/file_store/google_cloud.py
+++ b/openhands/app_server/file_store/google_cloud.py
@@ -50,6 +50,11 @@ class GoogleCloudFileStore(FileStore):
with blob.open(mode) as f:
f.write(contents)
+ def write_from_path(self, path: str, source_path: str) -> None:
+ # Streams the file to GCS in chunks; never buffers the whole blob in RAM.
+ blob: Blob = self.bucket.blob(path)
+ blob.upload_from_filename(source_path)
+
def read(self, path: str) -> str:
blob: Blob = self.bucket.blob(path)
try:
diff --git a/openhands/app_server/file_store/local.py b/openhands/app_server/file_store/local.py
index 80058cd5dc..bdcafd9550 100644
--- a/openhands/app_server/file_store/local.py
+++ b/openhands/app_server/file_store/local.py
@@ -42,6 +42,20 @@ class LocalFileStore(FileStore):
os.remove(temp_path)
raise
+ def write_from_path(self, path: str, source_path: str) -> None:
+ # shutil.copyfile streams in chunks (never the whole file in RAM); keep
+ # the same write-temp-then-atomic-rename to avoid torn concurrent writes.
+ full_path = self.get_full_path(path)
+ os.makedirs(os.path.dirname(full_path), exist_ok=True)
+ temp_path = f'{full_path}.tmp.{os.getpid()}.{threading.get_ident()}'
+ try:
+ shutil.copyfile(source_path, temp_path)
+ os.replace(temp_path, full_path)
+ except Exception:
+ if os.path.exists(temp_path):
+ os.remove(temp_path)
+ raise
+
def read(self, path: str) -> str:
full_path = self.get_full_path(path)
with open(full_path, 'r') as f:
diff --git a/openhands/app_server/file_store/memory.py b/openhands/app_server/file_store/memory.py
index 5eba1b396a..b6e0530e6a 100644
--- a/openhands/app_server/file_store/memory.py
+++ b/openhands/app_server/file_store/memory.py
@@ -7,6 +7,10 @@ from openhands.app_server.utils.logger import openhands_logger as logger
class InMemoryFileStore(FileStore):
+ # Text-only by design: this store is part of the env-parsed FileStore config
+ # union (so the value type must stay a primitive) and read() returns str, so
+ # it cannot round-trip a binary archive. Not a valid RUNTIME_FILE_ARCHIVE
+ # store β€” see workspace_archive._archive_store_type.
files: dict[str, str] = Field(default_factory=dict)
def write(self, path: str, contents: str | bytes) -> None:
diff --git a/openhands/app_server/file_store/s3.py b/openhands/app_server/file_store/s3.py
index 430c0537b0..561e307918 100644
--- a/openhands/app_server/file_store/s3.py
+++ b/openhands/app_server/file_store/s3.py
@@ -75,6 +75,24 @@ class S3FileStore(FileStore):
f"Error: Failed to write to bucket '{self._get_bucket_name()}' at path {path}: {e}"
)
+ def write_from_path(self, path: str, source_path: str) -> None:
+ # upload_file streams the file from disk in parts; never buffers the whole
+ # object in RAM (unlike put_object with Body=f.read()).
+ try:
+ self.client.upload_file(source_path, self._get_bucket_name(), path)
+ except botocore.exceptions.ClientError as e:
+ if e.response['Error']['Code'] == 'AccessDenied':
+ raise FileNotFoundError(
+ f"Error: Access denied to bucket '{self._get_bucket_name()}'."
+ )
+ elif e.response['Error']['Code'] == 'NoSuchBucket':
+ raise FileNotFoundError(
+ f"Error: The bucket '{self._get_bucket_name()}' does not exist."
+ )
+ raise FileNotFoundError(
+ f"Error: Failed to write to bucket '{self._get_bucket_name()}' at path {path}: {e}"
+ )
+
def read(self, path: str) -> str:
try:
response: GetObjectOutputDict = self.client.get_object(
diff --git a/openhands/app_server/sandbox/process_sandbox_service.py b/openhands/app_server/sandbox/process_sandbox_service.py
index 8272793021..59fdf7207b 100644
--- a/openhands/app_server/sandbox/process_sandbox_service.py
+++ b/openhands/app_server/sandbox/process_sandbox_service.py
@@ -386,7 +386,7 @@ class ProcessSandboxService(SandboxService):
return False
async def delete_sandbox(self, sandbox_id: str) -> bool:
- """Delete a sandbox."""
+ """Delete a sandbox. (No workspace archiving for local processes.)"""
process_info = _processes.get(sandbox_id)
if process_info is None:
return False
diff --git a/openhands/app_server/sandbox/remote_sandbox_service.py b/openhands/app_server/sandbox/remote_sandbox_service.py
index f1e67241d5..19d34e60a8 100644
--- a/openhands/app_server/sandbox/remote_sandbox_service.py
+++ b/openhands/app_server/sandbox/remote_sandbox_service.py
@@ -24,7 +24,8 @@ from openhands.agent_server.utils import utc_now
from openhands.app_server.app_conversation.app_conversation_models import (
AppConversationInfo,
)
-from openhands.app_server.errors import SandboxError
+from openhands.app_server.errors import SandboxDeleteRetryError, SandboxError
+from openhands.app_server.sandbox import workspace_archive
from openhands.app_server.sandbox.sandbox_models import (
AGENT_SERVER,
VSCODE,
@@ -48,8 +49,12 @@ from openhands.app_server.sandbox.sandbox_spec_service import (
resolve_sandbox_spec,
)
from openhands.app_server.services.injector import InjectorState
+from openhands.app_server.settings.settings_models import grouped_workspace_dir
from openhands.app_server.user.specifiy_user_context import ADMIN, USER_CONTEXT_ATTR
from openhands.app_server.user.user_context import UserContext
+from openhands.app_server.utils.docker_utils import (
+ replace_localhost_hostname_for_docker,
+)
from openhands.app_server.utils.sql_utils import Base, UtcDateTime
from openhands.sdk.utils.paging import page_iterator
@@ -571,17 +576,49 @@ class RemoteSandboxService(SandboxService):
async def delete_sandbox(self, sandbox_id: str) -> bool:
"""Delete a sandbox by stopping its runtime.
- Security: Deleting the stored_sandbox record also removes the
- session_api_key_hash, invalidating any leaked session keys.
+ Purely sandbox-scoped: stop the runtime and delete the record. Workspace
+ capture is a separate conversation-scoped step
+ (``archive_conversation_workspace``) the conversation-delete finalizer runs
+ BEFORE tearing the sandbox down β€” so a long archive never blocks this call
+ (and the direct sandbox DELETE route can't 504 on it).
+
+ If the runtime is already gone (paused/reaped/double-delete, a 404 from
+ the runtime API), the record is deleted directly to avoid orphaning it.
+
+ Returns False ONLY when the sandbox does not exist (router -> 404). A
+ transient runtime /stop / lookup failure raises ``SandboxDeleteRetryError``
+ (router -> 503) and keeps the row + runtime for a retry β€” so a live sandbox
+ is never reported as 404.
+
+ Security: the session_api_key_hash is invalidated UP FRONT (like
+ ``pause_sandbox`` clears it before pausing) so a delete β€” commonly a
+ revoke of a leaked key β€” kills it promptly. This goes further than pause:
+ on a transient stop failure the invalidation is committed before raising,
+ so the caller's rollback cannot resurrect the just-revoked key (pause does
+ not commit, so its clear can still be rolled back). The row is kept for
+ retry.
"""
+ had_key = False
try:
stored_sandbox = await self._get_stored_sandbox(sandbox_id)
if not stored_sandbox:
return False
- # Deleting the record also removes the session_api_key_hash,
- # which invalidates any leaked session keys.
- await self.db_session.delete(stored_sandbox)
- runtime_data = await self._get_runtime(sandbox_id)
+ # Security: drop the key now, before the (fallible) runtime stop.
+ had_key = stored_sandbox.session_api_key_hash is not None
+ stored_sandbox.session_api_key_hash = None
+ try:
+ runtime_data = await self._get_runtime(sandbox_id)
+ except httpx.HTTPStatusError as e:
+ if e.response.status_code != 404:
+ raise
+ # Runtime already gone: nothing to stop. Delete the orphaned row.
+ _logger.info(
+ f'Runtime for sandbox {sandbox_id} already gone (404); '
+ 'deleting record'
+ )
+ await self.db_session.delete(stored_sandbox)
+ return True
+
response = await self._send_runtime_api_request(
'POST',
'/stop',
@@ -589,10 +626,153 @@ class RemoteSandboxService(SandboxService):
)
if response.status_code != 404:
response.raise_for_status()
+ await self.db_session.delete(stored_sandbox)
return True
except httpx.HTTPError as e:
+ # Transient runtime lookup/stop failure: keep the row + runtime and
+ # signal retryable (503) β€” never a 404. Persist the key invalidation
+ # now: the caller rolls back on this raise, which would otherwise
+ # restore the hash and leave a just-revoked key valid.
_logger.error(f'Error deleting sandbox {sandbox_id}: {e}')
- return False
+ if had_key:
+ await self.db_session.commit()
+ raise SandboxDeleteRetryError(
+ f'Could not complete delete for sandbox {sandbox_id}: {e}'
+ ) from e
+
+ async def _resolve_archive_path(
+ self,
+ stored_sandbox: StoredRemoteSandbox,
+ conversation_id: str | None,
+ workspace_path: str | None,
+ ) -> str:
+ """Path to archive: the value pinned at conversation creation if present,
+ else rebuilt from the SAME base the clone used (the sandbox spec's
+ ``working_dir``) plus the grouping nesting.
+
+ Pre-pinning conversations have no pinned path; the legacy fallback re-reads
+ the live grouping strategy, which can disagree with creation if the user
+ toggled it β€” but a resulting 404 no longer silently tears the sandbox down
+ under REQUIRED (it blocks for the idle reap). Raises if the layout cannot
+ be resolved, so the caller never archives to the wrong path.
+ """
+ if workspace_path:
+ return workspace_path
+ # For cloud conversations the sandbox id is the conversation_id.hex.
+ conversation_key = conversation_id or stored_sandbox.id
+ sandbox_spec = await self.sandbox_spec_service.get_sandbox_spec(
+ stored_sandbox.sandbox_spec_id
+ )
+ if sandbox_spec is None:
+ raise SandboxError(
+ f'No sandbox spec {stored_sandbox.sandbox_spec_id} for archive'
+ )
+ grouping = (await self.user_context.get_user_info()).sandbox_grouping_strategy
+ return grouped_workspace_dir(
+ sandbox_spec.working_dir, grouping, conversation_key
+ )
+
+ async def _archive_workspace(
+ self,
+ stored_sandbox: StoredRemoteSandbox,
+ conversation_id: str | None,
+ runtime_data: dict,
+ workspace_path: str | None,
+ ) -> bool:
+ """Archive one workspace via the in-pod agent-server; return may-proceed.
+
+ Returns True when the workspace was captured, when there was nothing to
+ capture, or when archiving failed but is not REQUIRED. Returns False only
+ when archiving is REQUIRED and could not confirm a capture (the caller
+ decides whether to block + retry). Never raises.
+ """
+ try:
+ archive_path = await self._resolve_archive_path(
+ stored_sandbox, conversation_id, workspace_path
+ )
+ # The runtime url is raw (localhost in Docker/local); transform it the
+ # same way every other agent-server URL resolution does.
+ runtime = dict(runtime_data)
+ url = runtime.get('url')
+ if url:
+ runtime['url'] = replace_localhost_hostname_for_docker(url)
+ return await workspace_archive.archive_workspace(
+ self.httpx_client,
+ runtime,
+ stored_sandbox.id,
+ archive_path=archive_path,
+ conversation_id=conversation_id,
+ )
+ except Exception:
+ # Could not resolve the workspace layout: never archive to the wrong
+ # path. Honor REQUIRED (block + retry) vs best-effort (proceed).
+ _logger.exception(
+ 'Could not resolve archive path for %s', stored_sandbox.id
+ )
+ return not workspace_archive.archive_required()
+
+ async def archive_conversation_workspace(
+ self,
+ sandbox_id: str,
+ conversation_id: str | None = None,
+ workspace_path: str | None = None,
+ ) -> bool:
+ """Archive ONE conversation's workspace; return whether delete may proceed.
+
+ The sole app-server capture path: the conversation-delete finalizer calls
+ this for every conversation delete (while the runtime is still up), then
+ tears the sandbox down only when this was its last conversation. Keying to
+ the conversation lets a grouped sandbox capture the right per-conversation
+ repo, and means no grouped conversation's work is lost when a sibling later
+ triggers the sandbox delete.
+
+ ``workspace_path`` is the path pinned at conversation creation; when given
+ the capture uses it verbatim instead of re-deriving the layout.
+
+ Returns True when the workspace was captured, when there was nothing to
+ capture (runtime already gone, or no repo at the path), or when archiving
+ failed but is not REQUIRED. Returns False only when archiving is REQUIRED
+ and could not confirm a capture, so the finalizer keeps the sandbox +
+ running runtime for the runtime-api idle reap (the durability backstop).
+ Never raises. No-op (returns True) unless archiving is enabled.
+ """
+ if not workspace_archive.archive_enabled():
+ return True
+ try:
+ stored_sandbox = await self._get_stored_sandbox(sandbox_id)
+ if not stored_sandbox:
+ return True
+ runtime_data = await self._get_runtime(sandbox_id)
+ except httpx.HTTPStatusError as e:
+ if e.response.status_code == 404:
+ # Runtime already gone: nothing to capture for this conversation.
+ return True
+ # Couldn't reach the runtime: honor REQUIRED (block + keep) vs
+ # best-effort (let the delete proceed; delete_sandbox re-checks).
+ _logger.exception(
+ 'Workspace archive lookup failed for %s (%s)',
+ sandbox_id,
+ conversation_id,
+ )
+ return not workspace_archive.archive_required()
+ except Exception:
+ _logger.exception(
+ 'Workspace archive lookup failed for %s (%s)',
+ sandbox_id,
+ conversation_id,
+ )
+ return not workspace_archive.archive_required()
+ archived = await self._archive_workspace(
+ stored_sandbox, conversation_id, runtime_data, workspace_path
+ )
+ if not archived:
+ _logger.warning(
+ 'Workspace archive required but failed for %s (%s); keeping the '
+ 'sandbox for the idle reap to capture',
+ sandbox_id,
+ conversation_id,
+ )
+ return archived
async def pause_old_sandboxes(self, max_num_sandboxes: int) -> list[str]:
"""Pause the oldest running sandboxes until at most max_num_sandboxes remain.
diff --git a/openhands/app_server/sandbox/sandbox_router.py b/openhands/app_server/sandbox/sandbox_router.py
index 7ad0fcd0d5..b4b42aea79 100644
--- a/openhands/app_server/sandbox/sandbox_router.py
+++ b/openhands/app_server/sandbox/sandbox_router.py
@@ -110,6 +110,9 @@ async def delete_sandbox(
sandbox_id: str,
sandbox_service: SandboxService = sandbox_service_dependency,
) -> Success:
+ # delete_sandbox is sandbox-scoped (stop + delete) and never archives, so this
+ # request handler can't block on a minutes-long capture. Workspace capture is
+ # owned by the conversation-delete finalizer and the runtime-api idle reaper.
exists = await sandbox_service.delete_sandbox(sandbox_id)
if not exists:
raise HTTPException(status.HTTP_404_NOT_FOUND)
diff --git a/openhands/app_server/sandbox/sandbox_service.py b/openhands/app_server/sandbox/sandbox_service.py
index e372c7a5a0..3fc0655865 100644
--- a/openhands/app_server/sandbox/sandbox_service.py
+++ b/openhands/app_server/sandbox/sandbox_service.py
@@ -197,8 +197,28 @@ class SandboxService(ABC):
async def delete_sandbox(self, sandbox_id: str) -> bool:
"""Begin the process of deleting a sandbox (which may involve stopping it).
- Return False if the sandbox did not exist.
+ Return False if the sandbox did not exist. Purely sandbox-scoped (stop the
+ runtime, delete the record); workspace capture is a separate
+ conversation-scoped step (``archive_conversation_workspace``) the
+ conversation-delete finalizer runs before the sandbox is torn down.
+ """
+
+ async def archive_conversation_workspace(
+ self,
+ sandbox_id: str,
+ conversation_id: str | None = None,
+ workspace_path: str | None = None,
+ ) -> bool:
+ """Archive one conversation's workspace; return whether delete may proceed.
+
+ Default no-op (returns True) for backends that do not archive; overridden
+ by RemoteSandboxService. The conversation-delete finalizer calls this
+ before ``delete_sandbox`` so the workspace is captured while the runtime is
+ still up. ``workspace_path`` is the path pinned at creation. Returns False
+ only when archiving is REQUIRED and failed, so the caller leaves the
+ sandbox up for a later (idle-reap) capture.
"""
+ return True
async def pause_old_sandboxes(self, max_num_sandboxes: int) -> list[str]:
"""Pause the oldest sandboxes if there are more than max_num_sandboxes running.
diff --git a/openhands/app_server/sandbox/workspace_archive.py b/openhands/app_server/sandbox/workspace_archive.py
new file mode 100644
index 0000000000..d091b20115
--- /dev/null
+++ b/openhands/app_server/sandbox/workspace_archive.py
@@ -0,0 +1,524 @@
+"""Archive a remote sandbox's workspace to object storage before deletion.
+
+Pulls a workspace archive from the in-pod agent-server endpoint
+(``GET /api/file/archive``) and stores it, plus a small manifest, in object
+storage so the agent's work β€” production OpenHands Cloud workspace state β€”
+survives sandbox deletion and is preserved for downstream use (e.g. dataset/eval
+creation).
+
+It covers the *explicit-delete-while-running* path. The dominant idle/expiry
+reap is handled separately in runtime-api at pause time, because that deletion
+never reaches the app-server.
+
+Configuration is environment-driven and the feature is a no-op unless
+``RUNTIME_FILE_ARCHIVE_ENABLED`` is set.
+"""
+
+import asyncio
+import json
+import logging
+import os
+import tempfile
+from typing import Any
+
+import httpx
+
+from openhands.agent_server.utils import utc_now
+from openhands.app_server.file_store import get_file_store
+from openhands.app_server.file_store.files import FileStore
+
+_logger = logging.getLogger(__name__)
+
+# Formats the SDK GET /api/file/archive producer accepts (git-delta | tar.gz);
+# anything else 422s, so validate before issuing the request.
+_ARCHIVE_SUFFIX = {'git-delta': 'patch', 'tar.gz': 'tar.gz'}
+
+
+def _archive_request_params(path: str, fmt: str) -> dict[str, str]:
+ """Query params for GET /api/file/archive.
+
+ The tar.gz is the SELF-CONTAINED full capture, so disable the endpoint's
+ default excludes for it: otherwise agent output under dist/build/node_modules
+ and the repo's .git history are dropped and it is no more complete than the
+ git-delta (defeating the whole point of capturing 'both'). Credential-bearing
+ git internals are still scrubbed server-side even with excludes off. git-delta
+ keeps the defaults β€” it is the compact companion, not the full capture.
+ """
+ params = {'path': path, 'format': fmt}
+ if fmt == 'tar.gz':
+ params['use_default_excludes'] = 'false'
+ return params
+
+
+def archive_enabled() -> bool:
+ return os.getenv('RUNTIME_FILE_ARCHIVE_ENABLED', 'false').lower() in ('true', '1')
+
+
+def archive_required() -> bool:
+ """When true, an archive failure blocks deletion so it can be retried."""
+ return os.getenv('RUNTIME_FILE_ARCHIVE_REQUIRED', 'false').lower() in ('true', '1')
+
+
+def _archive_bucket() -> str:
+ return os.getenv('RUNTIME_FILE_ARCHIVE_BUCKET', '')
+
+
+def _archive_prefix() -> str:
+ return os.getenv('RUNTIME_FILE_ARCHIVE_PREFIX', 'workspace-archives')
+
+
+def _archive_format() -> str:
+ # Default to 'both' β€” the compact git-delta AND a self-contained full tar.gz.
+ # git-delta alone is lossy as a sole capture: it respects the repo's
+ # .gitignore (so agent-authored gitignored files are dropped) and needs the
+ # base tree to reconstruct, whereas the tar.gz is self-contained and captures
+ # those files. Keep both until the storage cost is measured, then narrow to
+ # 'git-delta' (+ bucket lifecycle) if warranted (infra#1444).
+ return os.getenv('RUNTIME_FILE_ARCHIVE_FORMAT', 'both')
+
+
+def _formats_to_capture() -> list[str] | None:
+ """Resolve RUNTIME_FILE_ARCHIVE_FORMAT to the list of formats to upload.
+
+ 'both' captures the git-delta AND the full tar.gz; a single format captures
+ just that one. Returns None for an unsupported value (a hard config error the
+ SDK producer would 422), so the caller can log + skip instead of mis-reading
+ it as "nothing to archive".
+ """
+ fmt = _archive_format()
+ if fmt == 'both':
+ return ['git-delta', 'tar.gz']
+ if fmt in _ARCHIVE_SUFFIX:
+ return [fmt]
+ return None
+
+
+def _float_env(name: str, default: float) -> float:
+ """Parse a float env var, falling back to default on a non-numeric value.
+
+ A bad override (``'120s'``, a stray newline) must not raise on every archive
+ call β€” that would wedge every REQUIRED delete forever.
+ """
+ raw = os.getenv(name)
+ if not raw:
+ return default
+ try:
+ value = float(raw)
+ except ValueError:
+ _logger.warning('Invalid %s=%r; using %s', name, raw, default)
+ return default
+ if value <= 0:
+ # A non-positive timeout/deadline would make httpx raise on every archive
+ # (the wedge this guard exists to prevent); fall back to the safe default.
+ _logger.warning('Non-positive %s=%r; using %s', name, raw, default)
+ return default
+ return value
+
+
+def _archive_timeout() -> float:
+ # Must cover the agent-server git build budget (read-tree 60 + add 300 + diff
+ # 300 = up to ~660s) before the first response byte flows, or large repos
+ # ReadTimeout and never capture. The final archive runs in the detached
+ # delete finalizer, so a long wait here doesn't block any user request; the
+ # initial snapshot is separately bounded by initial_archive_deadline().
+ return _float_env('RUNTIME_FILE_ARCHIVE_TIMEOUT', 660.0)
+
+
+def _archive_store_type() -> str:
+ # Default to GCS to preserve current behavior; local/s3 also work. NOT
+ # 'memory' β€” it is text-only (read() returns str) and would corrupt the
+ # binary archive (see InMemoryFileStore).
+ return os.getenv('RUNTIME_FILE_ARCHIVE_STORE_TYPE', 'google_cloud')
+
+
+def _get_archive_file_store() -> FileStore:
+ """Object store for archives, built via the backend-portable factory."""
+ return get_file_store(_archive_store_type(), _archive_bucket())
+
+
+def _cleanup_tempfile(path: str | None) -> None:
+ if not path:
+ return
+ try:
+ os.unlink(path)
+ except OSError:
+ pass
+
+
+async def _stream_to_tempfile(response: Any) -> tuple[str, int]:
+ """Stream a 200 response body to a temp file; return (path, byte_count).
+
+ Avoids buffering the whole archive in app-server RAM (OOM risk under
+ concurrent large deletes). Cleans up its own file if streaming fails.
+ """
+ tmp = tempfile.NamedTemporaryFile(delete=False)
+ byte_count = 0
+ try:
+ async for chunk in response.aiter_bytes():
+ tmp.write(chunk)
+ byte_count += len(chunk)
+ tmp.close()
+ return tmp.name, byte_count
+ except BaseException:
+ tmp.close()
+ _cleanup_tempfile(tmp.name)
+ raise
+
+
+def _write_file_to_store(store: FileStore, name: str, path: str) -> None:
+ """Stream a temp file to the store, never buffering the whole archive in RAM.
+
+ The download was streamed to a tempfile precisely to avoid holding the
+ archive in memory (see ``_stream_to_tempfile``); ``write_from_path`` keeps
+ that guarantee on upload (GCS/local/S3 stream from disk) instead of reading
+ the whole file back with ``store.write(name, f.read())``.
+ """
+ store.write_from_path(name, path)
+
+
+async def archive_workspace(
+ httpx_client: httpx.AsyncClient,
+ runtime: dict[str, Any],
+ sandbox_id: str,
+ *,
+ archive_path: str,
+ conversation_id: str | None = None,
+) -> bool:
+ """Archive the workspace at ``archive_path``; return whether delete may proceed.
+
+ ``archive_path`` is resolved by the caller β€” the path pinned at conversation
+ creation, NOT re-derived here from live settings β€” so a capture can never be
+ misrouted to the wrong directory. The agent-server descends from it to the
+ cloned repo.
+
+ Returns True when the workspace was archived, when the path holds nothing to
+ archive (agent-server 400: not a directory / not a git repo), or when
+ archiving failed but is not required (best-effort). Returns False when
+ archiving is required and either hit a transient failure (5xx / network /
+ 422 / 429) or could not confirm a capture (401 auth / 404 missing path), so
+ the caller leaves the sandbox intact for the idle-reap retry. Never raises.
+
+ A pure configuration error (unsupported RUNTIME_FILE_ARCHIVE_FORMAT, or
+ RUNTIME_FILE_ARCHIVE_BUCKET unset) cannot be fixed by retrying, so it is
+ logged loudly and the delete is allowed to proceed rather than wedging every
+ delete forever when archiving is required.
+ """
+ agent_server_url = runtime.get('url')
+ session_api_key = runtime.get('session_api_key')
+ if not agent_server_url:
+ _logger.warning(
+ 'Workspace archive skipped for %s: runtime has no agent-server URL',
+ sandbox_id,
+ )
+ return not archive_required()
+ if not _archive_bucket():
+ # Misconfiguration, not a transient failure: no amount of retrying makes
+ # a missing bucket appear. Proceed (with a loud error) so a
+ # REQUIRED-without-bucket setup does not block every sandbox delete.
+ _logger.error(
+ 'Workspace archive enabled for %s but RUNTIME_FILE_ARCHIVE_BUCKET '
+ 'is not set; proceeding with delete (fix the config to capture)',
+ sandbox_id,
+ )
+ return True
+
+ formats = _formats_to_capture()
+ if formats is None:
+ # Unsupported RUNTIME_FILE_ARCHIVE_FORMAT is a pure config error, exactly
+ # like the unset bucket above: no retry makes a valid format appear, so
+ # proceed loudly rather than wedging every REQUIRED delete forever (the
+ # app-server has no idle-reap backstop). Validated here so a bad format
+ # never reaches the producer (which would 422 it).
+ _logger.error(
+ 'Workspace archive for %s: unsupported RUNTIME_FILE_ARCHIVE_FORMAT '
+ '%r (valid: %s); proceeding with delete (fix the config to capture)',
+ sandbox_id,
+ _archive_format(),
+ ['git-delta', 'tar.gz', 'both'],
+ )
+ return True
+
+ headers = {'X-Session-API-Key': session_api_key} if session_api_key else {}
+ # For cloud conversations the sandbox id is the conversation_id.hex.
+ conversation_key = conversation_id or sandbox_id
+ ts = utc_now().strftime('%Y%m%dT%H%M%SZ')
+ # Key by conversation, not just sandbox: under grouping a sandbox is shared by
+ # siblings, and the 1s ts is not unique β€” without the conversation segment two
+ # sibling captures in the same second overwrite each other at the object level.
+ base_path = f'{_archive_prefix()}/{sandbox_id}/{conversation_key}/{ts}'
+
+ # 'both' uploads each format under its own suffix ({ts}.patch + {ts}.tar.gz),
+ # each with its own manifest. base_commit only rides the git-delta response
+ # header, so capture it there and reuse it for the tar.gz manifest.
+ # A retry under REQUIRED re-uploads under a fresh {ts}; any blob/manifest left
+ # by a partially-failed prior attempt becomes an orphan reaped by the bucket
+ # lifecycle policy (we favor capture completeness over upload dedup).
+ retryable_failure = False
+ # A capture we could not confirm happened (401 auth / 404 missing path):
+ # under REQUIRED this must NOT permit teardown β€” it is the misrouted-path
+ # symptom this feature most needs to guard against, not "nothing to archive".
+ unconfirmed_capture = False
+ base_commit = ''
+ # One store per call (not per format) β€” building it lazily spins up a client.
+ store = _get_archive_file_store()
+ for fmt in formats:
+ suffix = _ARCHIVE_SUFFIX[fmt]
+ tmp_path: str | None = None
+ byte_count = 0
+ try:
+ async with httpx_client.stream(
+ 'GET',
+ f'{agent_server_url}/api/file/archive',
+ params=_archive_request_params(archive_path, fmt),
+ headers=headers,
+ timeout=_archive_timeout(),
+ ) as response:
+ if response.status_code != 200:
+ code = response.status_code
+ if code == 400:
+ # Path exists but holds no archivable repo (not a git
+ # repo / not a directory). A positive "nothing here" β€”
+ # safe to skip this format and proceed.
+ detail = 'nothing to archive'
+ elif code in (401, 404):
+ # 401 auth rejected / 404 path missing: the capture did
+ # NOT happen and this is not a confirmed-empty workspace,
+ # so it must block a REQUIRED delete (idle reap retries).
+ unconfirmed_capture = True
+ detail = 'capture unconfirmed (auth/path)'
+ else:
+ # 422 / 429 / 5xx β€” transient.
+ retryable_failure = True
+ detail = 'retryable failure'
+ _logger.warning(
+ 'Workspace archive (%s) for %s: agent-server returned %s; %s',
+ fmt,
+ sandbox_id,
+ code,
+ detail,
+ )
+ continue
+ header_base = response.headers.get('X-Archive-Base-Commit', '')
+ if header_base:
+ base_commit = header_base
+ # Stream to disk so the archive never sits whole in RAM.
+ tmp_path, byte_count = await _stream_to_tempfile(response)
+ except Exception as e:
+ # Network/timeout error: genuinely transient.
+ _logger.warning(
+ 'Workspace archive fetch (%s) failed for %s: %s', fmt, sandbox_id, e
+ )
+ retryable_failure = True
+ _cleanup_tempfile(tmp_path)
+ continue
+
+ assert tmp_path is not None # set on the 200 path above
+ try:
+ await asyncio.to_thread(
+ _write_file_to_store, store, f'{base_path}.{suffix}', tmp_path
+ )
+ manifest = json.dumps(
+ {
+ 'sandbox_id': sandbox_id,
+ 'conversation_id': conversation_key,
+ 'phase': 'final',
+ 'base_commit': base_commit,
+ 'format': fmt,
+ 'source_path': archive_path,
+ 'byte_count': byte_count,
+ 'created_at': ts,
+ },
+ sort_keys=True,
+ ).encode('utf-8')
+ await asyncio.to_thread(
+ store.write, f'{base_path}.{suffix}.manifest.json', manifest
+ )
+ _logger.info(
+ 'Archived workspace (%s) for %s (%d bytes) to %s.%s',
+ fmt,
+ sandbox_id,
+ byte_count,
+ base_path,
+ suffix,
+ )
+ except Exception as e:
+ _logger.exception(
+ 'Workspace archive upload (%s) failed for %s: %s', fmt, sandbox_id, e
+ )
+ retryable_failure = True
+ finally:
+ _cleanup_tempfile(tmp_path)
+
+ # Deletion may proceed unless archiving is REQUIRED and we either hit a
+ # retryable failure or could not confirm a capture (401/404) β€” both leave us
+ # short of the data we were meant to preserve.
+ if archive_required() and (retryable_failure or unconfirmed_capture):
+ return False
+ return True
+
+
+def initial_archive_enabled() -> bool:
+ """Whether to capture the workspace's INITIAL state (before the first step).
+
+ Independent of ``RUNTIME_FILE_ARCHIVE_ENABLED`` (the delete/pause capture of
+ the *final* state) so the pre-agent snapshot can be toggled on its own. Off
+ by default β€” like every other capture knob, nothing happens until enabled.
+ """
+ return os.getenv('RUNTIME_FILE_ARCHIVE_INITIAL_ENABLED', 'false').lower() in (
+ 'true',
+ '1',
+ )
+
+
+def initial_archive_deadline() -> float:
+ """Hard ceiling (seconds) on how long the inline, pre-setup initial snapshot
+ may delay conversation startup.
+
+ The snapshot is awaited before setup.sh so it captures the repo exactly as
+ cloned, with no concurrent mutation. ``wait_for`` only charges the time
+ actually spent, so a fast snapshot adds no latency; this caps the worst case
+ (a large repo or a hung endpoint) so it can never dominate startup. An overrun
+ is logged and startup proceeds without the snapshot. A fresh clone's tar.gz is
+ just the repo tree, so the default comfortably covers typical repos; raise it
+ for a very large monorepo.
+ """
+ return _float_env('RUNTIME_FILE_ARCHIVE_INITIAL_DEADLINE', 120.0)
+
+
+def _initial_archive_format() -> str:
+ """Format for the initial snapshot. Defaults to a self-contained tar.gz.
+
+ At conversation start the working tree has no changes yet, so a ``git-delta``
+ would be empty; a full ``tar.gz`` is the only format that captures anything
+ and, unlike a delta keyed to ``base_commit``, it survives the upstream repo
+ or branch later disappearing (the fragile re-clone path we want to avoid).
+ """
+ return os.getenv('RUNTIME_FILE_ARCHIVE_INITIAL_FORMAT', 'tar.gz')
+
+
+async def archive_initial_workspace(
+ httpx_client: httpx.AsyncClient,
+ *,
+ agent_server_url: str | None,
+ session_api_key: str | None,
+ project_dir: str,
+ sandbox_id: str,
+ conversation_id: str | None = None,
+ base_commit: str = '',
+) -> bool:
+ """Snapshot the workspace BEFORE the agent's first step; return success.
+
+ Captures the repo exactly as cloned (option A β€” the pre- vs post-setup choice
+ is the open design question tracked in All-Hands-AI/infra#1444) as a
+ self-contained ``tar.gz`` plus a ``phase=initial`` manifest, so the snapshot
+ records the true starting state even if the source repo later disappears.
+
+ This is strictly best-effort: it never raises and never blocks conversation
+ startup. A failure (feature off, misconfig, agent-server hiccup) just means no
+ initial snapshot for this run, logged and swallowed. Returns True only when an
+ archive was actually written.
+ """
+ if not initial_archive_enabled():
+ return False
+ if not agent_server_url:
+ _logger.warning(
+ 'Initial workspace archive skipped for %s: no agent-server URL',
+ sandbox_id,
+ )
+ return False
+ if not _archive_bucket():
+ _logger.error(
+ 'Initial workspace archive enabled for %s but '
+ 'RUNTIME_FILE_ARCHIVE_BUCKET is not set; skipping initial snapshot',
+ sandbox_id,
+ )
+ return False
+
+ fmt = _initial_archive_format()
+ if fmt not in _ARCHIVE_SUFFIX:
+ _logger.error(
+ 'Initial workspace archive for %s: unsupported '
+ 'RUNTIME_FILE_ARCHIVE_INITIAL_FORMAT %r (valid: %s); skipping',
+ sandbox_id,
+ fmt,
+ sorted(_ARCHIVE_SUFFIX),
+ )
+ return False
+ suffix = _ARCHIVE_SUFFIX[fmt]
+ headers = {'X-Session-API-Key': session_api_key} if session_api_key else {}
+
+ tmp_path: str | None = None
+ byte_count = 0
+ try:
+ async with httpx_client.stream(
+ 'GET',
+ f'{agent_server_url}/api/file/archive',
+ params=_archive_request_params(project_dir, fmt),
+ headers=headers,
+ timeout=_archive_timeout(),
+ ) as response:
+ if response.status_code != 200:
+ _logger.warning(
+ 'Initial workspace archive for %s: agent-server returned %s; '
+ 'no initial snapshot',
+ sandbox_id,
+ response.status_code,
+ )
+ return False
+ # tar.gz carries no base-commit header (git-delta sets it); fall back
+ # to the caller-provided HEAD sha so the snapshot still records the
+ # commit it came from.
+ captured_base = (
+ response.headers.get('X-Archive-Base-Commit', '') or base_commit
+ )
+ # Stream to disk so the archive never sits whole in RAM.
+ tmp_path, byte_count = await _stream_to_tempfile(response)
+ except Exception as e:
+ _logger.warning(
+ 'Initial workspace archive fetch failed for %s: %s', sandbox_id, e
+ )
+ _cleanup_tempfile(tmp_path)
+ return False
+
+ assert tmp_path is not None # set on the 200 path above
+ try:
+ store = _get_archive_file_store()
+ ts = utc_now().strftime('%Y%m%dT%H%M%SZ')
+ conversation_key = conversation_id or sandbox_id
+ # Key by conversation (siblings share a grouped sandbox) and nest under
+ # /initial/ so it never collides with that conversation's final capture
+ # ({prefix}/{sandbox_id}/{conversation_key}/{ts}).
+ blob_name = (
+ f'{_archive_prefix()}/{sandbox_id}/{conversation_key}/initial/{ts}.{suffix}'
+ )
+ await asyncio.to_thread(_write_file_to_store, store, blob_name, tmp_path)
+ manifest = json.dumps(
+ {
+ 'sandbox_id': sandbox_id,
+ 'conversation_id': conversation_key,
+ 'phase': 'initial',
+ 'base_commit': captured_base,
+ 'format': fmt,
+ 'source_path': project_dir,
+ 'byte_count': byte_count,
+ 'created_at': ts,
+ },
+ sort_keys=True,
+ ).encode('utf-8')
+ # Shared contract: manifest = blob + '.manifest.json' (was dropping the
+ # format suffix, so a downstream enricher could never locate it).
+ await asyncio.to_thread(store.write, f'{blob_name}.manifest.json', manifest)
+ _logger.info(
+ 'Archived INITIAL workspace for %s (%d bytes) to %s',
+ sandbox_id,
+ byte_count,
+ blob_name,
+ )
+ return True
+ except Exception as e:
+ _logger.exception(
+ 'Initial workspace archive upload failed for %s: %s', sandbox_id, e
+ )
+ return False
+ finally:
+ _cleanup_tempfile(tmp_path)
diff --git a/openhands/app_server/settings/settings_models.py b/openhands/app_server/settings/settings_models.py
index 82cc77ba43..6ccc3e3490 100644
--- a/openhands/app_server/settings/settings_models.py
+++ b/openhands/app_server/settings/settings_models.py
@@ -92,6 +92,23 @@ class SandboxGroupingStrategy(str, Enum):
ADD_TO_ANY = 'ADD_TO_ANY' # Add to any available sandbox (first found)
+def grouped_workspace_dir(
+ base_working_dir: str,
+ grouping_strategy: SandboxGroupingStrategy,
+ conversation_id_hex: str,
+) -> str:
+ """Workspace dir for a conversation given the grouping strategy.
+
+ Single source of truth for the relocation used at conversation start and at
+ archive time. Under any grouping strategy the workspace is nested under the
+ conversation id so co-located conversations stay isolated; NO_GROUPING keeps
+ the bare base dir.
+ """
+ if grouping_strategy == SandboxGroupingStrategy.NO_GROUPING:
+ return base_working_dir
+ return f'{base_working_dir}/{conversation_id_hex}'
+
+
# Fields the batch ``update()`` method refuses to touch:
# - ``secrets_store`` is frozen (Pydantic would raise).
# - ``llm_profiles`` is off-limits for the generic settings POST; profile
diff --git a/tests/unit/app_server/test_app_conversation_router.py b/tests/unit/app_server/test_app_conversation_router.py
index e8a41c2dc0..0f34843bbe 100644
--- a/tests/unit/app_server/test_app_conversation_router.py
+++ b/tests/unit/app_server/test_app_conversation_router.py
@@ -21,6 +21,7 @@ from openhands.app_server.app_conversation.app_conversation_models import (
)
from openhands.app_server.app_conversation.app_conversation_router import (
AgentServerContext,
+ _finalize_sandbox_delete,
batch_get_app_conversations,
count_app_conversations,
get_conversation_git_changes,
@@ -1082,3 +1083,88 @@ class TestGitProxyEndpoints:
assert exc_info.value.status_code == status.HTTP_409_CONFLICT
assert 'paused' in exc_info.value.detail.lower()
+
+
+class TestFinalizeSandboxDelete:
+ """The detached finalizer captures THIS conversation's workspace first
+ (archive_conversation_workspace, while the runtime is still up), then tears the
+ sandbox down only when no other conversation still references it (count == 0).
+ delete_sandbox is sandbox-scoped β€” no conversation_id. A REQUIRED archive
+ failure (archive returns False) keeps the sandbox for the idle reap."""
+
+ def _deps(self, conversation_count: int, archived: bool = True):
+ info_service = MagicMock()
+ info_service.count_conversations_by_sandbox_id = AsyncMock(
+ return_value=conversation_count
+ )
+ sandbox_service = MagicMock()
+ sandbox_service.delete_sandbox = AsyncMock(return_value=True)
+ sandbox_service.archive_conversation_workspace = AsyncMock(
+ return_value=archived
+ )
+ db_session = AsyncMock()
+ httpx_client = AsyncMock()
+ return sandbox_service, info_service, db_session, httpx_client
+
+ @pytest.mark.asyncio
+ async def test_archives_then_deletes_when_unreferenced(self):
+ sandbox_service, info_service, db_session, httpx_client = self._deps(0)
+ conv = uuid4()
+ await _finalize_sandbox_delete(
+ sandbox_service, info_service, 'sbx-1', db_session, httpx_client, conv
+ )
+ # Workspace captured first (conversation-scoped), then the sandbox torn
+ # down sandbox-scoped β€” note delete_sandbox takes no conversation_id.
+ sandbox_service.archive_conversation_workspace.assert_awaited_once_with(
+ 'sbx-1', conversation_id=conv.hex, workspace_path=None
+ )
+ sandbox_service.delete_sandbox.assert_awaited_once_with('sbx-1')
+ db_session.commit.assert_awaited_once()
+
+ @pytest.mark.asyncio
+ async def test_archives_but_keeps_shared_sandbox(self):
+ sandbox_service, info_service, db_session, httpx_client = self._deps(2)
+ conv = uuid4()
+ await _finalize_sandbox_delete(
+ sandbox_service, info_service, 'sbx-1', db_session, httpx_client, conv
+ )
+ # Shared sandbox: still capture this conversation, but don't tear it down.
+ sandbox_service.archive_conversation_workspace.assert_awaited_once_with(
+ 'sbx-1', conversation_id=conv.hex, workspace_path=None
+ )
+ sandbox_service.delete_sandbox.assert_not_called()
+
+ @pytest.mark.asyncio
+ async def test_required_archive_failure_keeps_sandbox(self):
+ # archive_conversation_workspace returns False (REQUIRED archive failed).
+ sandbox_service, info_service, db_session, httpx_client = self._deps(
+ 0, archived=False
+ )
+ conv = uuid4()
+ await _finalize_sandbox_delete(
+ sandbox_service, info_service, 'sbx-1', db_session, httpx_client, conv
+ )
+ # Sandbox + runtime kept for the idle reap: no count check, no delete.
+ sandbox_service.archive_conversation_workspace.assert_awaited_once()
+ info_service.count_conversations_by_sandbox_id.assert_not_called()
+ sandbox_service.delete_sandbox.assert_not_called()
+
+ @pytest.mark.asyncio
+ async def test_forwards_pinned_workspace_path(self):
+ sandbox_service, info_service, db_session, httpx_client = self._deps(0)
+ conv = uuid4()
+ await _finalize_sandbox_delete(
+ sandbox_service,
+ info_service,
+ 'sbx-1',
+ db_session,
+ httpx_client,
+ conv,
+ workspace_path='/home/openhands/workspace/' + conv.hex,
+ )
+ # The path pinned at creation is forwarded to the capture verbatim.
+ sandbox_service.archive_conversation_workspace.assert_awaited_once_with(
+ 'sbx-1',
+ conversation_id=conv.hex,
+ workspace_path='/home/openhands/workspace/' + conv.hex,
+ )
diff --git a/tests/unit/app_server/test_live_status_app_conversation_service.py b/tests/unit/app_server/test_live_status_app_conversation_service.py
index e7779887b9..cd55b843bd 100644
--- a/tests/unit/app_server/test_live_status_app_conversation_service.py
+++ b/tests/unit/app_server/test_live_status_app_conversation_service.py
@@ -1893,7 +1893,9 @@ class TestLiveStatusAppConversationService:
task.sandbox_id = self.mock_sandbox.id
yield task
- async def mock_run_setup_scripts(task, sandbox, workspace, agent_server_url):
+ async def mock_run_setup_scripts(
+ task, sandbox, workspace, agent_server_url, conversation_id
+ ):
yield task
self.service._wait_for_sandbox_start = mock_wait_for_sandbox
@@ -1997,7 +1999,9 @@ class TestLiveStatusAppConversationService:
task.sandbox_id = self.mock_sandbox.id
yield task
- async def mock_run_setup_scripts(task, sandbox, workspace, agent_server_url):
+ async def mock_run_setup_scripts(
+ task, sandbox, workspace, agent_server_url, conversation_id
+ ):
yield task
self.service._wait_for_sandbox_start = mock_wait_for_sandbox
diff --git a/tests/unit/app_server/test_remote_sandbox_service.py b/tests/unit/app_server/test_remote_sandbox_service.py
index 6e538677d4..209609764f 100644
--- a/tests/unit/app_server/test_remote_sandbox_service.py
+++ b/tests/unit/app_server/test_remote_sandbox_service.py
@@ -12,6 +12,7 @@ This module tests the RemoteSandboxService implementation, focusing on:
"""
import asyncio
+import json
from contextlib import ExitStack, asynccontextmanager
from datetime import datetime, timezone
from typing import Any
@@ -22,7 +23,7 @@ import httpx
import pytest
from sqlalchemy.ext.asyncio import AsyncSession
-from openhands.app_server.errors import SandboxError
+from openhands.app_server.errors import SandboxDeleteRetryError, SandboxError
from openhands.app_server.sandbox.remote_sandbox_service import (
ALLOW_CORS_ORIGINS_VARIABLE,
STATUS_MAPPING,
@@ -39,6 +40,7 @@ from openhands.app_server.sandbox.sandbox_models import (
SandboxStatus,
)
from openhands.app_server.sandbox.sandbox_spec_models import SandboxSpecInfo
+from openhands.app_server.settings.settings_models import SandboxGroupingStrategy
from openhands.app_server.user.user_context import UserContext
@@ -97,6 +99,39 @@ def remote_sandbox_service(
)
+def _make_stream_response(
+ status_code: int, content: bytes = b'', headers: dict | None = None
+):
+ """A fake httpx streaming response: status/headers + async aiter_bytes()."""
+ resp = MagicMock()
+ resp.status_code = status_code
+ resp.headers = headers or {}
+
+ async def _aiter_bytes():
+ yield content
+
+ resp.aiter_bytes = _aiter_bytes
+ return resp
+
+
+def _stream_client(resp_or_map):
+ """AsyncMock httpx client whose .stream(...) yields a fake response.
+
+ ``resp_or_map`` is a single fake response or a ``{format: response}`` map.
+ """
+ client = AsyncMock()
+
+ @asynccontextmanager
+ async def _stream(method, url, **kwargs):
+ if isinstance(resp_or_map, dict):
+ yield resp_or_map[kwargs['params']['format']]
+ else:
+ yield resp_or_map
+
+ client.stream = MagicMock(side_effect=_stream)
+ return client
+
+
def create_runtime_data(
session_id: str = 'test-sandbox-123',
status: str = 'running',
@@ -633,7 +668,9 @@ class TestSandboxLifecycle:
# Verify
assert result is True
remote_sandbox_service.db_session.delete.assert_called_once_with(stored_sandbox)
- remote_sandbox_service.db_session.commit.assert_not_called()
+ # delete_sandbox no longer commits internally: the session key is dropped
+ # atomically with the row delete, which the caller commits.
+ remote_sandbox_service.db_session.commit.assert_not_awaited()
remote_sandbox_service.httpx_client.request.assert_called_once_with(
'POST',
'https://api.example.com/stop',
@@ -641,6 +678,52 @@ class TestSandboxLifecycle:
json={'runtime_id': 'runtime-456'},
)
+ @pytest.mark.asyncio
+ async def test_archive_conversation_workspace_archives_without_stopping(
+ self, remote_sandbox_service
+ ):
+ """A grouped, non-terminal conversation's workspace is captured without
+ stopping the shared sandbox (no /stop, no row delete)."""
+ stored_sandbox = create_stored_sandbox()
+ runtime_data = create_runtime_data()
+ remote_sandbox_service._get_stored_sandbox = AsyncMock(
+ return_value=stored_sandbox
+ )
+ remote_sandbox_service._get_runtime = AsyncMock(return_value=runtime_data)
+ remote_sandbox_service.db_session.delete = AsyncMock()
+ remote_sandbox_service.httpx_client.request = AsyncMock()
+
+ with patch(
+ 'openhands.app_server.sandbox.remote_sandbox_service.workspace_archive'
+ ) as mock_wa:
+ mock_wa.archive_enabled.return_value = True
+ mock_wa.archive_workspace = AsyncMock(return_value=True)
+ ok = await remote_sandbox_service.archive_conversation_workspace(
+ 'test-sandbox-123', conversation_id='conv-1'
+ )
+
+ assert ok is True
+ mock_wa.archive_workspace.assert_awaited_once()
+ # Captured, but the shared sandbox was neither stopped nor deleted.
+ remote_sandbox_service.httpx_client.request.assert_not_called()
+ remote_sandbox_service.db_session.delete.assert_not_called()
+
+ @pytest.mark.asyncio
+ async def test_archive_conversation_workspace_noop_when_disabled(
+ self, remote_sandbox_service
+ ):
+ """No-op (returns True) when archiving is disabled β€” no lookups at all."""
+ remote_sandbox_service._get_stored_sandbox = AsyncMock()
+ with patch(
+ 'openhands.app_server.sandbox.remote_sandbox_service.workspace_archive'
+ ) as mock_wa:
+ mock_wa.archive_enabled.return_value = False
+ ok = await remote_sandbox_service.archive_conversation_workspace(
+ 'test-sandbox-123', conversation_id='conv-1'
+ )
+ assert ok is True
+ remote_sandbox_service._get_stored_sandbox.assert_not_called()
+
@pytest.mark.asyncio
async def test_delete_sandbox_runtime_not_found_ignored(
self, remote_sandbox_service
@@ -667,6 +750,87 @@ class TestSandboxLifecycle:
# Verify
assert result is True # 404 should be ignored for delete operations
+ @pytest.mark.asyncio
+ async def test_delete_sandbox_runtime_gone_deletes_record(
+ self, remote_sandbox_service
+ ):
+ """_get_runtime 404 (runtime already gone) still deletes the DB row."""
+ stored_sandbox = create_stored_sandbox()
+
+ remote_sandbox_service._get_stored_sandbox = AsyncMock(
+ return_value=stored_sandbox
+ )
+ not_found = httpx.HTTPStatusError(
+ 'Not Found',
+ request=httpx.Request('GET', 'https://api.example.com/sessions/x'),
+ response=httpx.Response(404),
+ )
+ remote_sandbox_service._get_runtime = AsyncMock(side_effect=not_found)
+ remote_sandbox_service.db_session.delete = AsyncMock()
+
+ result = await remote_sandbox_service.delete_sandbox('test-sandbox-123')
+
+ assert result is True
+ remote_sandbox_service.db_session.delete.assert_called_once_with(stored_sandbox)
+ # Nothing to stop when the runtime is already gone.
+ remote_sandbox_service.httpx_client.request.assert_not_called()
+
+ @pytest.mark.asyncio
+ async def test_delete_sandbox_runtime_transient_error_keeps_record(
+ self, remote_sandbox_service
+ ):
+ """A transient (5xx) _get_runtime error leaves the row for a retry."""
+ stored_sandbox = create_stored_sandbox()
+
+ remote_sandbox_service._get_stored_sandbox = AsyncMock(
+ return_value=stored_sandbox
+ )
+ server_error = httpx.HTTPStatusError(
+ 'Server Error',
+ request=httpx.Request('GET', 'https://api.example.com/sessions/x'),
+ response=httpx.Response(503),
+ )
+ remote_sandbox_service._get_runtime = AsyncMock(side_effect=server_error)
+ remote_sandbox_service.db_session.delete = AsyncMock()
+
+ # Sandbox still exists -> raise (router 503), keep the row for retry;
+ # never report 404.
+ with pytest.raises(SandboxDeleteRetryError):
+ await remote_sandbox_service.delete_sandbox('test-sandbox-123')
+ remote_sandbox_service.db_session.delete.assert_not_called()
+
+ @pytest.mark.asyncio
+ async def test_delete_sandbox_transient_error_invalidates_session_key(
+ self, remote_sandbox_service
+ ):
+ """A transient runtime error keeps the row + running runtime for retry, but
+ the session key is invalidated UP FRONT and committed β€” a delete is often a
+ revoke of a leaked key, so the key must die promptly and the caller's
+ rollback on the raised error must not be able to resurrect it."""
+ stored_sandbox = create_stored_sandbox(session_api_key_hash='live-hash')
+
+ remote_sandbox_service._get_stored_sandbox = AsyncMock(
+ return_value=stored_sandbox
+ )
+ remote_sandbox_service._get_runtime = AsyncMock(
+ side_effect=httpx.HTTPStatusError(
+ 'Server Error',
+ request=httpx.Request('GET', 'https://api.example.com/sessions/x'),
+ response=httpx.Response(503),
+ )
+ )
+ remote_sandbox_service.db_session.delete = AsyncMock()
+ remote_sandbox_service.db_session.commit = AsyncMock()
+
+ with pytest.raises(SandboxDeleteRetryError):
+ await remote_sandbox_service.delete_sandbox('test-sandbox-123')
+
+ # Row kept for retry, but the key is dead and that has been persisted so a
+ # rollback cannot bring it back.
+ remote_sandbox_service.db_session.delete.assert_not_called()
+ assert stored_sandbox.session_api_key_hash is None
+ remote_sandbox_service.db_session.commit.assert_awaited_once()
+
class TestSandboxSearch:
"""Test cases for sandbox search and retrieval."""
@@ -981,11 +1145,9 @@ class TestErrorHandling:
'API Error'
)
- # Execute
- result = await remote_sandbox_service.delete_sandbox('test-sandbox-123')
-
- # Verify
- assert result is False
+ # A transient runtime error on a live sandbox raises (router 503), not 404.
+ with pytest.raises(SandboxDeleteRetryError):
+ await remote_sandbox_service.delete_sandbox('test-sandbox-123')
class TestGetSandboxBySessionApiKey:
@@ -1768,3 +1930,985 @@ class TestBatchGetSandboxes:
assert results[0] is not None
assert results[0].id == 'sandbox-1'
assert results[0].status == SandboxStatus.MISSING
+
+
+class TestArchiveConversationWorkspace:
+ """archive_conversation_workspace contract (APP-2403): the sole app-server
+ capture path, run by the conversation-delete finalizer BEFORE delete_sandbox
+ (which is now purely sandbox-scoped). Its bool return gates the finalizer β€”
+ False (REQUIRED + retryable failure) keeps the sandbox for the idle reap; True
+ lets the delete proceed. It never stops the runtime or deletes the row."""
+
+ @pytest.mark.asyncio
+ async def test_required_failure_returns_false(self, remote_sandbox_service):
+ """A REQUIRED archive failure returns False (and never stops/deletes), so
+ the finalizer keeps the sandbox + runtime for the idle reap."""
+ stored = create_stored_sandbox()
+ remote_sandbox_service._get_stored_sandbox = AsyncMock(return_value=stored)
+ remote_sandbox_service._get_runtime = AsyncMock(
+ return_value=create_runtime_data()
+ )
+ remote_sandbox_service.db_session.delete = AsyncMock()
+ remote_sandbox_service.httpx_client.request = AsyncMock()
+
+ with (
+ patch(
+ 'openhands.app_server.sandbox.workspace_archive.archive_enabled',
+ return_value=True,
+ ),
+ patch(
+ 'openhands.app_server.sandbox.workspace_archive.archive_workspace',
+ new=AsyncMock(return_value=False),
+ ),
+ ):
+ ok = await remote_sandbox_service.archive_conversation_workspace(
+ 'test-sandbox-123', conversation_id='conv-1'
+ )
+
+ assert ok is False
+ remote_sandbox_service.httpx_client.request.assert_not_called()
+ remote_sandbox_service.db_session.delete.assert_not_called()
+
+ @pytest.mark.asyncio
+ async def test_nothing_to_archive_returns_true(self, remote_sandbox_service):
+ """Nothing to capture (archive_workspace returns True for a 400/404) lets
+ the delete proceed even under REQUIRED, so the sandbox is never orphaned."""
+ stored = create_stored_sandbox()
+ remote_sandbox_service._get_stored_sandbox = AsyncMock(return_value=stored)
+ remote_sandbox_service._get_runtime = AsyncMock(
+ return_value=create_runtime_data()
+ )
+
+ with (
+ patch(
+ 'openhands.app_server.sandbox.workspace_archive.archive_enabled',
+ return_value=True,
+ ),
+ patch(
+ 'openhands.app_server.sandbox.workspace_archive.archive_workspace',
+ new=AsyncMock(return_value=True),
+ ),
+ ):
+ ok = await remote_sandbox_service.archive_conversation_workspace(
+ 'test-sandbox-123', conversation_id='conv-1'
+ )
+
+ assert ok is True
+
+ @pytest.mark.asyncio
+ async def test_runtime_already_gone_returns_true(self, remote_sandbox_service):
+ """Runtime already gone (404): nothing to capture, so the delete proceeds."""
+ stored = create_stored_sandbox()
+ remote_sandbox_service._get_stored_sandbox = AsyncMock(return_value=stored)
+ remote_sandbox_service._get_runtime = AsyncMock(
+ side_effect=httpx.HTTPStatusError(
+ 'gone',
+ request=httpx.Request('GET', 'https://api.example.com/x'),
+ response=httpx.Response(404),
+ )
+ )
+ with patch(
+ 'openhands.app_server.sandbox.workspace_archive.archive_enabled',
+ return_value=True,
+ ):
+ ok = await remote_sandbox_service.archive_conversation_workspace(
+ 'test-sandbox-123', conversation_id='conv-1'
+ )
+ assert ok is True
+
+ @pytest.mark.asyncio
+ async def test_pinned_path_forwarded_to_archive(self, remote_sandbox_service):
+ """A pinned workspace_path is forwarded to archive_workspace verbatim and
+ no grouping/spec re-derivation happens."""
+ stored = create_stored_sandbox()
+ remote_sandbox_service._get_stored_sandbox = AsyncMock(return_value=stored)
+ remote_sandbox_service._get_runtime = AsyncMock(
+ return_value=create_runtime_data()
+ )
+ with (
+ patch(
+ 'openhands.app_server.sandbox.workspace_archive.archive_enabled',
+ return_value=True,
+ ),
+ patch(
+ 'openhands.app_server.sandbox.workspace_archive.archive_workspace',
+ new=AsyncMock(return_value=True),
+ ) as mock_archive,
+ ):
+ ok = await remote_sandbox_service.archive_conversation_workspace(
+ 'test-sandbox-123',
+ conversation_id='conv-1',
+ workspace_path='/home/openhands/workspace/conv-1',
+ )
+ assert ok is True
+ _, kwargs = mock_archive.call_args
+ assert kwargs['archive_path'] == '/home/openhands/workspace/conv-1'
+ remote_sandbox_service.sandbox_spec_service.get_sandbox_spec.assert_not_called()
+
+ @pytest.mark.asyncio
+ async def test_resolve_archive_path_uses_pinned(self, remote_sandbox_service):
+ """A pinned path is returned verbatim β€” no spec/settings lookup, so a
+ grouping toggle since creation can never misroute the capture."""
+ stored = create_stored_sandbox()
+ path = await remote_sandbox_service._resolve_archive_path(
+ stored, 'conv-1', '/home/openhands/workspace/conv-1'
+ )
+ assert path == '/home/openhands/workspace/conv-1'
+ remote_sandbox_service.sandbox_spec_service.get_sandbox_spec.assert_not_called()
+
+ @pytest.mark.asyncio
+ async def test_resolve_archive_path_legacy_grouped(self, remote_sandbox_service):
+ """No pinned path (pre-pinning conversation): rebuilt from the sandbox
+ spec's working_dir nested under the conversation id."""
+ stored = create_stored_sandbox()
+ remote_sandbox_service.user_context.get_user_info = AsyncMock(
+ return_value=MagicMock(
+ sandbox_grouping_strategy=SandboxGroupingStrategy.GROUP_BY_NEWEST
+ )
+ )
+ path = await remote_sandbox_service._resolve_archive_path(
+ stored, 'deadbeef', None
+ )
+ # mock_sandbox_spec_service fixture pins working_dir='/workspace/project'.
+ assert path == '/workspace/project/deadbeef'
+
+ @pytest.mark.asyncio
+ async def test_resolve_archive_path_legacy_no_grouping(
+ self, remote_sandbox_service
+ ):
+ """NO_GROUPING fallback keeps the bare spec working_dir."""
+ stored = create_stored_sandbox()
+ remote_sandbox_service.user_context.get_user_info = AsyncMock(
+ return_value=MagicMock(
+ sandbox_grouping_strategy=SandboxGroupingStrategy.NO_GROUPING
+ )
+ )
+ path = await remote_sandbox_service._resolve_archive_path(
+ stored, 'deadbeef', None
+ )
+ assert path == '/workspace/project'
+
+
+class TestArchiveWorkspaceHelper:
+ """Unit tests for the workspace_archive helper."""
+
+ @pytest.mark.asyncio
+ async def test_archive_uploads_archive_and_manifest(self, monkeypatch):
+ from openhands.app_server.sandbox import workspace_archive
+
+ monkeypatch.setenv('RUNTIME_FILE_ARCHIVE_BUCKET', 'archive-bkt')
+ monkeypatch.setenv('RUNTIME_FILE_ARCHIVE_FORMAT', 'git-delta')
+
+ client = _stream_client(
+ _make_stream_response(
+ 200, b'patch-bytes', {'X-Archive-Base-Commit': 'abc123'}
+ )
+ )
+ store = MagicMock()
+ writes: dict[str, bytes] = {}
+ store.write.side_effect = lambda path, data: writes.__setitem__(path, data)
+ # Archive blobs are streamed from a tempfile via write_from_path (the OOM
+ # fix); record them too so blob-path assertions still see the .patch/.tar.gz.
+ store.write_from_path.side_effect = lambda path, src: writes.__setitem__(
+ path, open(src, 'rb').read()
+ )
+
+ with patch.object(
+ workspace_archive, '_get_archive_file_store', return_value=store
+ ):
+ ok = await workspace_archive.archive_workspace(
+ client,
+ create_runtime_data(),
+ 'sandbox-1',
+ archive_path='/workspace/project',
+ conversation_id='conv-1',
+ )
+
+ assert ok is True
+ # session key forwarded; the given archive_path is used, format passed.
+ _, kwargs = client.stream.call_args
+ assert kwargs['headers']['X-Session-API-Key'] == 'test-session-key'
+ assert kwargs['params'] == {
+ 'path': '/workspace/project',
+ 'format': 'git-delta',
+ }
+ # archive + manifest written.
+ assert any(p.endswith('.patch') for p in writes)
+ manifest_path = next(p for p in writes if p.endswith('.manifest.json'))
+ manifest = json.loads(writes[manifest_path])
+ assert manifest['base_commit'] == 'abc123'
+ assert manifest['conversation_id'] == 'conv-1'
+ assert manifest['source_path'] == '/workspace/project'
+
+ @pytest.mark.asyncio
+ async def test_archive_missing_base_commit_header_defaults_empty(self, monkeypatch):
+ from openhands.app_server.sandbox import workspace_archive
+
+ monkeypatch.setenv('RUNTIME_FILE_ARCHIVE_BUCKET', 'archive-bkt')
+ monkeypatch.setenv('RUNTIME_FILE_ARCHIVE_FORMAT', 'git-delta')
+
+ client = _stream_client(_make_stream_response(200, b'patch-bytes', {}))
+ store = MagicMock()
+ writes: dict[str, bytes] = {}
+ store.write.side_effect = lambda path, data: writes.__setitem__(path, data)
+ # Archive blobs are streamed from a tempfile via write_from_path (the OOM
+ # fix); record them too so blob-path assertions still see the .patch/.tar.gz.
+ store.write_from_path.side_effect = lambda path, src: writes.__setitem__(
+ path, open(src, 'rb').read()
+ )
+
+ with patch.object(
+ workspace_archive, '_get_archive_file_store', return_value=store
+ ):
+ ok = await workspace_archive.archive_workspace(
+ client,
+ create_runtime_data(),
+ 'sandbox-1',
+ archive_path='/workspace/project',
+ )
+
+ assert ok is True
+ manifest_path = next(p for p in writes if p.endswith('.manifest.json'))
+ manifest = json.loads(writes[manifest_path])
+ assert manifest['base_commit'] == ''
+ # Prod case: delete_sandbox passes no conversation_id, so the manifest
+ # falls back to the sandbox id (which is the conversation_id.hex).
+ assert manifest['conversation_id'] == 'sandbox-1'
+
+ @pytest.mark.asyncio
+ async def test_archive_non_200_not_required_allows_delete(self, monkeypatch):
+ from openhands.app_server.sandbox import workspace_archive
+
+ monkeypatch.setenv('RUNTIME_FILE_ARCHIVE_BUCKET', 'archive-bkt')
+ monkeypatch.delenv('RUNTIME_FILE_ARCHIVE_REQUIRED', raising=False)
+
+ client = _stream_client(_make_stream_response(500))
+
+ ok = await workspace_archive.archive_workspace(
+ client,
+ create_runtime_data(),
+ 'sandbox-1',
+ archive_path='/workspace/project',
+ )
+ # Not required -> best-effort -> deletion may proceed.
+ assert ok is True
+
+ @pytest.mark.asyncio
+ async def test_archive_non_200_required_blocks_delete(self, monkeypatch):
+ from openhands.app_server.sandbox import workspace_archive
+
+ monkeypatch.setenv('RUNTIME_FILE_ARCHIVE_BUCKET', 'archive-bkt')
+ monkeypatch.setenv('RUNTIME_FILE_ARCHIVE_REQUIRED', 'true')
+
+ client = _stream_client(_make_stream_response(500))
+
+ ok = await workspace_archive.archive_workspace(
+ client,
+ create_runtime_data(),
+ 'sandbox-1',
+ archive_path='/workspace/project',
+ )
+ # Required -> a failure blocks deletion.
+ assert ok is False
+
+ @pytest.mark.asyncio
+ async def test_archive_no_url_skips(self, monkeypatch):
+ from openhands.app_server.sandbox import workspace_archive
+
+ monkeypatch.setenv('RUNTIME_FILE_ARCHIVE_BUCKET', 'archive-bkt')
+ monkeypatch.delenv('RUNTIME_FILE_ARCHIVE_REQUIRED', raising=False)
+ client = AsyncMock()
+
+ ok = await workspace_archive.archive_workspace(
+ client,
+ create_runtime_data(url=''),
+ 'sandbox-1',
+ archive_path='/workspace/project',
+ )
+ assert ok is True
+ client.stream.assert_not_called()
+
+ @pytest.mark.asyncio
+ async def test_archive_400_no_repo_required_allows_delete(self, monkeypatch):
+ """No git repo to capture is a permanent 400 from the agent-server, not a
+ failure: even when archiving is required, the delete may proceed (BUG 2)."""
+ from openhands.app_server.sandbox import workspace_archive
+
+ monkeypatch.setenv('RUNTIME_FILE_ARCHIVE_BUCKET', 'archive-bkt')
+ monkeypatch.setenv('RUNTIME_FILE_ARCHIVE_REQUIRED', 'true')
+
+ # "Not a git repository" from GET /api/file/archive
+ client = _stream_client(_make_stream_response(400))
+
+ ok = await workspace_archive.archive_workspace(
+ client,
+ create_runtime_data(),
+ 'sandbox-1',
+ archive_path='/workspace/project',
+ )
+ # Permanent 4xx -> nothing to lose -> deletion proceeds despite REQUIRED.
+ assert ok is True
+
+ @pytest.mark.asyncio
+ async def test_archive_404_required_blocks_delete(self, monkeypatch):
+ """A 404 (path missing) is the misrouted-path symptom, NOT a confirmed-empty
+ workspace: under REQUIRED it must block the delete (keep for the idle reap)
+ rather than silently tearing the sandbox down uncaptured."""
+ from openhands.app_server.sandbox import workspace_archive
+
+ monkeypatch.setenv('RUNTIME_FILE_ARCHIVE_BUCKET', 'archive-bkt')
+ monkeypatch.setenv('RUNTIME_FILE_ARCHIVE_REQUIRED', 'true')
+
+ client = _stream_client(_make_stream_response(404))
+
+ ok = await workspace_archive.archive_workspace(
+ client,
+ create_runtime_data(),
+ 'sandbox-1',
+ archive_path='/workspace/project',
+ )
+ assert ok is False
+
+ @pytest.mark.asyncio
+ async def test_archive_500_required_blocks_delete(self, monkeypatch):
+ """A 5xx is a genuine transient failure: required archiving blocks the
+ delete so it can be retried (BUG 2 β€” only 5xx/network block)."""
+ from openhands.app_server.sandbox import workspace_archive
+
+ monkeypatch.setenv('RUNTIME_FILE_ARCHIVE_BUCKET', 'archive-bkt')
+ monkeypatch.setenv('RUNTIME_FILE_ARCHIVE_REQUIRED', 'true')
+
+ client = _stream_client(_make_stream_response(500))
+
+ ok = await workspace_archive.archive_workspace(
+ client,
+ create_runtime_data(),
+ 'sandbox-1',
+ archive_path='/workspace/project',
+ )
+ assert ok is False
+
+ @pytest.mark.asyncio
+ async def test_archive_422_required_blocks_delete(self, monkeypatch):
+ """A 422 (malformed request from the producer) is NOT "nothing to archive"
+ β€” under REQUIRED it is treated as retryable and must block the delete
+ instead of being misread as permanent (infra#1444 M1)."""
+ from openhands.app_server.sandbox import workspace_archive
+
+ monkeypatch.setenv('RUNTIME_FILE_ARCHIVE_BUCKET', 'archive-bkt')
+ monkeypatch.setenv('RUNTIME_FILE_ARCHIVE_REQUIRED', 'true')
+
+ client = _stream_client(_make_stream_response(422))
+
+ ok = await workspace_archive.archive_workspace(
+ client,
+ create_runtime_data(),
+ 'sandbox-1',
+ archive_path='/workspace/project',
+ )
+ assert ok is False
+
+ @pytest.mark.asyncio
+ async def test_archive_401_required_blocks_delete(self, monkeypatch):
+ """A 401 (session key rejected) means the capture did NOT happen and is not
+ a confirmed-empty workspace, so under REQUIRED it must block the delete
+ (keep for the idle reap) rather than tear the sandbox down uncaptured."""
+ from openhands.app_server.sandbox import workspace_archive
+
+ monkeypatch.setenv('RUNTIME_FILE_ARCHIVE_BUCKET', 'archive-bkt')
+ monkeypatch.setenv('RUNTIME_FILE_ARCHIVE_REQUIRED', 'true')
+
+ client = _stream_client(_make_stream_response(401))
+
+ ok = await workspace_archive.archive_workspace(
+ client,
+ create_runtime_data(),
+ 'sandbox-1',
+ archive_path='/workspace/project',
+ )
+ assert ok is False
+
+ @pytest.mark.asyncio
+ async def test_archive_unsupported_format_proceeds_and_skips_request(
+ self, monkeypatch
+ ):
+ """An unsupported RUNTIME_FILE_ARCHIVE_FORMAT (e.g. the removed 'zip') is a
+ pure config error caught up front: no request is issued, and the delete
+ PROCEEDS even under REQUIRED. Like an unset bucket, no retry makes a valid
+ format appear, so blocking would wedge every delete forever with no
+ app-server backstop; the producer never sees the bad format."""
+ from openhands.app_server.sandbox import workspace_archive
+
+ monkeypatch.setenv('RUNTIME_FILE_ARCHIVE_BUCKET', 'archive-bkt')
+ monkeypatch.setenv('RUNTIME_FILE_ARCHIVE_FORMAT', 'zip')
+ monkeypatch.setenv('RUNTIME_FILE_ARCHIVE_REQUIRED', 'true')
+
+ client = AsyncMock()
+
+ ok = await workspace_archive.archive_workspace(
+ client,
+ create_runtime_data(),
+ 'sandbox-1',
+ archive_path='/workspace/project',
+ )
+ assert ok is True
+ client.stream.assert_not_called()
+
+ @pytest.mark.asyncio
+ async def test_archive_unsupported_format_not_required_proceeds(self, monkeypatch):
+ from openhands.app_server.sandbox import workspace_archive
+
+ monkeypatch.setenv('RUNTIME_FILE_ARCHIVE_BUCKET', 'archive-bkt')
+ monkeypatch.setenv('RUNTIME_FILE_ARCHIVE_FORMAT', 'zip')
+ monkeypatch.delenv('RUNTIME_FILE_ARCHIVE_REQUIRED', raising=False)
+
+ client = AsyncMock()
+
+ ok = await workspace_archive.archive_workspace(
+ client,
+ create_runtime_data(),
+ 'sandbox-1',
+ archive_path='/workspace/project',
+ )
+ assert ok is True
+ client.stream.assert_not_called()
+
+ @pytest.mark.asyncio
+ async def test_archive_no_bucket_required_proceeds(self, monkeypatch):
+ """RUNTIME_FILE_ARCHIVE_BUCKET unset is a config error no retry can fix;
+ it must NOT wedge every delete under REQUIRED β€” proceed loudly instead
+ (infra#1444 L8)."""
+ from openhands.app_server.sandbox import workspace_archive
+
+ monkeypatch.delenv('RUNTIME_FILE_ARCHIVE_BUCKET', raising=False)
+ monkeypatch.setenv('RUNTIME_FILE_ARCHIVE_REQUIRED', 'true')
+
+ client = AsyncMock()
+
+ ok = await workspace_archive.archive_workspace(
+ client,
+ create_runtime_data(),
+ 'sandbox-1',
+ archive_path='/workspace/project',
+ )
+ assert ok is True
+ client.stream.assert_not_called()
+
+ @pytest.mark.asyncio
+ async def test_archive_uses_given_path_verbatim(self, monkeypatch):
+ """archive_workspace targets the caller-resolved archive_path exactly (the
+ path was pinned at creation; the grouping nesting is resolved upstream in
+ RemoteSandboxService._resolve_archive_path)."""
+ from openhands.app_server.sandbox import workspace_archive
+
+ monkeypatch.setenv('RUNTIME_FILE_ARCHIVE_BUCKET', 'archive-bkt')
+ monkeypatch.setenv('RUNTIME_FILE_ARCHIVE_FORMAT', 'git-delta')
+
+ client = _stream_client(_make_stream_response(200, b'patch-bytes', {}))
+ store = MagicMock()
+ writes: dict[str, bytes] = {}
+ store.write.side_effect = lambda path, data: writes.__setitem__(path, data)
+ store.write_from_path.side_effect = lambda path, src: writes.__setitem__(
+ path, open(src, 'rb').read()
+ )
+
+ with patch.object(
+ workspace_archive, '_get_archive_file_store', return_value=store
+ ):
+ ok = await workspace_archive.archive_workspace(
+ client,
+ create_runtime_data(),
+ 'sandbox-1',
+ archive_path='/workspace/project/deadbeef',
+ conversation_id='deadbeef',
+ )
+
+ assert ok is True
+ _, kwargs = client.stream.call_args
+ assert kwargs['params']['path'] == '/workspace/project/deadbeef'
+ manifest_path = next(p for p in writes if p.endswith('.manifest.json'))
+ manifest = json.loads(writes[manifest_path])
+ assert manifest['source_path'] == '/workspace/project/deadbeef'
+ assert manifest['conversation_id'] == 'deadbeef'
+
+ @pytest.mark.asyncio
+ async def test_archive_both_uploads_delta_and_targz(self, monkeypatch):
+ """FORMAT=both (the default) captures git-delta AND a full tar.gz, each
+ with its own artifact + manifest; base_commit rides the git-delta header
+ and is reused for the tar.gz manifest."""
+ from openhands.app_server.sandbox import workspace_archive
+
+ monkeypatch.setenv('RUNTIME_FILE_ARCHIVE_BUCKET', 'archive-bkt')
+ monkeypatch.setenv('RUNTIME_FILE_ARCHIVE_FORMAT', 'both')
+
+ responses = {
+ 'git-delta': _make_stream_response(
+ 200, b'patch-bytes', {'X-Archive-Base-Commit': 'abc123'}
+ ),
+ 'tar.gz': _make_stream_response(200, b'tar-bytes', {}),
+ }
+ client = _stream_client(responses)
+ store = MagicMock()
+ writes: dict[str, bytes] = {}
+ store.write.side_effect = lambda path, data: writes.__setitem__(path, data)
+ # Archive blobs are streamed from a tempfile via write_from_path (the OOM
+ # fix); record them too so blob-path assertions still see the .patch/.tar.gz.
+ store.write_from_path.side_effect = lambda path, src: writes.__setitem__(
+ path, open(src, 'rb').read()
+ )
+
+ with patch.object(
+ workspace_archive, '_get_archive_file_store', return_value=store
+ ):
+ ok = await workspace_archive.archive_workspace(
+ client,
+ create_runtime_data(),
+ 'sandbox-1',
+ archive_path='/workspace/project',
+ conversation_id='conv-1',
+ )
+
+ assert ok is True
+ # Two artifacts under the same {ts} base, one per format.
+ assert any(p.endswith('.patch') for p in writes)
+ assert any(p.endswith('.tar.gz') for p in writes)
+ patch_manifest = json.loads(
+ writes[next(p for p in writes if p.endswith('.patch.manifest.json'))]
+ )
+ targz_manifest = json.loads(
+ writes[next(p for p in writes if p.endswith('.tar.gz.manifest.json'))]
+ )
+ assert patch_manifest['format'] == 'git-delta'
+ assert targz_manifest['format'] == 'tar.gz'
+ # base_commit from the git-delta header is carried onto the tar.gz too.
+ assert patch_manifest['base_commit'] == 'abc123'
+ assert targz_manifest['base_commit'] == 'abc123'
+ assert patch_manifest['phase'] == targz_manifest['phase'] == 'final'
+
+ @pytest.mark.asyncio
+ async def test_archive_both_required_blocks_if_one_format_fails(self, monkeypatch):
+ """In REQUIRED both-mode, a retryable failure on either format blocks the
+ delete so a retry can capture the complete pair."""
+ from openhands.app_server.sandbox import workspace_archive
+
+ monkeypatch.setenv('RUNTIME_FILE_ARCHIVE_BUCKET', 'archive-bkt')
+ monkeypatch.setenv('RUNTIME_FILE_ARCHIVE_FORMAT', 'both')
+ monkeypatch.setenv('RUNTIME_FILE_ARCHIVE_REQUIRED', 'true')
+
+ responses = {
+ 'git-delta': _make_stream_response(
+ 200, b'patch-bytes', {'X-Archive-Base-Commit': 'abc123'}
+ ),
+ 'tar.gz': _make_stream_response(500),
+ }
+ client = _stream_client(responses)
+ store = MagicMock()
+ store.write.side_effect = lambda path, data: None
+ store.write_from_path.side_effect = lambda path, src: None
+
+ with patch.object(
+ workspace_archive, '_get_archive_file_store', return_value=store
+ ):
+ ok = await workspace_archive.archive_workspace(
+ client,
+ create_runtime_data(),
+ 'sandbox-1',
+ archive_path='/workspace/project',
+ )
+
+ # tar.gz 500 is retryable + REQUIRED -> block the delete for a retry.
+ assert ok is False
+
+
+class TestArchiveInitialWorkspaceHelper:
+ """Unit tests for the initial-state (pre-agent) workspace snapshot."""
+
+ @pytest.mark.asyncio
+ async def test_initial_archive_uploads_tar_gz_and_manifest(self, monkeypatch):
+ from openhands.app_server.sandbox import workspace_archive
+
+ monkeypatch.setenv('RUNTIME_FILE_ARCHIVE_INITIAL_ENABLED', 'true')
+ monkeypatch.setenv('RUNTIME_FILE_ARCHIVE_BUCKET', 'archive-bkt')
+
+ client = _stream_client(_make_stream_response(200, b'tar-bytes', {}))
+ store = MagicMock()
+ writes: dict[str, bytes] = {}
+ store.write.side_effect = lambda path, data: writes.__setitem__(path, data)
+ # Archive blobs are streamed from a tempfile via write_from_path (the OOM
+ # fix); record them too so blob-path assertions still see the .patch/.tar.gz.
+ store.write_from_path.side_effect = lambda path, src: writes.__setitem__(
+ path, open(src, 'rb').read()
+ )
+
+ with patch.object(
+ workspace_archive, '_get_archive_file_store', return_value=store
+ ):
+ ok = await workspace_archive.archive_initial_workspace(
+ client,
+ agent_server_url='https://sandbox.example.com',
+ session_api_key='sk',
+ project_dir='/workspace/project/repo',
+ sandbox_id='sandbox-1',
+ conversation_id='conv-1',
+ base_commit='deadbeef',
+ )
+
+ assert ok is True
+ # tar.gz requested at the (already-resolved) project dir; key forwarded.
+ _, kwargs = client.stream.call_args
+ assert kwargs['headers']['X-Session-API-Key'] == 'sk'
+ # tar.gz is the full capture, so default excludes are disabled.
+ assert kwargs['params'] == {
+ 'path': '/workspace/project/repo',
+ 'format': 'tar.gz',
+ 'use_default_excludes': 'false',
+ }
+ # Keyed by conversation and nested under /initial/ so it can never collide
+ # with a sibling or with this conversation's final capture (which writes to
+ # {prefix}/{sandbox_id}/{conversation_key}/{ts}).
+ archive_path = next(p for p in writes if p.endswith('.tar.gz'))
+ assert '/sandbox-1/conv-1/initial/' in archive_path
+ manifest_path = next(p for p in writes if p.endswith('.manifest.json'))
+ # Shared contract: manifest = blob + '.manifest.json' (keeps the suffix).
+ assert manifest_path == archive_path + '.manifest.json'
+ manifest = json.loads(writes[manifest_path])
+ assert manifest['phase'] == 'initial'
+ assert manifest['base_commit'] == 'deadbeef'
+ assert manifest['conversation_id'] == 'conv-1'
+ assert manifest['format'] == 'tar.gz'
+ assert manifest['source_path'] == '/workspace/project/repo'
+
+ @pytest.mark.asyncio
+ async def test_archive_sibling_conversations_distinct_keys(self, monkeypatch):
+ """Two sibling conversations on the SAME grouped sandbox, captured in the
+ same second, must land at distinct blob keys: the conversation id is in the
+ object path, not just the manifest body (regression β€” same-second overwrite
+ silently dropped one conversation's archive)."""
+ from datetime import datetime, timezone
+
+ from openhands.app_server.sandbox import workspace_archive
+
+ monkeypatch.setenv('RUNTIME_FILE_ARCHIVE_BUCKET', 'archive-bkt')
+ monkeypatch.setenv('RUNTIME_FILE_ARCHIVE_FORMAT', 'git-delta')
+ # Pin the timestamp so both captures compute the SAME {ts}; only the
+ # conversation segment can keep them apart.
+ fixed = datetime(2026, 1, 1, 0, 0, 0, tzinfo=timezone.utc)
+ monkeypatch.setattr(workspace_archive, 'utc_now', lambda: fixed)
+
+ writes: dict[str, bytes] = {}
+ store = MagicMock()
+ store.write.side_effect = lambda path, data: writes.__setitem__(path, data)
+ store.write_from_path.side_effect = lambda path, src: writes.__setitem__(
+ path, open(src, 'rb').read()
+ )
+
+ with patch.object(
+ workspace_archive, '_get_archive_file_store', return_value=store
+ ):
+ for conv in ('conva', 'convb'):
+ client = _stream_client(_make_stream_response(200, b'patch-bytes', {}))
+ ok = await workspace_archive.archive_workspace(
+ client,
+ create_runtime_data(),
+ 'shared-sandbox',
+ archive_path=f'/workspace/project/{conv}',
+ conversation_id=conv,
+ )
+ assert ok is True
+
+ # Both artifacts AND both manifests survive β€” nothing overwritten.
+ patch_blobs = [p for p in writes if p.endswith('.patch')]
+ manifests = [p for p in writes if p.endswith('.manifest.json')]
+ assert len(patch_blobs) == 2
+ assert len(manifests) == 2
+ assert any('/shared-sandbox/conva/' in p for p in patch_blobs)
+ assert any('/shared-sandbox/convb/' in p for p in patch_blobs)
+
+ @pytest.mark.asyncio
+ async def test_initial_archive_sibling_conversations_distinct_keys(
+ self, monkeypatch
+ ):
+ """Same same-second path-collision guard for the initial snapshot under
+ grouping: sibling initial captures land at distinct keys."""
+ from datetime import datetime, timezone
+
+ from openhands.app_server.sandbox import workspace_archive
+
+ monkeypatch.setenv('RUNTIME_FILE_ARCHIVE_INITIAL_ENABLED', 'true')
+ monkeypatch.setenv('RUNTIME_FILE_ARCHIVE_BUCKET', 'archive-bkt')
+ fixed = datetime(2026, 1, 1, 0, 0, 0, tzinfo=timezone.utc)
+ monkeypatch.setattr(workspace_archive, 'utc_now', lambda: fixed)
+
+ writes: dict[str, bytes] = {}
+ store = MagicMock()
+ store.write.side_effect = lambda path, data: writes.__setitem__(path, data)
+ store.write_from_path.side_effect = lambda path, src: writes.__setitem__(
+ path, open(src, 'rb').read()
+ )
+
+ with patch.object(
+ workspace_archive, '_get_archive_file_store', return_value=store
+ ):
+ for conv in ('conva', 'convb'):
+ client = _stream_client(_make_stream_response(200, b'tar-bytes', {}))
+ ok = await workspace_archive.archive_initial_workspace(
+ client,
+ agent_server_url='https://sandbox.example.com',
+ session_api_key='sk',
+ project_dir=f'/workspace/project/{conv}',
+ sandbox_id='shared-sandbox',
+ conversation_id=conv,
+ )
+ assert ok is True
+
+ tarballs = [p for p in writes if p.endswith('.tar.gz')]
+ manifests = [p for p in writes if p.endswith('.manifest.json')]
+ assert len(tarballs) == 2
+ assert len(manifests) == 2
+ assert any('/shared-sandbox/conva/initial/' in p for p in tarballs)
+ assert any('/shared-sandbox/convb/initial/' in p for p in tarballs)
+
+ @pytest.mark.asyncio
+ async def test_initial_archive_disabled_is_noop(self, monkeypatch):
+ from openhands.app_server.sandbox import workspace_archive
+
+ monkeypatch.delenv('RUNTIME_FILE_ARCHIVE_INITIAL_ENABLED', raising=False)
+ monkeypatch.setenv('RUNTIME_FILE_ARCHIVE_BUCKET', 'archive-bkt')
+
+ client = AsyncMock()
+
+ ok = await workspace_archive.archive_initial_workspace(
+ client,
+ agent_server_url='https://sandbox.example.com',
+ session_api_key='sk',
+ project_dir='/workspace/project/repo',
+ sandbox_id='sandbox-1',
+ )
+ # Off by default: no request, no upload.
+ assert ok is False
+ client.stream.assert_not_called()
+
+ @pytest.mark.asyncio
+ async def test_initial_archive_non_200_returns_false(self, monkeypatch):
+ from openhands.app_server.sandbox import workspace_archive
+
+ monkeypatch.setenv('RUNTIME_FILE_ARCHIVE_INITIAL_ENABLED', 'true')
+ monkeypatch.setenv('RUNTIME_FILE_ARCHIVE_BUCKET', 'archive-bkt')
+
+ client = _stream_client(_make_stream_response(500))
+
+ ok = await workspace_archive.archive_initial_workspace(
+ client,
+ agent_server_url='https://sandbox.example.com',
+ session_api_key='sk',
+ project_dir='/workspace/project/repo',
+ sandbox_id='sandbox-1',
+ )
+ # Best-effort: a failed capture is swallowed (never blocks startup).
+ assert ok is False
+
+ @pytest.mark.asyncio
+ async def test_initial_archive_no_bucket_returns_false(self, monkeypatch):
+ from openhands.app_server.sandbox import workspace_archive
+
+ monkeypatch.setenv('RUNTIME_FILE_ARCHIVE_INITIAL_ENABLED', 'true')
+ monkeypatch.delenv('RUNTIME_FILE_ARCHIVE_BUCKET', raising=False)
+
+ client = AsyncMock()
+
+ ok = await workspace_archive.archive_initial_workspace(
+ client,
+ agent_server_url='https://sandbox.example.com',
+ session_api_key='sk',
+ project_dir='/workspace/project/repo',
+ sandbox_id='sandbox-1',
+ )
+ assert ok is False
+ client.stream.assert_not_called()
+
+
+class TestDeleteSandboxKeyHandling:
+ """The session_api_key_hash is invalidated UP FRONT on delete (a delete is
+ often a revoke of a leaked key). When a transient error keeps the row for
+ retry, the invalidation is committed first so the DELETE route's rollback
+ cannot resurrect the key. Backed by a real SQLite session so persistence is
+ provable across a rollback."""
+
+ @pytest.fixture
+ async def async_engine(self):
+ from sqlalchemy.ext.asyncio import create_async_engine
+ from sqlalchemy.pool import StaticPool
+
+ from openhands.app_server.utils.sql_utils import Base
+
+ engine = create_async_engine(
+ 'sqlite+aiosqlite:///:memory:',
+ poolclass=StaticPool,
+ connect_args={'check_same_thread': False},
+ echo=False,
+ )
+ async with engine.begin() as conn:
+ await conn.run_sync(Base.metadata.create_all)
+ yield engine
+ await engine.dispose()
+
+ @pytest.fixture
+ async def real_session(self, async_engine):
+ from sqlalchemy.ext.asyncio import async_sessionmaker
+
+ maker = async_sessionmaker(
+ async_engine, class_=AsyncSession, expire_on_commit=False
+ )
+ async with maker() as session:
+ yield session
+
+ @pytest.fixture
+ def service_with_real_db(
+ self, mock_sandbox_spec_service, mock_user_context, real_session
+ ):
+ return RemoteSandboxService(
+ sandbox_spec_service=mock_sandbox_spec_service,
+ api_url='https://api.example.com',
+ api_key='test-api-key',
+ web_url='https://web.example.com',
+ resource_factor=1,
+ runtime_class='gvisor',
+ start_sandbox_timeout=120,
+ max_num_sandboxes=10,
+ user_context=mock_user_context,
+ httpx_client=AsyncMock(spec=httpx.AsyncClient),
+ db_session=real_session,
+ )
+
+ @pytest.mark.asyncio
+ async def test_session_key_invalidated_and_survives_rollback(
+ self, service_with_real_db, real_session
+ ):
+ # Seed a running sandbox with a live session key hash.
+ row = create_stored_sandbox(session_api_key_hash='live-hash')
+ real_session.add(row)
+ await real_session.commit()
+
+ # Transient (non-404) runtime error -> delete_sandbox raises (keeps the
+ # row + running runtime for retry) without deleting the row.
+ service_with_real_db._get_runtime = AsyncMock(
+ side_effect=httpx.HTTPStatusError(
+ 'Server Error',
+ request=httpx.Request('GET', 'https://api.example.com/sessions/x'),
+ response=httpx.Response(503),
+ )
+ )
+
+ with pytest.raises(SandboxDeleteRetryError):
+ await service_with_real_db.delete_sandbox('test-sandbox-123')
+
+ # Simulate the DELETE route's session rolling back the failed delete.
+ await real_session.rollback()
+ real_session.expire_all()
+
+ # Row kept for retry, but the key is DEAD and the rollback could not bring
+ # it back β€” delete_sandbox committed the invalidation before raising. This
+ # is the security fix: a leaked key is revoked even when the stop fails.
+ refreshed = await service_with_real_db._get_stored_sandbox('test-sandbox-123')
+ assert refreshed is not None
+ assert refreshed.session_api_key_hash is None
+
+ @pytest.mark.asyncio
+ async def test_session_key_cleared_on_successful_delete(
+ self, service_with_real_db, real_session, monkeypatch
+ ):
+ # Archiving off -> a clean delete goes straight to /stop + row delete.
+ monkeypatch.delenv('RUNTIME_FILE_ARCHIVE_ENABLED', raising=False)
+ row = create_stored_sandbox(session_api_key_hash='live-hash')
+ real_session.add(row)
+ await real_session.commit()
+
+ service_with_real_db._get_runtime = AsyncMock(
+ return_value={'runtime_id': 'rt-123'}
+ )
+ stop_response = MagicMock()
+ stop_response.status_code = 200
+ stop_response.raise_for_status.return_value = None
+ service_with_real_db._send_runtime_api_request = AsyncMock(
+ return_value=stop_response
+ )
+
+ assert await service_with_real_db.delete_sandbox('test-sandbox-123') is True
+ await real_session.commit()
+ real_session.expire_all()
+
+ # The row β€” and with it the session key β€” is gone after a clean delete.
+ refreshed = await service_with_real_db._get_stored_sandbox('test-sandbox-123')
+ assert refreshed is None
+
+
+class TestArchiveEnvToggles:
+ """Per OpenHands/AGENTS.md, env enable-toggles must accept '1' as well as
+ 'true' (older Helm charts default these to '1'). A regression to == 'true'
+ would silently disable the entire workspace-archive capture, so pin both."""
+
+ @pytest.mark.parametrize('value', ['1', 'true', 'TRUE', 'True'])
+ def test_archive_enabled_truthy(self, monkeypatch, value):
+ from openhands.app_server.sandbox import workspace_archive
+
+ monkeypatch.setenv('RUNTIME_FILE_ARCHIVE_ENABLED', value)
+ assert workspace_archive.archive_enabled() is True
+
+ @pytest.mark.parametrize('value', ['0', 'false', '', 'no'])
+ def test_archive_enabled_falsy(self, monkeypatch, value):
+ from openhands.app_server.sandbox import workspace_archive
+
+ monkeypatch.setenv('RUNTIME_FILE_ARCHIVE_ENABLED', value)
+ assert workspace_archive.archive_enabled() is False
+
+ @pytest.mark.parametrize('value', ['1', 'true'])
+ def test_archive_required_truthy(self, monkeypatch, value):
+ from openhands.app_server.sandbox import workspace_archive
+
+ monkeypatch.setenv('RUNTIME_FILE_ARCHIVE_REQUIRED', value)
+ assert workspace_archive.archive_required() is True
+
+ @pytest.mark.parametrize('value', ['1', 'true'])
+ def test_initial_archive_enabled_truthy(self, monkeypatch, value):
+ from openhands.app_server.sandbox import workspace_archive
+
+ monkeypatch.setenv('RUNTIME_FILE_ARCHIVE_INITIAL_ENABLED', value)
+ assert workspace_archive.initial_archive_enabled() is True
+
+ def test_toggles_default_off(self, monkeypatch):
+ from openhands.app_server.sandbox import workspace_archive
+
+ for var in (
+ 'RUNTIME_FILE_ARCHIVE_ENABLED',
+ 'RUNTIME_FILE_ARCHIVE_REQUIRED',
+ 'RUNTIME_FILE_ARCHIVE_INITIAL_ENABLED',
+ ):
+ monkeypatch.delenv(var, raising=False)
+ assert workspace_archive.archive_enabled() is False
+ assert workspace_archive.archive_required() is False
+ assert workspace_archive.initial_archive_enabled() is False
+
+
+class TestArchiveRequestParams:
+ """The tar.gz is the self-contained full capture, so it disables the
+ endpoint's default excludes; git-delta stays compact (keeps them)."""
+
+ def test_tar_gz_disables_default_excludes(self):
+ from openhands.app_server.sandbox import workspace_archive
+
+ params = workspace_archive._archive_request_params(
+ '/workspace/project', 'tar.gz'
+ )
+ assert params == {
+ 'path': '/workspace/project',
+ 'format': 'tar.gz',
+ 'use_default_excludes': 'false',
+ }
+
+ def test_git_delta_keeps_default_excludes(self):
+ from openhands.app_server.sandbox import workspace_archive
+
+ params = workspace_archive._archive_request_params(
+ '/workspace/project', 'git-delta'
+ )
+ assert params == {'path': '/workspace/project', 'format': 'git-delta'}