π OpenHands: AI-Driven Development
feat(app-server): capture production workspace state β initial snapshot at start + archive before delete (APP-2403) (#14953)
simonrosenberg77bd80d3a44c
diff --git a/openhands/app_server/app_conversation/app_conversation_models.py b/openhands/app_server/app_conversation/app_conversation_models.pyindex 5fc97bed68..fc4c928399 100644--- a/openhands/app_server/app_conversation/app_conversation_models.py+++ b/openhands/app_server/app_conversation/app_conversation_models.py@@ -33,6 +33,12 @@ __all__ = ['SandboxGroupingStrategy']# The typed ``AppConversationInfo.acp_server`` field is a projection of this tag.ACP_SERVER_TAG_KEY = 'acpserver'+# Conversation-tag key pinning the resolved (grouped) workspace path at creation+# so the delete-time archive captures the right directory without re-deriving it+# from settings that may have changed. Must satisfy the SDK ^[a-z0-9]+$ tag-key+# rule β no underscores.+ARCHIVE_WORKSPACE_PATH_TAG_KEY = 'archiveworkspacepath'+class ConversationTrigger(Enum):RESOLVER = 'resolver'diff --git a/openhands/app_server/app_conversation/app_conversation_router.py b/openhands/app_server/app_conversation/app_conversation_router.pyindex 83e26e7fca..fed3db1f1e 100644--- a/openhands/app_server/app_conversation/app_conversation_router.py+++ b/openhands/app_server/app_conversation/app_conversation_router.py@@ -23,6 +23,7 @@ from openhands.app_server.app_conversation.app_conversation_info_service importAppConversationInfoService,)from openhands.app_server.app_conversation.app_conversation_models import (+ ARCHIVE_WORKSPACE_PATH_TAG_KEY,AppConversation,AppConversationInfo,AppConversationPage,@@ -871,17 +872,52 @@ async def _finalize_sandbox_delete(sandbox_id: str,db_session: AsyncSession,httpx_client: httpx.AsyncClient,+ conversation_id: UUID | None = None,+ workspace_path: str | None = None,) -> None:- """Delete sandbox if no other conversations reference it, then close connections."""+ """Archive the conversation's workspace, then delete the sandbox if unreferenced.++ Runs detached (background task) AFTER the delete response was already returned.+ The workspace is captured FIRST β a conversation-scoped step, while the runtime+ is still up β and only if that succeeds (or archiving is not REQUIRED) is the+ sandbox torn down, and only when no other conversation still references it+ (under grouping a sibling conversation keeps it alive). When archiving is+ REQUIRED and fails, the sandbox + running runtime are kept so the runtime-api+ idle reap captures the workspace later (the durability backstop). delete_sandbox+ is sandbox-scoped (stop + delete) and knows nothing about conversations.+ """try:- conversation_count = (- await app_conversation_info_service.count_conversations_by_sandbox_id(- sandbox_id+ archived = await sandbox_service.archive_conversation_workspace(+ sandbox_id,+ conversation_id=conversation_id.hex if conversation_id else None,+ workspace_path=workspace_path,+ )+ if not archived:+ # REQUIRED archive failed: keep the sandbox + running runtime for the+ # runtime-api idle reap to capture (the durability backstop).+ logger.warning(+ 'Workspace archive required but failed for %s; leaving the '+ 'sandbox + runtime for the idle reap',+ sandbox_id,)- )- if conversation_count == 0:- await sandbox_service.delete_sandbox(sandbox_id)+ else:+ conversation_count = (+ await app_conversation_info_service.count_conversations_by_sandbox_id(+ sandbox_id+ )+ )+ if conversation_count == 0:+ await sandbox_service.delete_sandbox(sandbox_id)await db_session.commit()+ except Exception:+ # Any failure in the finalizer (a transient stop/lookup error, the count+ # query, the commit itself): do NOT commit a half-done delete, so no+ # orphaned row is left; the row + running runtime stay for the runtime-api+ # idle reap to capture + reap.+ logger.exception(+ 'Deferred sandbox cleanup failed for %s; kept for retry', sandbox_id+ )+ await db_session.rollback()finally:await asyncio.gather(db_session.aclose(),@@ -965,6 +1001,8 @@ async def delete_app_conversation(# Delete the sandbox in the background if no other conversations reference itif sandbox_id:+ # Path pinned at creation; the finalizer archives exactly this directory.+ workspace_path = app_conversation_info.tags.get(ARCHIVE_WORKSPACE_PATH_TAG_KEY)asyncio.create_task(_finalize_sandbox_delete(sandbox_service,@@ -972,6 +1010,8 @@ async def delete_app_conversation(sandbox_id,db_session,httpx_client,+ conversation_id=conversation_uuid,+ workspace_path=workspace_path,))diff --git a/openhands/app_server/app_conversation/app_conversation_service.py b/openhands/app_server/app_conversation/app_conversation_service.pyindex b2aa155eb7..484c615bd8 100644--- a/openhands/app_server/app_conversation/app_conversation_service.py+++ b/openhands/app_server/app_conversation/app_conversation_service.py@@ -122,6 +122,7 @@ class AppConversationService(ABC):sandbox: SandboxInfo,workspace: AsyncRemoteWorkspace,agent_server_url: str,+ conversation_id: UUID,) -> AsyncGenerator[AppConversationStartTask, None]:"""Run the setup scripts for the project and yield status updates"""yield taskdiff --git a/openhands/app_server/app_conversation/app_conversation_service_base.py b/openhands/app_server/app_conversation/app_conversation_service_base.pyindex 174f7085d6..06e69a14fe 100644--- a/openhands/app_server/app_conversation/app_conversation_service_base.py+++ b/openhands/app_server/app_conversation/app_conversation_service_base.py@@ -1,3 +1,4 @@+import asyncioimport loggingimport osimport shlex@@ -27,6 +28,7 @@ from openhands.app_server.app_conversation.skill_loader import (load_skills_from_agent_server,)from openhands.app_server.integrations.service_types import ProviderType+from openhands.app_server.sandbox import workspace_archivefrom openhands.app_server.sandbox.sandbox_models import SandboxInfofrom openhands.app_server.user.user_context import UserContextfrom openhands.app_server.utils.auth import looks_like_jwt@@ -46,6 +48,7 @@ from openhands.sdk.skills import Skillfrom openhands.sdk.workspace.remote.async_remote_workspace import AsyncRemoteWorkspace_logger = logging.getLogger(__name__)+PRE_COMMIT_HOOK = '.git/hooks/pre-commit'PRE_COMMIT_LOCAL = '.git/hooks/pre-commit.local'@@ -252,6 +255,7 @@ class AppConversationServiceBase(AppConversationService, ABC):sandbox: SandboxInfo,workspace: AsyncRemoteWorkspace,agent_server_url: str,+ conversation_id: UUID,) -> AsyncGenerator[AppConversationStartTask, None]:task.status = AppConversationStartTaskStatus.PREPARING_REPOSITORYyield task@@ -264,6 +268,28 @@ class AppConversationServiceBase(AppConversationService, ABC):workspace.working_dir, task.request.selected_repository)+ # Snapshot the INITIAL workspace (repo exactly as cloned, before setup.sh)+ # for downstream dataset/eval creation. Awaited HERE β before any+ # mutating step β so+ # the capture is deterministic and can't race setup.sh writing the same+ # tree. Best-effort and bounded so it never fails or unduly delays startup.+ try:+ await asyncio.wait_for(+ self._maybe_archive_initial_state(+ task,+ sandbox,+ workspace,+ agent_server_url,+ project_dir,+ conversation_id,+ ),+ timeout=workspace_archive.initial_archive_deadline(),+ )+ except Exception as e:+ _logger.warning(+ 'Initial workspace archive did not complete in time (ignored): %s', e+ )+task.status = AppConversationStartTaskStatus.RUNNING_SETUP_SCRIPTyield taskawait self.maybe_run_setup_script(workspace, project_dir)@@ -281,6 +307,55 @@ class AppConversationServiceBase(AppConversationService, ABC):agent_server_url,)+ async def _maybe_archive_initial_state(+ self,+ task: AppConversationStartTask,+ sandbox: SandboxInfo,+ workspace: AsyncRemoteWorkspace,+ agent_server_url: str,+ project_dir: str,+ conversation_id: UUID,+ ) -> None:+ """Snapshot the initial (as-cloned) workspace state β best-effort.++ Awaited before setup.sh, so it captures the repo exactly as cloned with no+ concurrent mutation, keyed to the real conversation id. No-op unless+ ``RUNTIME_FILE_ARCHIVE_INITIAL_ENABLED`` is set and a repository was+ actually cloned (an empty workspace is not a useful starting state).+ Every failure is swallowed so this can never break conversation startup.+ """+ try:+ if not workspace_archive.initial_archive_enabled():+ return+ if not task.request.selected_repository:+ # Only repo-backed conversations have a meaningful initial state.+ return++ # Record the commit the snapshot came from β tar.gz carries no+ # base-commit header. Best-effort: a miss just leaves base_commit ''.+ base_commit = ''+ try:+ result = await workspace.execute_command(+ 'git rev-parse HEAD', project_dir+ )+ if not result.exit_code:+ base_commit = result.stdout.strip()+ except Exception as e:+ _logger.debug('Initial-state base commit lookup failed: %s', e)++ conv_hex = conversation_id.hex if conversation_id else None+ await workspace_archive.archive_initial_workspace(+ workspace.client,+ agent_server_url=agent_server_url,+ session_api_key=sandbox.session_api_key,+ project_dir=project_dir,+ sandbox_id=sandbox.id,+ conversation_id=conv_hex,+ base_commit=base_commit,+ )+ except Exception as e:+ _logger.warning('Initial workspace archive step failed (ignored): %s', e)+async def _configure_git_user_settings(self,workspace: AsyncRemoteWorkspace,diff --git a/openhands/app_server/app_conversation/live_status_app_conversation_service.py b/openhands/app_server/app_conversation/live_status_app_conversation_service.pyindex 2d79f95234..de4f59f79c 100644--- a/openhands/app_server/app_conversation/live_status_app_conversation_service.py+++ b/openhands/app_server/app_conversation/live_status_app_conversation_service.py@@ -28,6 +28,7 @@ from openhands.app_server.app_conversation.app_conversation_info_service import)from openhands.app_server.app_conversation.app_conversation_models import (ACP_SERVER_TAG_KEY,+ ARCHIVE_WORKSPACE_PATH_TAG_KEY,AgentType,AppConversation,AppConversationInfo,@@ -97,6 +98,7 @@ from openhands.app_server.sandbox.sandbox_spec_service import (from openhands.app_server.services.injector import InjectorStatefrom openhands.app_server.services.jwt_service import JwtServicefrom openhands.app_server.settings.llm_profiles import resolve_profile_llm+from openhands.app_server.settings.settings_models import grouped_workspace_dirfrom openhands.app_server.user.user_context import UserContextfrom openhands.app_server.user.user_models import UserInfofrom openhands.app_server.utils.docker_utils import (@@ -395,10 +397,12 @@ class LiveStatusAppConversationService(AppConversationServiceBase):conversation_id = request.conversation_id or uuid4()# Setup working dir based on grouping- working_dir = sandbox_spec.working_dirsandbox_grouping_strategy = await self._get_sandbox_grouping_strategy()- if sandbox_grouping_strategy != SandboxGroupingStrategy.NO_GROUPING:- working_dir = f'{working_dir}/{conversation_id.hex}'+ working_dir = grouped_workspace_dir(+ sandbox_spec.working_dir,+ sandbox_grouping_strategy,+ conversation_id.hex,+ )# Run setup scriptsremote_workspace = AsyncRemoteWorkspace(@@ -407,7 +411,7 @@ class LiveStatusAppConversationService(AppConversationServiceBase):working_dir=working_dir,)async for updated_task in self.run_setup_scripts(- task, sandbox, remote_workspace, agent_server_url+ task, sandbox, remote_workspace, agent_server_url, conversation_id):yield updated_task@@ -486,6 +490,10 @@ class LiveStatusAppConversationService(AppConversationServiceBase):# the same agent back through the AgentBase discriminator.request_agent = start_conversation_request.agenttags: dict[str, str] = {}+ # Pin where the workspace was actually created so the delete-time+ # archive captures the right directory without re-deriving the path+ # from settings (e.g. grouping) that may change before delete.+ tags[ARCHIVE_WORKSPACE_PATH_TAG_KEY] = working_dirif request_agent.agent_kind == 'acp':llm_model = request_agent.acp_modelagent_kind = 'acp'diff --git a/openhands/app_server/errors.py b/openhands/app_server/errors.pyindex c7cf9614c0..4a8548fdf9 100644--- a/openhands/app_server/errors.py+++ b/openhands/app_server/errors.py@@ -41,3 +41,22 @@ class PermissionsError(OpenHandsError):class SandboxError(OpenHandsError):"""Error in Sandbox."""+++class SandboxDeleteRetryError(OpenHandsError):+ """The sandbox exists but its delete could not complete and was kept for retry.++ Raised by ``delete_sandbox`` when the runtime /stop or lookup hits a transient+ failure. (Archiving never raises β it returns False from+ ``archive_conversation_workspace`` to signal a REQUIRED capture should block.)+ 503 (vs 404) so a client distinguishes "still here, try again" from "not+ found" and keeps retrying.+ """++ def __init__(+ self,+ detail: Any = None,+ headers: dict[str, str] | None = None,+ status_code: int = status.HTTP_503_SERVICE_UNAVAILABLE,+ ):+ super().__init__(status_code=status_code, detail=detail, headers=headers)diff --git a/openhands/app_server/file_store/files.py b/openhands/app_server/file_store/files.pyindex 0beb8ebfe9..ce5eedf518 100644--- a/openhands/app_server/file_store/files.py+++ b/openhands/app_server/file_store/files.py@@ -17,6 +17,18 @@ class FileStore(DiscriminatedUnionMixin, ABC):def write(self, path: str, contents: str | bytes) -> None:pass+ def write_from_path(self, path: str, source_path: str) -> None:+ """Store the object at ``path`` from a local file at ``source_path``.++ The default reads the whole file into memory and delegates to ``write``;+ backends that can stream from disk override this to bound peak memory.+ Callers uploading large blobs (e.g. multi-GB workspace archives, where+ buffering one whole copy in RAM under concurrent deletes risks+ OOM-killing the pod) should prefer this over ``write(path, f.read())``.+ """+ with open(source_path, 'rb') as f:+ self.write(path, f.read())+@abstractmethoddef read(self, path: str) -> str:passdiff --git a/openhands/app_server/file_store/google_cloud.py b/openhands/app_server/file_store/google_cloud.pyindex 271768e28f..882a0ee5f5 100644--- a/openhands/app_server/file_store/google_cloud.py+++ b/openhands/app_server/file_store/google_cloud.py@@ -50,6 +50,11 @@ class GoogleCloudFileStore(FileStore):with blob.open(mode) as f:f.write(contents)+ def write_from_path(self, path: str, source_path: str) -> None:+ # Streams the file to GCS in chunks; never buffers the whole blob in RAM.+ blob: Blob = self.bucket.blob(path)+ blob.upload_from_filename(source_path)+def read(self, path: str) -> str:blob: Blob = self.bucket.blob(path)try:diff --git a/openhands/app_server/file_store/local.py b/openhands/app_server/file_store/local.pyindex 80058cd5dc..bdcafd9550 100644--- a/openhands/app_server/file_store/local.py+++ b/openhands/app_server/file_store/local.py@@ -42,6 +42,20 @@ class LocalFileStore(FileStore):os.remove(temp_path)raise+ def write_from_path(self, path: str, source_path: str) -> None:+ # shutil.copyfile streams in chunks (never the whole file in RAM); keep+ # the same write-temp-then-atomic-rename to avoid torn concurrent writes.+ full_path = self.get_full_path(path)+ os.makedirs(os.path.dirname(full_path), exist_ok=True)+ temp_path = f'{full_path}.tmp.{os.getpid()}.{threading.get_ident()}'+ try:+ shutil.copyfile(source_path, temp_path)+ os.replace(temp_path, full_path)+ except Exception:+ if os.path.exists(temp_path):+ os.remove(temp_path)+ raise+def read(self, path: str) -> str:full_path = self.get_full_path(path)with open(full_path, 'r') as f:diff --git a/openhands/app_server/file_store/memory.py b/openhands/app_server/file_store/memory.pyindex 5eba1b396a..b6e0530e6a 100644--- a/openhands/app_server/file_store/memory.py+++ b/openhands/app_server/file_store/memory.py@@ -7,6 +7,10 @@ from openhands.app_server.utils.logger import openhands_logger as loggerclass InMemoryFileStore(FileStore):+ # Text-only by design: this store is part of the env-parsed FileStore config+ # union (so the value type must stay a primitive) and read() returns str, so+ # it cannot round-trip a binary archive. Not a valid RUNTIME_FILE_ARCHIVE+ # store β see workspace_archive._archive_store_type.files: dict[str, str] = Field(default_factory=dict)def write(self, path: str, contents: str | bytes) -> None:diff --git a/openhands/app_server/file_store/s3.py b/openhands/app_server/file_store/s3.pyindex 430c0537b0..561e307918 100644--- a/openhands/app_server/file_store/s3.py+++ b/openhands/app_server/file_store/s3.py@@ -75,6 +75,24 @@ class S3FileStore(FileStore):f"Error: Failed to write to bucket '{self._get_bucket_name()}' at path {path}: {e}")+ def write_from_path(self, path: str, source_path: str) -> None:+ # upload_file streams the file from disk in parts; never buffers the whole+ # object in RAM (unlike put_object with Body=f.read()).+ try:+ self.client.upload_file(source_path, self._get_bucket_name(), path)+ except botocore.exceptions.ClientError as e:+ if e.response['Error']['Code'] == 'AccessDenied':+ raise FileNotFoundError(+ f"Error: Access denied to bucket '{self._get_bucket_name()}'."+ )+ elif e.response['Error']['Code'] == 'NoSuchBucket':+ raise FileNotFoundError(+ f"Error: The bucket '{self._get_bucket_name()}' does not exist."+ )+ raise FileNotFoundError(+ f"Error: Failed to write to bucket '{self._get_bucket_name()}' at path {path}: {e}"+ )+def read(self, path: str) -> str:try:response: GetObjectOutputDict = self.client.get_object(diff --git a/openhands/app_server/sandbox/process_sandbox_service.py b/openhands/app_server/sandbox/process_sandbox_service.pyindex 8272793021..59fdf7207b 100644--- a/openhands/app_server/sandbox/process_sandbox_service.py+++ b/openhands/app_server/sandbox/process_sandbox_service.py@@ -386,7 +386,7 @@ class ProcessSandboxService(SandboxService):return Falseasync def delete_sandbox(self, sandbox_id: str) -> bool:- """Delete a sandbox."""+ """Delete a sandbox. (No workspace archiving for local processes.)"""process_info = _processes.get(sandbox_id)if process_info is None:return Falsediff --git a/openhands/app_server/sandbox/remote_sandbox_service.py b/openhands/app_server/sandbox/remote_sandbox_service.pyindex f1e67241d5..19d34e60a8 100644--- a/openhands/app_server/sandbox/remote_sandbox_service.py+++ b/openhands/app_server/sandbox/remote_sandbox_service.py@@ -24,7 +24,8 @@ from openhands.agent_server.utils import utc_nowfrom openhands.app_server.app_conversation.app_conversation_models import (AppConversationInfo,)-from openhands.app_server.errors import SandboxError+from openhands.app_server.errors import SandboxDeleteRetryError, SandboxError+from openhands.app_server.sandbox import workspace_archivefrom openhands.app_server.sandbox.sandbox_models import (AGENT_SERVER,VSCODE,@@ -48,8 +49,12 @@ from openhands.app_server.sandbox.sandbox_spec_service import (resolve_sandbox_spec,)from openhands.app_server.services.injector import InjectorState+from openhands.app_server.settings.settings_models import grouped_workspace_dirfrom openhands.app_server.user.specifiy_user_context import ADMIN, USER_CONTEXT_ATTRfrom openhands.app_server.user.user_context import UserContext+from openhands.app_server.utils.docker_utils import (+ replace_localhost_hostname_for_docker,+)from openhands.app_server.utils.sql_utils import Base, UtcDateTimefrom openhands.sdk.utils.paging import page_iterator@@ -571,17 +576,49 @@ class RemoteSandboxService(SandboxService):async def delete_sandbox(self, sandbox_id: str) -> bool:"""Delete a sandbox by stopping its runtime.- Security: Deleting the stored_sandbox record also removes the- session_api_key_hash, invalidating any leaked session keys.+ Purely sandbox-scoped: stop the runtime and delete the record. Workspace+ capture is a separate conversation-scoped step+ (``archive_conversation_workspace``) the conversation-delete finalizer runs+ BEFORE tearing the sandbox down β so a long archive never blocks this call+ (and the direct sandbox DELETE route can't 504 on it).++ If the runtime is already gone (paused/reaped/double-delete, a 404 from+ the runtime API), the record is deleted directly to avoid orphaning it.++ Returns False ONLY when the sandbox does not exist (router -> 404). A+ transient runtime /stop / lookup failure raises ``SandboxDeleteRetryError``+ (router -> 503) and keeps the row + runtime for a retry β so a live sandbox+ is never reported as 404.++ Security: the session_api_key_hash is invalidated UP FRONT (like+ ``pause_sandbox`` clears it before pausing) so a delete β commonly a+ revoke of a leaked key β kills it promptly. This goes further than pause:+ on a transient stop failure the invalidation is committed before raising,+ so the caller's rollback cannot resurrect the just-revoked key (pause does+ not commit, so its clear can still be rolled back). The row is kept for+ retry."""+ had_key = Falsetry:stored_sandbox = await self._get_stored_sandbox(sandbox_id)if not stored_sandbox:return False- # Deleting the record also removes the session_api_key_hash,- # which invalidates any leaked session keys.- await self.db_session.delete(stored_sandbox)- runtime_data = await self._get_runtime(sandbox_id)+ # Security: drop the key now, before the (fallible) runtime stop.+ had_key = stored_sandbox.session_api_key_hash is not None+ stored_sandbox.session_api_key_hash = None+ try:+ runtime_data = await self._get_runtime(sandbox_id)+ except httpx.HTTPStatusError as e:+ if e.response.status_code != 404:+ raise+ # Runtime already gone: nothing to stop. Delete the orphaned row.+ _logger.info(+ f'Runtime for sandbox {sandbox_id} already gone (404); '+ 'deleting record'+ )+ await self.db_session.delete(stored_sandbox)+ return True+response = await self._send_runtime_api_request('POST','/stop',@@ -589,10 +626,153 @@ class RemoteSandboxService(SandboxService):)if response.status_code != 404:response.raise_for_status()+ await self.db_session.delete(stored_sandbox)return Trueexcept httpx.HTTPError as e:+ # Transient runtime lookup/stop failure: keep the row + runtime and+ # signal retryable (503) β never a 404. Persist the key invalidation+ # now: the caller rolls back on this raise, which would otherwise+ # restore the hash and leave a just-revoked key valid._logger.error(f'Error deleting sandbox {sandbox_id}: {e}')- return False+ if had_key:+ await self.db_session.commit()+ raise SandboxDeleteRetryError(+ f'Could not complete delete for sandbox {sandbox_id}: {e}'+ ) from e++ async def _resolve_archive_path(+ self,+ stored_sandbox: StoredRemoteSandbox,+ conversation_id: str | None,+ workspace_path: str | None,+ ) -> str:+ """Path to archive: the value pinned at conversation creation if present,+ else rebuilt from the SAME base the clone used (the sandbox spec's+ ``working_dir``) plus the grouping nesting.++ Pre-pinning conversations have no pinned path; the legacy fallback re-reads+ the live grouping strategy, which can disagree with creation if the user+ toggled it β but a resulting 404 no longer silently tears the sandbox down+ under REQUIRED (it blocks for the idle reap). Raises if the layout cannot+ be resolved, so the caller never archives to the wrong path.+ """+ if workspace_path:+ return workspace_path+ # For cloud conversations the sandbox id is the conversation_id.hex.+ conversation_key = conversation_id or stored_sandbox.id+ sandbox_spec = await self.sandbox_spec_service.get_sandbox_spec(+ stored_sandbox.sandbox_spec_id+ )+ if sandbox_spec is None:+ raise SandboxError(+ f'No sandbox spec {stored_sandbox.sandbox_spec_id} for archive'+ )+ grouping = (await self.user_context.get_user_info()).sandbox_grouping_strategy+ return grouped_workspace_dir(+ sandbox_spec.working_dir, grouping, conversation_key+ )++ async def _archive_workspace(+ self,+ stored_sandbox: StoredRemoteSandbox,+ conversation_id: str | None,+ runtime_data: dict,+ workspace_path: str | None,+ ) -> bool:+ """Archive one workspace via the in-pod agent-server; return may-proceed.++ Returns True when the workspace was captured, when there was nothing to+ capture, or when archiving failed but is not REQUIRED. Returns False only+ when archiving is REQUIRED and could not confirm a capture (the caller+ decides whether to block + retry). Never raises.+ """+ try:+ archive_path = await self._resolve_archive_path(+ stored_sandbox, conversation_id, workspace_path+ )+ # The runtime url is raw (localhost in Docker/local); transform it the+ # same way every other agent-server URL resolution does.+ runtime = dict(runtime_data)+ url = runtime.get('url')+ if url:+ runtime['url'] = replace_localhost_hostname_for_docker(url)+ return await workspace_archive.archive_workspace(+ self.httpx_client,+ runtime,+ stored_sandbox.id,+ archive_path=archive_path,+ conversation_id=conversation_id,+ )+ except Exception:+ # Could not resolve the workspace layout: never archive to the wrong+ # path. Honor REQUIRED (block + retry) vs best-effort (proceed).+ _logger.exception(+ 'Could not resolve archive path for %s', stored_sandbox.id+ )+ return not workspace_archive.archive_required()++ async def archive_conversation_workspace(+ self,+ sandbox_id: str,+ conversation_id: str | None = None,+ workspace_path: str | None = None,+ ) -> bool:+ """Archive ONE conversation's workspace; return whether delete may proceed.++ The sole app-server capture path: the conversation-delete finalizer calls+ this for every conversation delete (while the runtime is still up), then+ tears the sandbox down only when this was its last conversation. Keying to+ the conversation lets a grouped sandbox capture the right per-conversation+ repo, and means no grouped conversation's work is lost when a sibling later+ triggers the sandbox delete.++ ``workspace_path`` is the path pinned at conversation creation; when given+ the capture uses it verbatim instead of re-deriving the layout.++ Returns True when the workspace was captured, when there was nothing to+ capture (runtime already gone, or no repo at the path), or when archiving+ failed but is not REQUIRED. Returns False only when archiving is REQUIRED+ and could not confirm a capture, so the finalizer keeps the sandbox ++ running runtime for the runtime-api idle reap (the durability backstop).+ Never raises. No-op (returns True) unless archiving is enabled.+ """+ if not workspace_archive.archive_enabled():+ return True+ try:+ stored_sandbox = await self._get_stored_sandbox(sandbox_id)+ if not stored_sandbox:+ return True+ runtime_data = await self._get_runtime(sandbox_id)+ except httpx.HTTPStatusError as e:+ if e.response.status_code == 404:+ # Runtime already gone: nothing to capture for this conversation.+ return True+ # Couldn't reach the runtime: honor REQUIRED (block + keep) vs+ # best-effort (let the delete proceed; delete_sandbox re-checks).+ _logger.exception(+ 'Workspace archive lookup failed for %s (%s)',+ sandbox_id,+ conversation_id,+ )+ return not workspace_archive.archive_required()+ except Exception:+ _logger.exception(+ 'Workspace archive lookup failed for %s (%s)',+ sandbox_id,+ conversation_id,+ )+ return not workspace_archive.archive_required()+ archived = await self._archive_workspace(+ stored_sandbox, conversation_id, runtime_data, workspace_path+ )+ if not archived:+ _logger.warning(+ 'Workspace archive required but failed for %s (%s); keeping the '+ 'sandbox for the idle reap to capture',+ sandbox_id,+ conversation_id,+ )+ return archivedasync def pause_old_sandboxes(self, max_num_sandboxes: int) -> list[str]:"""Pause the oldest running sandboxes until at most max_num_sandboxes remain.diff --git a/openhands/app_server/sandbox/sandbox_router.py b/openhands/app_server/sandbox/sandbox_router.pyindex 7ad0fcd0d5..b4b42aea79 100644--- a/openhands/app_server/sandbox/sandbox_router.py+++ b/openhands/app_server/sandbox/sandbox_router.py@@ -110,6 +110,9 @@ async def delete_sandbox(sandbox_id: str,sandbox_service: SandboxService = sandbox_service_dependency,) -> Success:+ # delete_sandbox is sandbox-scoped (stop + delete) and never archives, so this+ # request handler can't block on a minutes-long capture. Workspace capture is+ # owned by the conversation-delete finalizer and the runtime-api idle reaper.exists = await sandbox_service.delete_sandbox(sandbox_id)if not exists:raise HTTPException(status.HTTP_404_NOT_FOUND)diff --git a/openhands/app_server/sandbox/sandbox_service.py b/openhands/app_server/sandbox/sandbox_service.pyindex e372c7a5a0..3fc0655865 100644--- a/openhands/app_server/sandbox/sandbox_service.py+++ b/openhands/app_server/sandbox/sandbox_service.py@@ -197,8 +197,28 @@ class SandboxService(ABC):async def delete_sandbox(self, sandbox_id: str) -> bool:"""Begin the process of deleting a sandbox (which may involve stopping it).- Return False if the sandbox did not exist.+ Return False if the sandbox did not exist. Purely sandbox-scoped (stop the+ runtime, delete the record); workspace capture is a separate+ conversation-scoped step (``archive_conversation_workspace``) the+ conversation-delete finalizer runs before the sandbox is torn down.+ """++ async def archive_conversation_workspace(+ self,+ sandbox_id: str,+ conversation_id: str | None = None,+ workspace_path: str | None = None,+ ) -> bool:+ """Archive one conversation's workspace; return whether delete may proceed.++ Default no-op (returns True) for backends that do not archive; overridden+ by RemoteSandboxService. The conversation-delete finalizer calls this+ before ``delete_sandbox`` so the workspace is captured while the runtime is+ still up. ``workspace_path`` is the path pinned at creation. Returns False+ only when archiving is REQUIRED and failed, so the caller leaves the+ sandbox up for a later (idle-reap) capture."""+ return Trueasync def pause_old_sandboxes(self, max_num_sandboxes: int) -> list[str]:"""Pause the oldest sandboxes if there are more than max_num_sandboxes running.diff --git a/openhands/app_server/sandbox/workspace_archive.py b/openhands/app_server/sandbox/workspace_archive.pynew file mode 100644index 0000000000..d091b20115--- /dev/null+++ b/openhands/app_server/sandbox/workspace_archive.py@@ -0,0 +1,524 @@+"""Archive a remote sandbox's workspace to object storage before deletion.++Pulls a workspace archive from the in-pod agent-server endpoint+(``GET /api/file/archive``) and stores it, plus a small manifest, in object+storage so the agent's work β production OpenHands Cloud workspace state β+survives sandbox deletion and is preserved for downstream use (e.g. dataset/eval+creation).++It covers the *explicit-delete-while-running* path. The dominant idle/expiry+reap is handled separately in runtime-api at pause time, because that deletion+never reaches the app-server.++Configuration is environment-driven and the feature is a no-op unless+``RUNTIME_FILE_ARCHIVE_ENABLED`` is set.+"""++import asyncio+import json+import logging+import os+import tempfile+from typing import Any++import httpx++from openhands.agent_server.utils import utc_now+from openhands.app_server.file_store import get_file_store+from openhands.app_server.file_store.files import FileStore++_logger = logging.getLogger(__name__)++# Formats the SDK GET /api/file/archive producer accepts (git-delta | tar.gz);+# anything else 422s, so validate before issuing the request.+_ARCHIVE_SUFFIX = {'git-delta': 'patch', 'tar.gz': 'tar.gz'}+++def _archive_request_params(path: str, fmt: str) -> dict[str, str]:+ """Query params for GET /api/file/archive.++ The tar.gz is the SELF-CONTAINED full capture, so disable the endpoint's+ default excludes for it: otherwise agent output under dist/build/node_modules+ and the repo's .git history are dropped and it is no more complete than the+ git-delta (defeating the whole point of capturing 'both'). Credential-bearing+ git internals are still scrubbed server-side even with excludes off. git-delta+ keeps the defaults β it is the compact companion, not the full capture.+ """+ params = {'path': path, 'format': fmt}+ if fmt == 'tar.gz':+ params['use_default_excludes'] = 'false'+ return params+++def archive_enabled() -> bool:+ return os.getenv('RUNTIME_FILE_ARCHIVE_ENABLED', 'false').lower() in ('true', '1')+++def archive_required() -> bool:+ """When true, an archive failure blocks deletion so it can be retried."""+ return os.getenv('RUNTIME_FILE_ARCHIVE_REQUIRED', 'false').lower() in ('true', '1')+++def _archive_bucket() -> str:+ return os.getenv('RUNTIME_FILE_ARCHIVE_BUCKET', '')+++def _archive_prefix() -> str:+ return os.getenv('RUNTIME_FILE_ARCHIVE_PREFIX', 'workspace-archives')+++def _archive_format() -> str:+ # Default to 'both' β the compact git-delta AND a self-contained full tar.gz.+ # git-delta alone is lossy as a sole capture: it respects the repo's+ # .gitignore (so agent-authored gitignored files are dropped) and needs the+ # base tree to reconstruct, whereas the tar.gz is self-contained and captures+ # those files. Keep both until the storage cost is measured, then narrow to+ # 'git-delta' (+ bucket lifecycle) if warranted (infra#1444).+ return os.getenv('RUNTIME_FILE_ARCHIVE_FORMAT', 'both')+++def _formats_to_capture() -> list[str] | None:+ """Resolve RUNTIME_FILE_ARCHIVE_FORMAT to the list of formats to upload.++ 'both' captures the git-delta AND the full tar.gz; a single format captures+ just that one. Returns None for an unsupported value (a hard config error the+ SDK producer would 422), so the caller can log + skip instead of mis-reading+ it as "nothing to archive".+ """+ fmt = _archive_format()+ if fmt == 'both':+ return ['git-delta', 'tar.gz']+ if fmt in _ARCHIVE_SUFFIX:+ return [fmt]+ return None+++def _float_env(name: str, default: float) -> float:+ """Parse a float env var, falling back to default on a non-numeric value.++ A bad override (``'120s'``, a stray newline) must not raise on every archive+ call β that would wedge every REQUIRED delete forever.+ """+ raw = os.getenv(name)+ if not raw:+ return default+ try:+ value = float(raw)+ except ValueError:+ _logger.warning('Invalid %s=%r; using %s', name, raw, default)+ return default+ if value <= 0:+ # A non-positive timeout/deadline would make httpx raise on every archive+ # (the wedge this guard exists to prevent); fall back to the safe default.+ _logger.warning('Non-positive %s=%r; using %s', name, raw, default)+ return default+ return value+++def _archive_timeout() -> float:+ # Must cover the agent-server git build budget (read-tree 60 + add 300 + diff+ # 300 = up to ~660s) before the first response byte flows, or large repos+ # ReadTimeout and never capture. The final archive runs in the detached+ # delete finalizer, so a long wait here doesn't block any user request; the+ # initial snapshot is separately bounded by initial_archive_deadline().+ return _float_env('RUNTIME_FILE_ARCHIVE_TIMEOUT', 660.0)+++def _archive_store_type() -> str:+ # Default to GCS to preserve current behavior; local/s3 also work. NOT+ # 'memory' β it is text-only (read() returns str) and would corrupt the+ # binary archive (see InMemoryFileStore).+ return os.getenv('RUNTIME_FILE_ARCHIVE_STORE_TYPE', 'google_cloud')+++def _get_archive_file_store() -> FileStore:+ """Object store for archives, built via the backend-portable factory."""+ return get_file_store(_archive_store_type(), _archive_bucket())+++def _cleanup_tempfile(path: str | None) -> None:+ if not path:+ return+ try:+ os.unlink(path)+ except OSError:+ pass+++async def _stream_to_tempfile(response: Any) -> tuple[str, int]:+ """Stream a 200 response body to a temp file; return (path, byte_count).++ Avoids buffering the whole archive in app-server RAM (OOM risk under+ concurrent large deletes). Cleans up its own file if streaming fails.+ """+ tmp = tempfile.NamedTemporaryFile(delete=False)+ byte_count = 0+ try:+ async for chunk in response.aiter_bytes():+ tmp.write(chunk)+ byte_count += len(chunk)+ tmp.close()+ return tmp.name, byte_count+ except BaseException:+ tmp.close()+ _cleanup_tempfile(tmp.name)+ raise+++def _write_file_to_store(store: FileStore, name: str, path: str) -> None:+ """Stream a temp file to the store, never buffering the whole archive in RAM.++ The download was streamed to a tempfile precisely to avoid holding the+ archive in memory (see ``_stream_to_tempfile``); ``write_from_path`` keeps+ that guarantee on upload (GCS/local/S3 stream from disk) instead of reading+ the whole file back with ``store.write(name, f.read())``.+ """+ store.write_from_path(name, path)+++async def archive_workspace(+ httpx_client: httpx.AsyncClient,+ runtime: dict[str, Any],+ sandbox_id: str,+ *,+ archive_path: str,+ conversation_id: str | None = None,+) -> bool:+ """Archive the workspace at ``archive_path``; return whether delete may proceed.++ ``archive_path`` is resolved by the caller β the path pinned at conversation+ creation, NOT re-derived here from live settings β so a capture can never be+ misrouted to the wrong directory. The agent-server descends from it to the+ cloned repo.++ Returns True when the workspace was archived, when the path holds nothing to+ archive (agent-server 400: not a directory / not a git repo), or when+ archiving failed but is not required (best-effort). Returns False when+ archiving is required and either hit a transient failure (5xx / network /+ 422 / 429) or could not confirm a capture (401 auth / 404 missing path), so+ the caller leaves the sandbox intact for the idle-reap retry. Never raises.++ A pure configuration error (unsupported RUNTIME_FILE_ARCHIVE_FORMAT, or+ RUNTIME_FILE_ARCHIVE_BUCKET unset) cannot be fixed by retrying, so it is+ logged loudly and the delete is allowed to proceed rather than wedging every+ delete forever when archiving is required.+ """+ agent_server_url = runtime.get('url')+ session_api_key = runtime.get('session_api_key')+ if not agent_server_url:+ _logger.warning(+ 'Workspace archive skipped for %s: runtime has no agent-server URL',+ sandbox_id,+ )+ return not archive_required()+ if not _archive_bucket():+ # Misconfiguration, not a transient failure: no amount of retrying makes+ # a missing bucket appear. Proceed (with a loud error) so a+ # REQUIRED-without-bucket setup does not block every sandbox delete.+ _logger.error(+ 'Workspace archive enabled for %s but RUNTIME_FILE_ARCHIVE_BUCKET '+ 'is not set; proceeding with delete (fix the config to capture)',+ sandbox_id,+ )+ return True++ formats = _formats_to_capture()+ if formats is None:+ # Unsupported RUNTIME_FILE_ARCHIVE_FORMAT is a pure config error, exactly+ # like the unset bucket above: no retry makes a valid format appear, so+ # proceed loudly rather than wedging every REQUIRED delete forever (the+ # app-server has no idle-reap backstop). Validated here so a bad format+ # never reaches the producer (which would 422 it).+ _logger.error(+ 'Workspace archive for %s: unsupported RUNTIME_FILE_ARCHIVE_FORMAT '+ '%r (valid: %s); proceeding with delete (fix the config to capture)',+ sandbox_id,+ _archive_format(),+ ['git-delta', 'tar.gz', 'both'],+ )+ return True++ headers = {'X-Session-API-Key': session_api_key} if session_api_key else {}+ # For cloud conversations the sandbox id is the conversation_id.hex.+ conversation_key = conversation_id or sandbox_id+ ts = utc_now().strftime('%Y%m%dT%H%M%SZ')+ # Key by conversation, not just sandbox: under grouping a sandbox is shared by+ # siblings, and the 1s ts is not unique β without the conversation segment two+ # sibling captures in the same second overwrite each other at the object level.+ base_path = f'{_archive_prefix()}/{sandbox_id}/{conversation_key}/{ts}'++ # 'both' uploads each format under its own suffix ({ts}.patch + {ts}.tar.gz),+ # each with its own manifest. base_commit only rides the git-delta response+ # header, so capture it there and reuse it for the tar.gz manifest.+ # A retry under REQUIRED re-uploads under a fresh {ts}; any blob/manifest left+ # by a partially-failed prior attempt becomes an orphan reaped by the bucket+ # lifecycle policy (we favor capture completeness over upload dedup).+ retryable_failure = False+ # A capture we could not confirm happened (401 auth / 404 missing path):+ # under REQUIRED this must NOT permit teardown β it is the misrouted-path+ # symptom this feature most needs to guard against, not "nothing to archive".+ unconfirmed_capture = False+ base_commit = ''+ # One store per call (not per format) β building it lazily spins up a client.+ store = _get_archive_file_store()+ for fmt in formats:+ suffix = _ARCHIVE_SUFFIX[fmt]+ tmp_path: str | None = None+ byte_count = 0+ try:+ async with httpx_client.stream(+ 'GET',+ f'{agent_server_url}/api/file/archive',+ params=_archive_request_params(archive_path, fmt),+ headers=headers,+ timeout=_archive_timeout(),+ ) as response:+ if response.status_code != 200:+ code = response.status_code+ if code == 400:+ # Path exists but holds no archivable repo (not a git+ # repo / not a directory). A positive "nothing here" β+ # safe to skip this format and proceed.+ detail = 'nothing to archive'+ elif code in (401, 404):+ # 401 auth rejected / 404 path missing: the capture did+ # NOT happen and this is not a confirmed-empty workspace,+ # so it must block a REQUIRED delete (idle reap retries).+ unconfirmed_capture = True+ detail = 'capture unconfirmed (auth/path)'+ else:+ # 422 / 429 / 5xx β transient.+ retryable_failure = True+ detail = 'retryable failure'+ _logger.warning(+ 'Workspace archive (%s) for %s: agent-server returned %s; %s',+ fmt,+ sandbox_id,+ code,+ detail,+ )+ continue+ header_base = response.headers.get('X-Archive-Base-Commit', '')+ if header_base:+ base_commit = header_base+ # Stream to disk so the archive never sits whole in RAM.+ tmp_path, byte_count = await _stream_to_tempfile(response)+ except Exception as e:+ # Network/timeout error: genuinely transient.+ _logger.warning(+ 'Workspace archive fetch (%s) failed for %s: %s', fmt, sandbox_id, e+ )+ retryable_failure = True+ _cleanup_tempfile(tmp_path)+ continue++ assert tmp_path is not None # set on the 200 path above+ try:+ await asyncio.to_thread(+ _write_file_to_store, store, f'{base_path}.{suffix}', tmp_path+ )+ manifest = json.dumps(+ {+ 'sandbox_id': sandbox_id,+ 'conversation_id': conversation_key,+ 'phase': 'final',+ 'base_commit': base_commit,+ 'format': fmt,+ 'source_path': archive_path,+ 'byte_count': byte_count,+ 'created_at': ts,+ },+ sort_keys=True,+ ).encode('utf-8')+ await asyncio.to_thread(+ store.write, f'{base_path}.{suffix}.manifest.json', manifest+ )+ _logger.info(+ 'Archived workspace (%s) for %s (%d bytes) to %s.%s',+ fmt,+ sandbox_id,+ byte_count,+ base_path,+ suffix,+ )+ except Exception as e:+ _logger.exception(+ 'Workspace archive upload (%s) failed for %s: %s', fmt, sandbox_id, e+ )+ retryable_failure = True+ finally:+ _cleanup_tempfile(tmp_path)++ # Deletion may proceed unless archiving is REQUIRED and we either hit a+ # retryable failure or could not confirm a capture (401/404) β both leave us+ # short of the data we were meant to preserve.+ if archive_required() and (retryable_failure or unconfirmed_capture):+ return False+ return True+++def initial_archive_enabled() -> bool:+ """Whether to capture the workspace's INITIAL state (before the first step).++ Independent of ``RUNTIME_FILE_ARCHIVE_ENABLED`` (the delete/pause capture of+ the *final* state) so the pre-agent snapshot can be toggled on its own. Off+ by default β like every other capture knob, nothing happens until enabled.+ """+ return os.getenv('RUNTIME_FILE_ARCHIVE_INITIAL_ENABLED', 'false').lower() in (+ 'true',+ '1',+ )+++def initial_archive_deadline() -> float:+ """Hard ceiling (seconds) on how long the inline, pre-setup initial snapshot+ may delay conversation startup.++ The snapshot is awaited before setup.sh so it captures the repo exactly as+ cloned, with no concurrent mutation. ``wait_for`` only charges the time+ actually spent, so a fast snapshot adds no latency; this caps the worst case+ (a large repo or a hung endpoint) so it can never dominate startup. An overrun+ is logged and startup proceeds without the snapshot. A fresh clone's tar.gz is+ just the repo tree, so the default comfortably covers typical repos; raise it+ for a very large monorepo.+ """+ return _float_env('RUNTIME_FILE_ARCHIVE_INITIAL_DEADLINE', 120.0)+++def _initial_archive_format() -> str:+ """Format for the initial snapshot. Defaults to a self-contained tar.gz.++ At conversation start the working tree has no changes yet, so a ``git-delta``+ would be empty; a full ``tar.gz`` is the only format that captures anything+ and, unlike a delta keyed to ``base_commit``, it survives the upstream repo+ or branch later disappearing (the fragile re-clone path we want to avoid).+ """+ return os.getenv('RUNTIME_FILE_ARCHIVE_INITIAL_FORMAT', 'tar.gz')+++async def archive_initial_workspace(+ httpx_client: httpx.AsyncClient,+ *,+ agent_server_url: str | None,+ session_api_key: str | None,+ project_dir: str,+ sandbox_id: str,+ conversation_id: str | None = None,+ base_commit: str = '',+) -> bool:+ """Snapshot the workspace BEFORE the agent's first step; return success.++ Captures the repo exactly as cloned (option A β the pre- vs post-setup choice+ is the open design question tracked in All-Hands-AI/infra#1444) as a+ self-contained ``tar.gz`` plus a ``phase=initial`` manifest, so the snapshot+ records the true starting state even if the source repo later disappears.++ This is strictly best-effort: it never raises and never blocks conversation+ startup. A failure (feature off, misconfig, agent-server hiccup) just means no+ initial snapshot for this run, logged and swallowed. Returns True only when an+ archive was actually written.+ """+ if not initial_archive_enabled():+ return False+ if not agent_server_url:+ _logger.warning(+ 'Initial workspace archive skipped for %s: no agent-server URL',+ sandbox_id,+ )+ return False+ if not _archive_bucket():+ _logger.error(+ 'Initial workspace archive enabled for %s but '+ 'RUNTIME_FILE_ARCHIVE_BUCKET is not set; skipping initial snapshot',+ sandbox_id,+ )+ return False++ fmt = _initial_archive_format()+ if fmt not in _ARCHIVE_SUFFIX:+ _logger.error(+ 'Initial workspace archive for %s: unsupported '+ 'RUNTIME_FILE_ARCHIVE_INITIAL_FORMAT %r (valid: %s); skipping',+ sandbox_id,+ fmt,+ sorted(_ARCHIVE_SUFFIX),+ )+ return False+ suffix = _ARCHIVE_SUFFIX[fmt]+ headers = {'X-Session-API-Key': session_api_key} if session_api_key else {}++ tmp_path: str | None = None+ byte_count = 0+ try:+ async with httpx_client.stream(+ 'GET',+ f'{agent_server_url}/api/file/archive',+ params=_archive_request_params(project_dir, fmt),+ headers=headers,+ timeout=_archive_timeout(),+ ) as response:+ if response.status_code != 200:+ _logger.warning(+ 'Initial workspace archive for %s: agent-server returned %s; '+ 'no initial snapshot',+ sandbox_id,+ response.status_code,+ )+ return False+ # tar.gz carries no base-commit header (git-delta sets it); fall back+ # to the caller-provided HEAD sha so the snapshot still records the+ # commit it came from.+ captured_base = (+ response.headers.get('X-Archive-Base-Commit', '') or base_commit+ )+ # Stream to disk so the archive never sits whole in RAM.+ tmp_path, byte_count = await _stream_to_tempfile(response)+ except Exception as e:+ _logger.warning(+ 'Initial workspace archive fetch failed for %s: %s', sandbox_id, e+ )+ _cleanup_tempfile(tmp_path)+ return False++ assert tmp_path is not None # set on the 200 path above+ try:+ store = _get_archive_file_store()+ ts = utc_now().strftime('%Y%m%dT%H%M%SZ')+ conversation_key = conversation_id or sandbox_id+ # Key by conversation (siblings share a grouped sandbox) and nest under+ # /initial/ so it never collides with that conversation's final capture+ # ({prefix}/{sandbox_id}/{conversation_key}/{ts}).+ blob_name = (+ f'{_archive_prefix()}/{sandbox_id}/{conversation_key}/initial/{ts}.{suffix}'+ )+ await asyncio.to_thread(_write_file_to_store, store, blob_name, tmp_path)+ manifest = json.dumps(+ {+ 'sandbox_id': sandbox_id,+ 'conversation_id': conversation_key,+ 'phase': 'initial',+ 'base_commit': captured_base,+ 'format': fmt,+ 'source_path': project_dir,+ 'byte_count': byte_count,+ 'created_at': ts,+ },+ sort_keys=True,+ ).encode('utf-8')+ # Shared contract: manifest = blob + '.manifest.json' (was dropping the+ # format suffix, so a downstream enricher could never locate it).+ await asyncio.to_thread(store.write, f'{blob_name}.manifest.json', manifest)+ _logger.info(+ 'Archived INITIAL workspace for %s (%d bytes) to %s',+ sandbox_id,+ byte_count,+ blob_name,+ )+ return True+ except Exception as e:+ _logger.exception(+ 'Initial workspace archive upload failed for %s: %s', sandbox_id, e+ )+ return False+ finally:+ _cleanup_tempfile(tmp_path)diff --git a/openhands/app_server/settings/settings_models.py b/openhands/app_server/settings/settings_models.pyindex 82cc77ba43..6ccc3e3490 100644--- a/openhands/app_server/settings/settings_models.py+++ b/openhands/app_server/settings/settings_models.py@@ -92,6 +92,23 @@ class SandboxGroupingStrategy(str, Enum):ADD_TO_ANY = 'ADD_TO_ANY' # Add to any available sandbox (first found)+def grouped_workspace_dir(+ base_working_dir: str,+ grouping_strategy: SandboxGroupingStrategy,+ conversation_id_hex: str,+) -> str:+ """Workspace dir for a conversation given the grouping strategy.++ Single source of truth for the relocation used at conversation start and at+ archive time. Under any grouping strategy the workspace is nested under the+ conversation id so co-located conversations stay isolated; NO_GROUPING keeps+ the bare base dir.+ """+ if grouping_strategy == SandboxGroupingStrategy.NO_GROUPING:+ return base_working_dir+ return f'{base_working_dir}/{conversation_id_hex}'++# Fields the batch ``update()`` method refuses to touch:# - ``secrets_store`` is frozen (Pydantic would raise).# - ``llm_profiles`` is off-limits for the generic settings POST; profilediff --git a/tests/unit/app_server/test_app_conversation_router.py b/tests/unit/app_server/test_app_conversation_router.pyindex e8a41c2dc0..0f34843bbe 100644--- a/tests/unit/app_server/test_app_conversation_router.py+++ b/tests/unit/app_server/test_app_conversation_router.py@@ -21,6 +21,7 @@ from openhands.app_server.app_conversation.app_conversation_models import ()from openhands.app_server.app_conversation.app_conversation_router import (AgentServerContext,+ _finalize_sandbox_delete,batch_get_app_conversations,count_app_conversations,get_conversation_git_changes,@@ -1082,3 +1083,88 @@ class TestGitProxyEndpoints:assert exc_info.value.status_code == status.HTTP_409_CONFLICTassert 'paused' in exc_info.value.detail.lower()+++class TestFinalizeSandboxDelete:+ """The detached finalizer captures THIS conversation's workspace first+ (archive_conversation_workspace, while the runtime is still up), then tears the+ sandbox down only when no other conversation still references it (count == 0).+ delete_sandbox is sandbox-scoped β no conversation_id. A REQUIRED archive+ failure (archive returns False) keeps the sandbox for the idle reap."""++ def _deps(self, conversation_count: int, archived: bool = True):+ info_service = MagicMock()+ info_service.count_conversations_by_sandbox_id = AsyncMock(+ return_value=conversation_count+ )+ sandbox_service = MagicMock()+ sandbox_service.delete_sandbox = AsyncMock(return_value=True)+ sandbox_service.archive_conversation_workspace = AsyncMock(+ return_value=archived+ )+ db_session = AsyncMock()+ httpx_client = AsyncMock()+ return sandbox_service, info_service, db_session, httpx_client++ @pytest.mark.asyncio+ async def test_archives_then_deletes_when_unreferenced(self):+ sandbox_service, info_service, db_session, httpx_client = self._deps(0)+ conv = uuid4()+ await _finalize_sandbox_delete(+ sandbox_service, info_service, 'sbx-1', db_session, httpx_client, conv+ )+ # Workspace captured first (conversation-scoped), then the sandbox torn+ # down sandbox-scoped β note delete_sandbox takes no conversation_id.+ sandbox_service.archive_conversation_workspace.assert_awaited_once_with(+ 'sbx-1', conversation_id=conv.hex, workspace_path=None+ )+ sandbox_service.delete_sandbox.assert_awaited_once_with('sbx-1')+ db_session.commit.assert_awaited_once()++ @pytest.mark.asyncio+ async def test_archives_but_keeps_shared_sandbox(self):+ sandbox_service, info_service, db_session, httpx_client = self._deps(2)+ conv = uuid4()+ await _finalize_sandbox_delete(+ sandbox_service, info_service, 'sbx-1', db_session, httpx_client, conv+ )+ # Shared sandbox: still capture this conversation, but don't tear it down.+ sandbox_service.archive_conversation_workspace.assert_awaited_once_with(+ 'sbx-1', conversation_id=conv.hex, workspace_path=None+ )+ sandbox_service.delete_sandbox.assert_not_called()++ @pytest.mark.asyncio+ async def test_required_archive_failure_keeps_sandbox(self):+ # archive_conversation_workspace returns False (REQUIRED archive failed).+ sandbox_service, info_service, db_session, httpx_client = self._deps(+ 0, archived=False+ )+ conv = uuid4()+ await _finalize_sandbox_delete(+ sandbox_service, info_service, 'sbx-1', db_session, httpx_client, conv+ )+ # Sandbox + runtime kept for the idle reap: no count check, no delete.+ sandbox_service.archive_conversation_workspace.assert_awaited_once()+ info_service.count_conversations_by_sandbox_id.assert_not_called()+ sandbox_service.delete_sandbox.assert_not_called()++ @pytest.mark.asyncio+ async def test_forwards_pinned_workspace_path(self):+ sandbox_service, info_service, db_session, httpx_client = self._deps(0)+ conv = uuid4()+ await _finalize_sandbox_delete(+ sandbox_service,+ info_service,+ 'sbx-1',+ db_session,+ httpx_client,+ conv,+ workspace_path='/home/openhands/workspace/' + conv.hex,+ )+ # The path pinned at creation is forwarded to the capture verbatim.+ sandbox_service.archive_conversation_workspace.assert_awaited_once_with(+ 'sbx-1',+ conversation_id=conv.hex,+ workspace_path='/home/openhands/workspace/' + conv.hex,+ )diff --git a/tests/unit/app_server/test_live_status_app_conversation_service.py b/tests/unit/app_server/test_live_status_app_conversation_service.pyindex e7779887b9..cd55b843bd 100644--- a/tests/unit/app_server/test_live_status_app_conversation_service.py+++ b/tests/unit/app_server/test_live_status_app_conversation_service.py@@ -1893,7 +1893,9 @@ class TestLiveStatusAppConversationService:task.sandbox_id = self.mock_sandbox.idyield task- async def mock_run_setup_scripts(task, sandbox, workspace, agent_server_url):+ async def mock_run_setup_scripts(+ task, sandbox, workspace, agent_server_url, conversation_id+ ):yield taskself.service._wait_for_sandbox_start = mock_wait_for_sandbox@@ -1997,7 +1999,9 @@ class TestLiveStatusAppConversationService:task.sandbox_id = self.mock_sandbox.idyield task- async def mock_run_setup_scripts(task, sandbox, workspace, agent_server_url):+ async def mock_run_setup_scripts(+ task, sandbox, workspace, agent_server_url, conversation_id+ ):yield taskself.service._wait_for_sandbox_start = mock_wait_for_sandboxdiff --git a/tests/unit/app_server/test_remote_sandbox_service.py b/tests/unit/app_server/test_remote_sandbox_service.pyindex 6e538677d4..209609764f 100644--- a/tests/unit/app_server/test_remote_sandbox_service.py+++ b/tests/unit/app_server/test_remote_sandbox_service.py@@ -12,6 +12,7 @@ This module tests the RemoteSandboxService implementation, focusing on:"""import asyncio+import jsonfrom contextlib import ExitStack, asynccontextmanagerfrom datetime import datetime, timezonefrom typing import Any@@ -22,7 +23,7 @@ import httpximport pytestfrom sqlalchemy.ext.asyncio import AsyncSession-from openhands.app_server.errors import SandboxError+from openhands.app_server.errors import SandboxDeleteRetryError, SandboxErrorfrom openhands.app_server.sandbox.remote_sandbox_service import (ALLOW_CORS_ORIGINS_VARIABLE,STATUS_MAPPING,@@ -39,6 +40,7 @@ from openhands.app_server.sandbox.sandbox_models import (SandboxStatus,)from openhands.app_server.sandbox.sandbox_spec_models import SandboxSpecInfo+from openhands.app_server.settings.settings_models import SandboxGroupingStrategyfrom openhands.app_server.user.user_context import UserContext@@ -97,6 +99,39 @@ def remote_sandbox_service()+def _make_stream_response(+ status_code: int, content: bytes = b'', headers: dict | None = None+):+ """A fake httpx streaming response: status/headers + async aiter_bytes()."""+ resp = MagicMock()+ resp.status_code = status_code+ resp.headers = headers or {}++ async def _aiter_bytes():+ yield content++ resp.aiter_bytes = _aiter_bytes+ return resp+++def _stream_client(resp_or_map):+ """AsyncMock httpx client whose .stream(...) yields a fake response.++ ``resp_or_map`` is a single fake response or a ``{format: response}`` map.+ """+ client = AsyncMock()++ @asynccontextmanager+ async def _stream(method, url, **kwargs):+ if isinstance(resp_or_map, dict):+ yield resp_or_map[kwargs['params']['format']]+ else:+ yield resp_or_map++ client.stream = MagicMock(side_effect=_stream)+ return client++def create_runtime_data(session_id: str = 'test-sandbox-123',status: str = 'running',@@ -633,7 +668,9 @@ class TestSandboxLifecycle:# Verifyassert result is Trueremote_sandbox_service.db_session.delete.assert_called_once_with(stored_sandbox)- remote_sandbox_service.db_session.commit.assert_not_called()+ # delete_sandbox no longer commits internally: the session key is dropped+ # atomically with the row delete, which the caller commits.+ remote_sandbox_service.db_session.commit.assert_not_awaited()remote_sandbox_service.httpx_client.request.assert_called_once_with('POST','https://api.example.com/stop',@@ -641,6 +678,52 @@ class TestSandboxLifecycle:json={'runtime_id': 'runtime-456'},)+ @pytest.mark.asyncio+ async def test_archive_conversation_workspace_archives_without_stopping(+ self, remote_sandbox_service+ ):+ """A grouped, non-terminal conversation's workspace is captured without+ stopping the shared sandbox (no /stop, no row delete)."""+ stored_sandbox = create_stored_sandbox()+ runtime_data = create_runtime_data()+ remote_sandbox_service._get_stored_sandbox = AsyncMock(+ return_value=stored_sandbox+ )+ remote_sandbox_service._get_runtime = AsyncMock(return_value=runtime_data)+ remote_sandbox_service.db_session.delete = AsyncMock()+ remote_sandbox_service.httpx_client.request = AsyncMock()++ with patch(+ 'openhands.app_server.sandbox.remote_sandbox_service.workspace_archive'+ ) as mock_wa:+ mock_wa.archive_enabled.return_value = True+ mock_wa.archive_workspace = AsyncMock(return_value=True)+ ok = await remote_sandbox_service.archive_conversation_workspace(+ 'test-sandbox-123', conversation_id='conv-1'+ )++ assert ok is True+ mock_wa.archive_workspace.assert_awaited_once()+ # Captured, but the shared sandbox was neither stopped nor deleted.+ remote_sandbox_service.httpx_client.request.assert_not_called()+ remote_sandbox_service.db_session.delete.assert_not_called()++ @pytest.mark.asyncio+ async def test_archive_conversation_workspace_noop_when_disabled(+ self, remote_sandbox_service+ ):+ """No-op (returns True) when archiving is disabled β no lookups at all."""+ remote_sandbox_service._get_stored_sandbox = AsyncMock()+ with patch(+ 'openhands.app_server.sandbox.remote_sandbox_service.workspace_archive'+ ) as mock_wa:+ mock_wa.archive_enabled.return_value = False+ ok = await remote_sandbox_service.archive_conversation_workspace(+ 'test-sandbox-123', conversation_id='conv-1'+ )+ assert ok is True+ remote_sandbox_service._get_stored_sandbox.assert_not_called()+@pytest.mark.asyncioasync def test_delete_sandbox_runtime_not_found_ignored(self, remote_sandbox_service@@ -667,6 +750,87 @@ class TestSandboxLifecycle:# Verifyassert result is True # 404 should be ignored for delete operations+ @pytest.mark.asyncio+ async def test_delete_sandbox_runtime_gone_deletes_record(+ self, remote_sandbox_service+ ):+ """_get_runtime 404 (runtime already gone) still deletes the DB row."""+ stored_sandbox = create_stored_sandbox()++ remote_sandbox_service._get_stored_sandbox = AsyncMock(+ return_value=stored_sandbox+ )+ not_found = httpx.HTTPStatusError(+ 'Not Found',+ request=httpx.Request('GET', 'https://api.example.com/sessions/x'),+ response=httpx.Response(404),+ )+ remote_sandbox_service._get_runtime = AsyncMock(side_effect=not_found)+ remote_sandbox_service.db_session.delete = AsyncMock()++ result = await remote_sandbox_service.delete_sandbox('test-sandbox-123')++ assert result is True+ remote_sandbox_service.db_session.delete.assert_called_once_with(stored_sandbox)+ # Nothing to stop when the runtime is already gone.+ remote_sandbox_service.httpx_client.request.assert_not_called()++ @pytest.mark.asyncio+ async def test_delete_sandbox_runtime_transient_error_keeps_record(+ self, remote_sandbox_service+ ):+ """A transient (5xx) _get_runtime error leaves the row for a retry."""+ stored_sandbox = create_stored_sandbox()++ remote_sandbox_service._get_stored_sandbox = AsyncMock(+ return_value=stored_sandbox+ )+ server_error = httpx.HTTPStatusError(+ 'Server Error',+ request=httpx.Request('GET', 'https://api.example.com/sessions/x'),+ response=httpx.Response(503),+ )+ remote_sandbox_service._get_runtime = AsyncMock(side_effect=server_error)+ remote_sandbox_service.db_session.delete = AsyncMock()++ # Sandbox still exists -> raise (router 503), keep the row for retry;+ # never report 404.+ with pytest.raises(SandboxDeleteRetryError):+ await remote_sandbox_service.delete_sandbox('test-sandbox-123')+ remote_sandbox_service.db_session.delete.assert_not_called()++ @pytest.mark.asyncio+ async def test_delete_sandbox_transient_error_invalidates_session_key(+ self, remote_sandbox_service+ ):+ """A transient runtime error keeps the row + running runtime for retry, but+ the session key is invalidated UP FRONT and committed β a delete is often a+ revoke of a leaked key, so the key must die promptly and the caller's+ rollback on the raised error must not be able to resurrect it."""+ stored_sandbox = create_stored_sandbox(session_api_key_hash='live-hash')++ remote_sandbox_service._get_stored_sandbox = AsyncMock(+ return_value=stored_sandbox+ )+ remote_sandbox_service._get_runtime = AsyncMock(+ side_effect=httpx.HTTPStatusError(+ 'Server Error',+ request=httpx.Request('GET', 'https://api.example.com/sessions/x'),+ response=httpx.Response(503),+ )+ )+ remote_sandbox_service.db_session.delete = AsyncMock()+ remote_sandbox_service.db_session.commit = AsyncMock()++ with pytest.raises(SandboxDeleteRetryError):+ await remote_sandbox_service.delete_sandbox('test-sandbox-123')++ # Row kept for retry, but the key is dead and that has been persisted so a+ # rollback cannot bring it back.+ remote_sandbox_service.db_session.delete.assert_not_called()+ assert stored_sandbox.session_api_key_hash is None+ remote_sandbox_service.db_session.commit.assert_awaited_once()+class TestSandboxSearch:"""Test cases for sandbox search and retrieval."""@@ -981,11 +1145,9 @@ class TestErrorHandling:'API Error')- # Execute- result = await remote_sandbox_service.delete_sandbox('test-sandbox-123')-- # Verify- assert result is False+ # A transient runtime error on a live sandbox raises (router 503), not 404.+ with pytest.raises(SandboxDeleteRetryError):+ await remote_sandbox_service.delete_sandbox('test-sandbox-123')class TestGetSandboxBySessionApiKey:@@ -1768,3 +1930,985 @@ class TestBatchGetSandboxes:assert results[0] is not Noneassert results[0].id == 'sandbox-1'assert results[0].status == SandboxStatus.MISSING+++class TestArchiveConversationWorkspace:+ """archive_conversation_workspace contract (APP-2403): the sole app-server+ capture path, run by the conversation-delete finalizer BEFORE delete_sandbox+ (which is now purely sandbox-scoped). Its bool return gates the finalizer β+ False (REQUIRED + retryable failure) keeps the sandbox for the idle reap; True+ lets the delete proceed. It never stops the runtime or deletes the row."""++ @pytest.mark.asyncio+ async def test_required_failure_returns_false(self, remote_sandbox_service):+ """A REQUIRED archive failure returns False (and never stops/deletes), so+ the finalizer keeps the sandbox + runtime for the idle reap."""+ stored = create_stored_sandbox()+ remote_sandbox_service._get_stored_sandbox = AsyncMock(return_value=stored)+ remote_sandbox_service._get_runtime = AsyncMock(+ return_value=create_runtime_data()+ )+ remote_sandbox_service.db_session.delete = AsyncMock()+ remote_sandbox_service.httpx_client.request = AsyncMock()++ with (+ patch(+ 'openhands.app_server.sandbox.workspace_archive.archive_enabled',+ return_value=True,+ ),+ patch(+ 'openhands.app_server.sandbox.workspace_archive.archive_workspace',+ new=AsyncMock(return_value=False),+ ),+ ):+ ok = await remote_sandbox_service.archive_conversation_workspace(+ 'test-sandbox-123', conversation_id='conv-1'+ )++ assert ok is False+ remote_sandbox_service.httpx_client.request.assert_not_called()+ remote_sandbox_service.db_session.delete.assert_not_called()++ @pytest.mark.asyncio+ async def test_nothing_to_archive_returns_true(self, remote_sandbox_service):+ """Nothing to capture (archive_workspace returns True for a 400/404) lets+ the delete proceed even under REQUIRED, so the sandbox is never orphaned."""+ stored = create_stored_sandbox()+ remote_sandbox_service._get_stored_sandbox = AsyncMock(return_value=stored)+ remote_sandbox_service._get_runtime = AsyncMock(+ return_value=create_runtime_data()+ )++ with (+ patch(+ 'openhands.app_server.sandbox.workspace_archive.archive_enabled',+ return_value=True,+ ),+ patch(+ 'openhands.app_server.sandbox.workspace_archive.archive_workspace',+ new=AsyncMock(return_value=True),+ ),+ ):+ ok = await remote_sandbox_service.archive_conversation_workspace(+ 'test-sandbox-123', conversation_id='conv-1'+ )++ assert ok is True++ @pytest.mark.asyncio+ async def test_runtime_already_gone_returns_true(self, remote_sandbox_service):+ """Runtime already gone (404): nothing to capture, so the delete proceeds."""+ stored = create_stored_sandbox()+ remote_sandbox_service._get_stored_sandbox = AsyncMock(return_value=stored)+ remote_sandbox_service._get_runtime = AsyncMock(+ side_effect=httpx.HTTPStatusError(+ 'gone',+ request=httpx.Request('GET', 'https://api.example.com/x'),+ response=httpx.Response(404),+ )+ )+ with patch(+ 'openhands.app_server.sandbox.workspace_archive.archive_enabled',+ return_value=True,+ ):+ ok = await remote_sandbox_service.archive_conversation_workspace(+ 'test-sandbox-123', conversation_id='conv-1'+ )+ assert ok is True++ @pytest.mark.asyncio+ async def test_pinned_path_forwarded_to_archive(self, remote_sandbox_service):+ """A pinned workspace_path is forwarded to archive_workspace verbatim and+ no grouping/spec re-derivation happens."""+ stored = create_stored_sandbox()+ remote_sandbox_service._get_stored_sandbox = AsyncMock(return_value=stored)+ remote_sandbox_service._get_runtime = AsyncMock(+ return_value=create_runtime_data()+ )+ with (+ patch(+ 'openhands.app_server.sandbox.workspace_archive.archive_enabled',+ return_value=True,+ ),+ patch(+ 'openhands.app_server.sandbox.workspace_archive.archive_workspace',+ new=AsyncMock(return_value=True),+ ) as mock_archive,+ ):+ ok = await remote_sandbox_service.archive_conversation_workspace(+ 'test-sandbox-123',+ conversation_id='conv-1',+ workspace_path='/home/openhands/workspace/conv-1',+ )+ assert ok is True+ _, kwargs = mock_archive.call_args+ assert kwargs['archive_path'] == '/home/openhands/workspace/conv-1'+ remote_sandbox_service.sandbox_spec_service.get_sandbox_spec.assert_not_called()++ @pytest.mark.asyncio+ async def test_resolve_archive_path_uses_pinned(self, remote_sandbox_service):+ """A pinned path is returned verbatim β no spec/settings lookup, so a+ grouping toggle since creation can never misroute the capture."""+ stored = create_stored_sandbox()+ path = await remote_sandbox_service._resolve_archive_path(+ stored, 'conv-1', '/home/openhands/workspace/conv-1'+ )+ assert path == '/home/openhands/workspace/conv-1'+ remote_sandbox_service.sandbox_spec_service.get_sandbox_spec.assert_not_called()++ @pytest.mark.asyncio+ async def test_resolve_archive_path_legacy_grouped(self, remote_sandbox_service):+ """No pinned path (pre-pinning conversation): rebuilt from the sandbox+ spec's working_dir nested under the conversation id."""+ stored = create_stored_sandbox()+ remote_sandbox_service.user_context.get_user_info = AsyncMock(+ return_value=MagicMock(+ sandbox_grouping_strategy=SandboxGroupingStrategy.GROUP_BY_NEWEST+ )+ )+ path = await remote_sandbox_service._resolve_archive_path(+ stored, 'deadbeef', None+ )+ # mock_sandbox_spec_service fixture pins working_dir='/workspace/project'.+ assert path == '/workspace/project/deadbeef'++ @pytest.mark.asyncio+ async def test_resolve_archive_path_legacy_no_grouping(+ self, remote_sandbox_service+ ):+ """NO_GROUPING fallback keeps the bare spec working_dir."""+ stored = create_stored_sandbox()+ remote_sandbox_service.user_context.get_user_info = AsyncMock(+ return_value=MagicMock(+ sandbox_grouping_strategy=SandboxGroupingStrategy.NO_GROUPING+ )+ )+ path = await remote_sandbox_service._resolve_archive_path(+ stored, 'deadbeef', None+ )+ assert path == '/workspace/project'+++class TestArchiveWorkspaceHelper:+ """Unit tests for the workspace_archive helper."""++ @pytest.mark.asyncio+ async def test_archive_uploads_archive_and_manifest(self, monkeypatch):+ from openhands.app_server.sandbox import workspace_archive++ monkeypatch.setenv('RUNTIME_FILE_ARCHIVE_BUCKET', 'archive-bkt')+ monkeypatch.setenv('RUNTIME_FILE_ARCHIVE_FORMAT', 'git-delta')++ client = _stream_client(+ _make_stream_response(+ 200, b'patch-bytes', {'X-Archive-Base-Commit': 'abc123'}+ )+ )+ store = MagicMock()+ writes: dict[str, bytes] = {}+ store.write.side_effect = lambda path, data: writes.__setitem__(path, data)+ # Archive blobs are streamed from a tempfile via write_from_path (the OOM+ # fix); record them too so blob-path assertions still see the .patch/.tar.gz.+ store.write_from_path.side_effect = lambda path, src: writes.__setitem__(+ path, open(src, 'rb').read()+ )++ with patch.object(+ workspace_archive, '_get_archive_file_store', return_value=store+ ):+ ok = await workspace_archive.archive_workspace(+ client,+ create_runtime_data(),+ 'sandbox-1',+ archive_path='/workspace/project',+ conversation_id='conv-1',+ )++ assert ok is True+ # session key forwarded; the given archive_path is used, format passed.+ _, kwargs = client.stream.call_args+ assert kwargs['headers']['X-Session-API-Key'] == 'test-session-key'+ assert kwargs['params'] == {+ 'path': '/workspace/project',+ 'format': 'git-delta',+ }+ # archive + manifest written.+ assert any(p.endswith('.patch') for p in writes)+ manifest_path = next(p for p in writes if p.endswith('.manifest.json'))+ manifest = json.loads(writes[manifest_path])+ assert manifest['base_commit'] == 'abc123'+ assert manifest['conversation_id'] == 'conv-1'+ assert manifest['source_path'] == '/workspace/project'++ @pytest.mark.asyncio+ async def test_archive_missing_base_commit_header_defaults_empty(self, monkeypatch):+ from openhands.app_server.sandbox import workspace_archive++ monkeypatch.setenv('RUNTIME_FILE_ARCHIVE_BUCKET', 'archive-bkt')+ monkeypatch.setenv('RUNTIME_FILE_ARCHIVE_FORMAT', 'git-delta')++ client = _stream_client(_make_stream_response(200, b'patch-bytes', {}))+ store = MagicMock()+ writes: dict[str, bytes] = {}+ store.write.side_effect = lambda path, data: writes.__setitem__(path, data)+ # Archive blobs are streamed from a tempfile via write_from_path (the OOM+ # fix); record them too so blob-path assertions still see the .patch/.tar.gz.+ store.write_from_path.side_effect = lambda path, src: writes.__setitem__(+ path, open(src, 'rb').read()+ )++ with patch.object(+ workspace_archive, '_get_archive_file_store', return_value=store+ ):+ ok = await workspace_archive.archive_workspace(+ client,+ create_runtime_data(),+ 'sandbox-1',+ archive_path='/workspace/project',+ )++ assert ok is True+ manifest_path = next(p for p in writes if p.endswith('.manifest.json'))+ manifest = json.loads(writes[manifest_path])+ assert manifest['base_commit'] == ''+ # Prod case: delete_sandbox passes no conversation_id, so the manifest+ # falls back to the sandbox id (which is the conversation_id.hex).+ assert manifest['conversation_id'] == 'sandbox-1'++ @pytest.mark.asyncio+ async def test_archive_non_200_not_required_allows_delete(self, monkeypatch):+ from openhands.app_server.sandbox import workspace_archive++ monkeypatch.setenv('RUNTIME_FILE_ARCHIVE_BUCKET', 'archive-bkt')+ monkeypatch.delenv('RUNTIME_FILE_ARCHIVE_REQUIRED', raising=False)++ client = _stream_client(_make_stream_response(500))++ ok = await workspace_archive.archive_workspace(+ client,+ create_runtime_data(),+ 'sandbox-1',+ archive_path='/workspace/project',+ )+ # Not required -> best-effort -> deletion may proceed.+ assert ok is True++ @pytest.mark.asyncio+ async def test_archive_non_200_required_blocks_delete(self, monkeypatch):+ from openhands.app_server.sandbox import workspace_archive++ monkeypatch.setenv('RUNTIME_FILE_ARCHIVE_BUCKET', 'archive-bkt')+ monkeypatch.setenv('RUNTIME_FILE_ARCHIVE_REQUIRED', 'true')++ client = _stream_client(_make_stream_response(500))++ ok = await workspace_archive.archive_workspace(+ client,+ create_runtime_data(),+ 'sandbox-1',+ archive_path='/workspace/project',+ )+ # Required -> a failure blocks deletion.+ assert ok is False++ @pytest.mark.asyncio+ async def test_archive_no_url_skips(self, monkeypatch):+ from openhands.app_server.sandbox import workspace_archive++ monkeypatch.setenv('RUNTIME_FILE_ARCHIVE_BUCKET', 'archive-bkt')+ monkeypatch.delenv('RUNTIME_FILE_ARCHIVE_REQUIRED', raising=False)+ client = AsyncMock()++ ok = await workspace_archive.archive_workspace(+ client,+ create_runtime_data(url=''),+ 'sandbox-1',+ archive_path='/workspace/project',+ )+ assert ok is True+ client.stream.assert_not_called()++ @pytest.mark.asyncio+ async def test_archive_400_no_repo_required_allows_delete(self, monkeypatch):+ """No git repo to capture is a permanent 400 from the agent-server, not a+ failure: even when archiving is required, the delete may proceed (BUG 2)."""+ from openhands.app_server.sandbox import workspace_archive++ monkeypatch.setenv('RUNTIME_FILE_ARCHIVE_BUCKET', 'archive-bkt')+ monkeypatch.setenv('RUNTIME_FILE_ARCHIVE_REQUIRED', 'true')++ # "Not a git repository" from GET /api/file/archive+ client = _stream_client(_make_stream_response(400))++ ok = await workspace_archive.archive_workspace(+ client,+ create_runtime_data(),+ 'sandbox-1',+ archive_path='/workspace/project',+ )+ # Permanent 4xx -> nothing to lose -> deletion proceeds despite REQUIRED.+ assert ok is True++ @pytest.mark.asyncio+ async def test_archive_404_required_blocks_delete(self, monkeypatch):+ """A 404 (path missing) is the misrouted-path symptom, NOT a confirmed-empty+ workspace: under REQUIRED it must block the delete (keep for the idle reap)+ rather than silently tearing the sandbox down uncaptured."""+ from openhands.app_server.sandbox import workspace_archive++ monkeypatch.setenv('RUNTIME_FILE_ARCHIVE_BUCKET', 'archive-bkt')+ monkeypatch.setenv('RUNTIME_FILE_ARCHIVE_REQUIRED', 'true')++ client = _stream_client(_make_stream_response(404))++ ok = await workspace_archive.archive_workspace(+ client,+ create_runtime_data(),+ 'sandbox-1',+ archive_path='/workspace/project',+ )+ assert ok is False++ @pytest.mark.asyncio+ async def test_archive_500_required_blocks_delete(self, monkeypatch):+ """A 5xx is a genuine transient failure: required archiving blocks the+ delete so it can be retried (BUG 2 β only 5xx/network block)."""+ from openhands.app_server.sandbox import workspace_archive++ monkeypatch.setenv('RUNTIME_FILE_ARCHIVE_BUCKET', 'archive-bkt')+ monkeypatch.setenv('RUNTIME_FILE_ARCHIVE_REQUIRED', 'true')++ client = _stream_client(_make_stream_response(500))++ ok = await workspace_archive.archive_workspace(+ client,+ create_runtime_data(),+ 'sandbox-1',+ archive_path='/workspace/project',+ )+ assert ok is False++ @pytest.mark.asyncio+ async def test_archive_422_required_blocks_delete(self, monkeypatch):+ """A 422 (malformed request from the producer) is NOT "nothing to archive"+ β under REQUIRED it is treated as retryable and must block the delete+ instead of being misread as permanent (infra#1444 M1)."""+ from openhands.app_server.sandbox import workspace_archive++ monkeypatch.setenv('RUNTIME_FILE_ARCHIVE_BUCKET', 'archive-bkt')+ monkeypatch.setenv('RUNTIME_FILE_ARCHIVE_REQUIRED', 'true')++ client = _stream_client(_make_stream_response(422))++ ok = await workspace_archive.archive_workspace(+ client,+ create_runtime_data(),+ 'sandbox-1',+ archive_path='/workspace/project',+ )+ assert ok is False++ @pytest.mark.asyncio+ async def test_archive_401_required_blocks_delete(self, monkeypatch):+ """A 401 (session key rejected) means the capture did NOT happen and is not+ a confirmed-empty workspace, so under REQUIRED it must block the delete+ (keep for the idle reap) rather than tear the sandbox down uncaptured."""+ from openhands.app_server.sandbox import workspace_archive++ monkeypatch.setenv('RUNTIME_FILE_ARCHIVE_BUCKET', 'archive-bkt')+ monkeypatch.setenv('RUNTIME_FILE_ARCHIVE_REQUIRED', 'true')++ client = _stream_client(_make_stream_response(401))++ ok = await workspace_archive.archive_workspace(+ client,+ create_runtime_data(),+ 'sandbox-1',+ archive_path='/workspace/project',+ )+ assert ok is False++ @pytest.mark.asyncio+ async def test_archive_unsupported_format_proceeds_and_skips_request(+ self, monkeypatch+ ):+ """An unsupported RUNTIME_FILE_ARCHIVE_FORMAT (e.g. the removed 'zip') is a+ pure config error caught up front: no request is issued, and the delete+ PROCEEDS even under REQUIRED. Like an unset bucket, no retry makes a valid+ format appear, so blocking would wedge every delete forever with no+ app-server backstop; the producer never sees the bad format."""+ from openhands.app_server.sandbox import workspace_archive++ monkeypatch.setenv('RUNTIME_FILE_ARCHIVE_BUCKET', 'archive-bkt')+ monkeypatch.setenv('RUNTIME_FILE_ARCHIVE_FORMAT', 'zip')+ monkeypatch.setenv('RUNTIME_FILE_ARCHIVE_REQUIRED', 'true')++ client = AsyncMock()++ ok = await workspace_archive.archive_workspace(+ client,+ create_runtime_data(),+ 'sandbox-1',+ archive_path='/workspace/project',+ )+ assert ok is True+ client.stream.assert_not_called()++ @pytest.mark.asyncio+ async def test_archive_unsupported_format_not_required_proceeds(self, monkeypatch):+ from openhands.app_server.sandbox import workspace_archive++ monkeypatch.setenv('RUNTIME_FILE_ARCHIVE_BUCKET', 'archive-bkt')+ monkeypatch.setenv('RUNTIME_FILE_ARCHIVE_FORMAT', 'zip')+ monkeypatch.delenv('RUNTIME_FILE_ARCHIVE_REQUIRED', raising=False)++ client = AsyncMock()++ ok = await workspace_archive.archive_workspace(+ client,+ create_runtime_data(),+ 'sandbox-1',+ archive_path='/workspace/project',+ )+ assert ok is True+ client.stream.assert_not_called()++ @pytest.mark.asyncio+ async def test_archive_no_bucket_required_proceeds(self, monkeypatch):+ """RUNTIME_FILE_ARCHIVE_BUCKET unset is a config error no retry can fix;+ it must NOT wedge every delete under REQUIRED β proceed loudly instead+ (infra#1444 L8)."""+ from openhands.app_server.sandbox import workspace_archive++ monkeypatch.delenv('RUNTIME_FILE_ARCHIVE_BUCKET', raising=False)+ monkeypatch.setenv('RUNTIME_FILE_ARCHIVE_REQUIRED', 'true')++ client = AsyncMock()++ ok = await workspace_archive.archive_workspace(+ client,+ create_runtime_data(),+ 'sandbox-1',+ archive_path='/workspace/project',+ )+ assert ok is True+ client.stream.assert_not_called()++ @pytest.mark.asyncio+ async def test_archive_uses_given_path_verbatim(self, monkeypatch):+ """archive_workspace targets the caller-resolved archive_path exactly (the+ path was pinned at creation; the grouping nesting is resolved upstream in+ RemoteSandboxService._resolve_archive_path)."""+ from openhands.app_server.sandbox import workspace_archive++ monkeypatch.setenv('RUNTIME_FILE_ARCHIVE_BUCKET', 'archive-bkt')+ monkeypatch.setenv('RUNTIME_FILE_ARCHIVE_FORMAT', 'git-delta')++ client = _stream_client(_make_stream_response(200, b'patch-bytes', {}))+ store = MagicMock()+ writes: dict[str, bytes] = {}+ store.write.side_effect = lambda path, data: writes.__setitem__(path, data)+ store.write_from_path.side_effect = lambda path, src: writes.__setitem__(+ path, open(src, 'rb').read()+ )++ with patch.object(+ workspace_archive, '_get_archive_file_store', return_value=store+ ):+ ok = await workspace_archive.archive_workspace(+ client,+ create_runtime_data(),+ 'sandbox-1',+ archive_path='/workspace/project/deadbeef',+ conversation_id='deadbeef',+ )++ assert ok is True+ _, kwargs = client.stream.call_args+ assert kwargs['params']['path'] == '/workspace/project/deadbeef'+ manifest_path = next(p for p in writes if p.endswith('.manifest.json'))+ manifest = json.loads(writes[manifest_path])+ assert manifest['source_path'] == '/workspace/project/deadbeef'+ assert manifest['conversation_id'] == 'deadbeef'++ @pytest.mark.asyncio+ async def test_archive_both_uploads_delta_and_targz(self, monkeypatch):+ """FORMAT=both (the default) captures git-delta AND a full tar.gz, each+ with its own artifact + manifest; base_commit rides the git-delta header+ and is reused for the tar.gz manifest."""+ from openhands.app_server.sandbox import workspace_archive++ monkeypatch.setenv('RUNTIME_FILE_ARCHIVE_BUCKET', 'archive-bkt')+ monkeypatch.setenv('RUNTIME_FILE_ARCHIVE_FORMAT', 'both')++ responses = {+ 'git-delta': _make_stream_response(+ 200, b'patch-bytes', {'X-Archive-Base-Commit': 'abc123'}+ ),+ 'tar.gz': _make_stream_response(200, b'tar-bytes', {}),+ }+ client = _stream_client(responses)+ store = MagicMock()+ writes: dict[str, bytes] = {}+ store.write.side_effect = lambda path, data: writes.__setitem__(path, data)+ # Archive blobs are streamed from a tempfile via write_from_path (the OOM+ # fix); record them too so blob-path assertions still see the .patch/.tar.gz.+ store.write_from_path.side_effect = lambda path, src: writes.__setitem__(+ path, open(src, 'rb').read()+ )++ with patch.object(+ workspace_archive, '_get_archive_file_store', return_value=store+ ):+ ok = await workspace_archive.archive_workspace(+ client,+ create_runtime_data(),+ 'sandbox-1',+ archive_path='/workspace/project',+ conversation_id='conv-1',+ )++ assert ok is True+ # Two artifacts under the same {ts} base, one per format.+ assert any(p.endswith('.patch') for p in writes)+ assert any(p.endswith('.tar.gz') for p in writes)+ patch_manifest = json.loads(+ writes[next(p for p in writes if p.endswith('.patch.manifest.json'))]+ )+ targz_manifest = json.loads(+ writes[next(p for p in writes if p.endswith('.tar.gz.manifest.json'))]+ )+ assert patch_manifest['format'] == 'git-delta'+ assert targz_manifest['format'] == 'tar.gz'+ # base_commit from the git-delta header is carried onto the tar.gz too.+ assert patch_manifest['base_commit'] == 'abc123'+ assert targz_manifest['base_commit'] == 'abc123'+ assert patch_manifest['phase'] == targz_manifest['phase'] == 'final'++ @pytest.mark.asyncio+ async def test_archive_both_required_blocks_if_one_format_fails(self, monkeypatch):+ """In REQUIRED both-mode, a retryable failure on either format blocks the+ delete so a retry can capture the complete pair."""+ from openhands.app_server.sandbox import workspace_archive++ monkeypatch.setenv('RUNTIME_FILE_ARCHIVE_BUCKET', 'archive-bkt')+ monkeypatch.setenv('RUNTIME_FILE_ARCHIVE_FORMAT', 'both')+ monkeypatch.setenv('RUNTIME_FILE_ARCHIVE_REQUIRED', 'true')++ responses = {+ 'git-delta': _make_stream_response(+ 200, b'patch-bytes', {'X-Archive-Base-Commit': 'abc123'}+ ),+ 'tar.gz': _make_stream_response(500),+ }+ client = _stream_client(responses)+ store = MagicMock()+ store.write.side_effect = lambda path, data: None+ store.write_from_path.side_effect = lambda path, src: None++ with patch.object(+ workspace_archive, '_get_archive_file_store', return_value=store+ ):+ ok = await workspace_archive.archive_workspace(+ client,+ create_runtime_data(),+ 'sandbox-1',+ archive_path='/workspace/project',+ )++ # tar.gz 500 is retryable + REQUIRED -> block the delete for a retry.+ assert ok is False+++class TestArchiveInitialWorkspaceHelper:+ """Unit tests for the initial-state (pre-agent) workspace snapshot."""++ @pytest.mark.asyncio+ async def test_initial_archive_uploads_tar_gz_and_manifest(self, monkeypatch):+ from openhands.app_server.sandbox import workspace_archive++ monkeypatch.setenv('RUNTIME_FILE_ARCHIVE_INITIAL_ENABLED', 'true')+ monkeypatch.setenv('RUNTIME_FILE_ARCHIVE_BUCKET', 'archive-bkt')++ client = _stream_client(_make_stream_response(200, b'tar-bytes', {}))+ store = MagicMock()+ writes: dict[str, bytes] = {}+ store.write.side_effect = lambda path, data: writes.__setitem__(path, data)+ # Archive blobs are streamed from a tempfile via write_from_path (the OOM+ # fix); record them too so blob-path assertions still see the .patch/.tar.gz.+ store.write_from_path.side_effect = lambda path, src: writes.__setitem__(+ path, open(src, 'rb').read()+ )++ with patch.object(+ workspace_archive, '_get_archive_file_store', return_value=store+ ):+ ok = await workspace_archive.archive_initial_workspace(+ client,+ agent_server_url='https://sandbox.example.com',+ session_api_key='sk',+ project_dir='/workspace/project/repo',+ sandbox_id='sandbox-1',+ conversation_id='conv-1',+ base_commit='deadbeef',+ )++ assert ok is True+ # tar.gz requested at the (already-resolved) project dir; key forwarded.+ _, kwargs = client.stream.call_args+ assert kwargs['headers']['X-Session-API-Key'] == 'sk'+ # tar.gz is the full capture, so default excludes are disabled.+ assert kwargs['params'] == {+ 'path': '/workspace/project/repo',+ 'format': 'tar.gz',+ 'use_default_excludes': 'false',+ }+ # Keyed by conversation and nested under /initial/ so it can never collide+ # with a sibling or with this conversation's final capture (which writes to+ # {prefix}/{sandbox_id}/{conversation_key}/{ts}).+ archive_path = next(p for p in writes if p.endswith('.tar.gz'))+ assert '/sandbox-1/conv-1/initial/' in archive_path+ manifest_path = next(p for p in writes if p.endswith('.manifest.json'))+ # Shared contract: manifest = blob + '.manifest.json' (keeps the suffix).+ assert manifest_path == archive_path + '.manifest.json'+ manifest = json.loads(writes[manifest_path])+ assert manifest['phase'] == 'initial'+ assert manifest['base_commit'] == 'deadbeef'+ assert manifest['conversation_id'] == 'conv-1'+ assert manifest['format'] == 'tar.gz'+ assert manifest['source_path'] == '/workspace/project/repo'++ @pytest.mark.asyncio+ async def test_archive_sibling_conversations_distinct_keys(self, monkeypatch):+ """Two sibling conversations on the SAME grouped sandbox, captured in the+ same second, must land at distinct blob keys: the conversation id is in the+ object path, not just the manifest body (regression β same-second overwrite+ silently dropped one conversation's archive)."""+ from datetime import datetime, timezone++ from openhands.app_server.sandbox import workspace_archive++ monkeypatch.setenv('RUNTIME_FILE_ARCHIVE_BUCKET', 'archive-bkt')+ monkeypatch.setenv('RUNTIME_FILE_ARCHIVE_FORMAT', 'git-delta')+ # Pin the timestamp so both captures compute the SAME {ts}; only the+ # conversation segment can keep them apart.+ fixed = datetime(2026, 1, 1, 0, 0, 0, tzinfo=timezone.utc)+ monkeypatch.setattr(workspace_archive, 'utc_now', lambda: fixed)++ writes: dict[str, bytes] = {}+ store = MagicMock()+ store.write.side_effect = lambda path, data: writes.__setitem__(path, data)+ store.write_from_path.side_effect = lambda path, src: writes.__setitem__(+ path, open(src, 'rb').read()+ )++ with patch.object(+ workspace_archive, '_get_archive_file_store', return_value=store+ ):+ for conv in ('conva', 'convb'):+ client = _stream_client(_make_stream_response(200, b'patch-bytes', {}))+ ok = await workspace_archive.archive_workspace(+ client,+ create_runtime_data(),+ 'shared-sandbox',+ archive_path=f'/workspace/project/{conv}',+ conversation_id=conv,+ )+ assert ok is True++ # Both artifacts AND both manifests survive β nothing overwritten.+ patch_blobs = [p for p in writes if p.endswith('.patch')]+ manifests = [p for p in writes if p.endswith('.manifest.json')]+ assert len(patch_blobs) == 2+ assert len(manifests) == 2+ assert any('/shared-sandbox/conva/' in p for p in patch_blobs)+ assert any('/shared-sandbox/convb/' in p for p in patch_blobs)++ @pytest.mark.asyncio+ async def test_initial_archive_sibling_conversations_distinct_keys(+ self, monkeypatch+ ):+ """Same same-second path-collision guard for the initial snapshot under+ grouping: sibling initial captures land at distinct keys."""+ from datetime import datetime, timezone++ from openhands.app_server.sandbox import workspace_archive++ monkeypatch.setenv('RUNTIME_FILE_ARCHIVE_INITIAL_ENABLED', 'true')+ monkeypatch.setenv('RUNTIME_FILE_ARCHIVE_BUCKET', 'archive-bkt')+ fixed = datetime(2026, 1, 1, 0, 0, 0, tzinfo=timezone.utc)+ monkeypatch.setattr(workspace_archive, 'utc_now', lambda: fixed)++ writes: dict[str, bytes] = {}+ store = MagicMock()+ store.write.side_effect = lambda path, data: writes.__setitem__(path, data)+ store.write_from_path.side_effect = lambda path, src: writes.__setitem__(+ path, open(src, 'rb').read()+ )++ with patch.object(+ workspace_archive, '_get_archive_file_store', return_value=store+ ):+ for conv in ('conva', 'convb'):+ client = _stream_client(_make_stream_response(200, b'tar-bytes', {}))+ ok = await workspace_archive.archive_initial_workspace(+ client,+ agent_server_url='https://sandbox.example.com',+ session_api_key='sk',+ project_dir=f'/workspace/project/{conv}',+ sandbox_id='shared-sandbox',+ conversation_id=conv,+ )+ assert ok is True++ tarballs = [p for p in writes if p.endswith('.tar.gz')]+ manifests = [p for p in writes if p.endswith('.manifest.json')]+ assert len(tarballs) == 2+ assert len(manifests) == 2+ assert any('/shared-sandbox/conva/initial/' in p for p in tarballs)+ assert any('/shared-sandbox/convb/initial/' in p for p in tarballs)++ @pytest.mark.asyncio+ async def test_initial_archive_disabled_is_noop(self, monkeypatch):+ from openhands.app_server.sandbox import workspace_archive++ monkeypatch.delenv('RUNTIME_FILE_ARCHIVE_INITIAL_ENABLED', raising=False)+ monkeypatch.setenv('RUNTIME_FILE_ARCHIVE_BUCKET', 'archive-bkt')++ client = AsyncMock()++ ok = await workspace_archive.archive_initial_workspace(+ client,+ agent_server_url='https://sandbox.example.com',+ session_api_key='sk',+ project_dir='/workspace/project/repo',+ sandbox_id='sandbox-1',+ )+ # Off by default: no request, no upload.+ assert ok is False+ client.stream.assert_not_called()++ @pytest.mark.asyncio+ async def test_initial_archive_non_200_returns_false(self, monkeypatch):+ from openhands.app_server.sandbox import workspace_archive++ monkeypatch.setenv('RUNTIME_FILE_ARCHIVE_INITIAL_ENABLED', 'true')+ monkeypatch.setenv('RUNTIME_FILE_ARCHIVE_BUCKET', 'archive-bkt')++ client = _stream_client(_make_stream_response(500))++ ok = await workspace_archive.archive_initial_workspace(+ client,+ agent_server_url='https://sandbox.example.com',+ session_api_key='sk',+ project_dir='/workspace/project/repo',+ sandbox_id='sandbox-1',+ )+ # Best-effort: a failed capture is swallowed (never blocks startup).+ assert ok is False++ @pytest.mark.asyncio+ async def test_initial_archive_no_bucket_returns_false(self, monkeypatch):+ from openhands.app_server.sandbox import workspace_archive++ monkeypatch.setenv('RUNTIME_FILE_ARCHIVE_INITIAL_ENABLED', 'true')+ monkeypatch.delenv('RUNTIME_FILE_ARCHIVE_BUCKET', raising=False)++ client = AsyncMock()++ ok = await workspace_archive.archive_initial_workspace(+ client,+ agent_server_url='https://sandbox.example.com',+ session_api_key='sk',+ project_dir='/workspace/project/repo',+ sandbox_id='sandbox-1',+ )+ assert ok is False+ client.stream.assert_not_called()+++class TestDeleteSandboxKeyHandling:+ """The session_api_key_hash is invalidated UP FRONT on delete (a delete is+ often a revoke of a leaked key). When a transient error keeps the row for+ retry, the invalidation is committed first so the DELETE route's rollback+ cannot resurrect the key. Backed by a real SQLite session so persistence is+ provable across a rollback."""++ @pytest.fixture+ async def async_engine(self):+ from sqlalchemy.ext.asyncio import create_async_engine+ from sqlalchemy.pool import StaticPool++ from openhands.app_server.utils.sql_utils import Base++ engine = create_async_engine(+ 'sqlite+aiosqlite:///:memory:',+ poolclass=StaticPool,+ connect_args={'check_same_thread': False},+ echo=False,+ )+ async with engine.begin() as conn:+ await conn.run_sync(Base.metadata.create_all)+ yield engine+ await engine.dispose()++ @pytest.fixture+ async def real_session(self, async_engine):+ from sqlalchemy.ext.asyncio import async_sessionmaker++ maker = async_sessionmaker(+ async_engine, class_=AsyncSession, expire_on_commit=False+ )+ async with maker() as session:+ yield session++ @pytest.fixture+ def service_with_real_db(+ self, mock_sandbox_spec_service, mock_user_context, real_session+ ):+ return RemoteSandboxService(+ sandbox_spec_service=mock_sandbox_spec_service,+ api_url='https://api.example.com',+ api_key='test-api-key',+ web_url='https://web.example.com',+ resource_factor=1,+ runtime_class='gvisor',+ start_sandbox_timeout=120,+ max_num_sandboxes=10,+ user_context=mock_user_context,+ httpx_client=AsyncMock(spec=httpx.AsyncClient),+ db_session=real_session,+ )++ @pytest.mark.asyncio+ async def test_session_key_invalidated_and_survives_rollback(+ self, service_with_real_db, real_session+ ):+ # Seed a running sandbox with a live session key hash.+ row = create_stored_sandbox(session_api_key_hash='live-hash')+ real_session.add(row)+ await real_session.commit()++ # Transient (non-404) runtime error -> delete_sandbox raises (keeps the+ # row + running runtime for retry) without deleting the row.+ service_with_real_db._get_runtime = AsyncMock(+ side_effect=httpx.HTTPStatusError(+ 'Server Error',+ request=httpx.Request('GET', 'https://api.example.com/sessions/x'),+ response=httpx.Response(503),+ )+ )++ with pytest.raises(SandboxDeleteRetryError):+ await service_with_real_db.delete_sandbox('test-sandbox-123')++ # Simulate the DELETE route's session rolling back the failed delete.+ await real_session.rollback()+ real_session.expire_all()++ # Row kept for retry, but the key is DEAD and the rollback could not bring+ # it back β delete_sandbox committed the invalidation before raising. This+ # is the security fix: a leaked key is revoked even when the stop fails.+ refreshed = await service_with_real_db._get_stored_sandbox('test-sandbox-123')+ assert refreshed is not None+ assert refreshed.session_api_key_hash is None++ @pytest.mark.asyncio+ async def test_session_key_cleared_on_successful_delete(+ self, service_with_real_db, real_session, monkeypatch+ ):+ # Archiving off -> a clean delete goes straight to /stop + row delete.+ monkeypatch.delenv('RUNTIME_FILE_ARCHIVE_ENABLED', raising=False)+ row = create_stored_sandbox(session_api_key_hash='live-hash')+ real_session.add(row)+ await real_session.commit()++ service_with_real_db._get_runtime = AsyncMock(+ return_value={'runtime_id': 'rt-123'}+ )+ stop_response = MagicMock()+ stop_response.status_code = 200+ stop_response.raise_for_status.return_value = None+ service_with_real_db._send_runtime_api_request = AsyncMock(+ return_value=stop_response+ )++ assert await service_with_real_db.delete_sandbox('test-sandbox-123') is True+ await real_session.commit()+ real_session.expire_all()++ # The row β and with it the session key β is gone after a clean delete.+ refreshed = await service_with_real_db._get_stored_sandbox('test-sandbox-123')+ assert refreshed is None+++class TestArchiveEnvToggles:+ """Per OpenHands/AGENTS.md, env enable-toggles must accept '1' as well as+ 'true' (older Helm charts default these to '1'). A regression to == 'true'+ would silently disable the entire workspace-archive capture, so pin both."""++ @pytest.mark.parametrize('value', ['1', 'true', 'TRUE', 'True'])+ def test_archive_enabled_truthy(self, monkeypatch, value):+ from openhands.app_server.sandbox import workspace_archive++ monkeypatch.setenv('RUNTIME_FILE_ARCHIVE_ENABLED', value)+ assert workspace_archive.archive_enabled() is True++ @pytest.mark.parametrize('value', ['0', 'false', '', 'no'])+ def test_archive_enabled_falsy(self, monkeypatch, value):+ from openhands.app_server.sandbox import workspace_archive++ monkeypatch.setenv('RUNTIME_FILE_ARCHIVE_ENABLED', value)+ assert workspace_archive.archive_enabled() is False++ @pytest.mark.parametrize('value', ['1', 'true'])+ def test_archive_required_truthy(self, monkeypatch, value):+ from openhands.app_server.sandbox import workspace_archive++ monkeypatch.setenv('RUNTIME_FILE_ARCHIVE_REQUIRED', value)+ assert workspace_archive.archive_required() is True++ @pytest.mark.parametrize('value', ['1', 'true'])+ def test_initial_archive_enabled_truthy(self, monkeypatch, value):+ from openhands.app_server.sandbox import workspace_archive++ monkeypatch.setenv('RUNTIME_FILE_ARCHIVE_INITIAL_ENABLED', value)+ assert workspace_archive.initial_archive_enabled() is True++ def test_toggles_default_off(self, monkeypatch):+ from openhands.app_server.sandbox import workspace_archive++ for var in (+ 'RUNTIME_FILE_ARCHIVE_ENABLED',+ 'RUNTIME_FILE_ARCHIVE_REQUIRED',+ 'RUNTIME_FILE_ARCHIVE_INITIAL_ENABLED',+ ):+ monkeypatch.delenv(var, raising=False)+ assert workspace_archive.archive_enabled() is False+ assert workspace_archive.archive_required() is False+ assert workspace_archive.initial_archive_enabled() is False+++class TestArchiveRequestParams:+ """The tar.gz is the self-contained full capture, so it disables the+ endpoint's default excludes; git-delta stays compact (keeps them)."""++ def test_tar_gz_disables_default_excludes(self):+ from openhands.app_server.sandbox import workspace_archive++ params = workspace_archive._archive_request_params(+ '/workspace/project', 'tar.gz'+ )+ assert params == {+ 'path': '/workspace/project',+ 'format': 'tar.gz',+ 'use_default_excludes': 'false',+ }++ def test_git_delta_keeps_default_excludes(self):+ from openhands.app_server.sandbox import workspace_archive++ params = workspace_archive._archive_request_params(+ '/workspace/project', 'git-delta'+ )+ assert params == {'path': '/workspace/project', 'format': 'git-delta'}