Secure, Fast, and Extensible Sandbox runtime for AI agents.
fix(server): map concurrent Docker lifecycle conflicts to 404/409
Jackmeson19e97f68cab95
diff --git a/docs/api/index.md b/docs/api/index.mdindex 8ccd65a6b..98f76efa4 100644--- a/docs/api/index.md+++ b/docs/api/index.md@@ -53,6 +53,8 @@ Defines the complete lifecycle interfaces for creating, managing, and destroying**Expiration updates:** `renew-expiration` sets `expiresAt` to the supplied future timestamp, which may be earlier than, equal to, or later than the current expiration. The configured `server.max_sandbox_timeout_seconds` applies only to `timeout` on creation; it does not cap expiration updates. See [Sandbox expiration](/getting-started/configuration#sandbox-expiration).+**Concurrent Docker lifecycle requests:** Recognized conflicts on the main container return `409 Conflict` when removal is already in progress or the container is no longer in the state required by pause or resume. A missing main container returns `404 Not Found` when reported by the operation or confirmed by the follow-up lookup. For resume, these mappings apply only after any required sidecar rollback succeeds. Pause/resume sidecar failures, failed rollbacks, and unrelated main-container or lookup failures retain their existing `500` responses.+**Optional `Sandbox.allocation` response field:**- Returned only when the runtime confirms the sandbox's current concrete Pool allocation.- Omitted for unconfirmed allocations, non-Pool sandboxes, and allocations being released.diff --git a/server/opensandbox_server/services/docker/docker_service.py b/server/opensandbox_server/services/docker/docker_service.pyindex cd9664720..15e6b94e7 100644--- a/server/opensandbox_server/services/docker/docker_service.py+++ b/server/opensandbox_server/services/docker/docker_service.py@@ -34,7 +34,7 @@ from threading import Lock, Thread, Timerfrom typing import Any, Dict, Optionalimport docker-from docker.errors import DockerException, NotFound as DockerNotFound+from docker.errors import APIError, DockerException, NotFound as DockerNotFoundfrom fastapi import HTTPException, statusfrom opensandbox_server.extensions import (@@ -1094,6 +1094,54 @@ class DockerSandboxService(DockerDiagnosticsMixin, DockerRuntimeMixin, DockerVolcontainer = self._get_container_by_sandbox_id(sandbox_id)return self._container_to_sandbox(container, sandbox_id)+ def _raise_if_lifecycle_race(+ self, sandbox_id: str, exc: DockerException, operation: str,+ ) -> None:+ """Map known main-container races using the sequential error envelope."""+ if isinstance(exc, DockerNotFound):+ if operation == "delete":+ self._cleanup_egress_sidecar(sandbox_id)+ raise HTTPException(+ status_code=status.HTTP_404_NOT_FOUND,+ detail={+ "code": SandboxErrorCodes.SANDBOX_NOT_FOUND,+ "message": f"Sandbox {sandbox_id} not found.",+ },+ ) from exc+ if not isinstance(exc, APIError) or exc.status_code != status.HTTP_409_CONFLICT:+ return++ message = (exc.explanation or "").lower()+ conflicts = {+ "delete": (+ ("removal of container " in message and "already in progress" in message)+ or "already being removed" in message,+ SandboxErrorCodes.SANDBOX_DELETE_FAILED,+ "Sandbox deletion is already in progress.",+ ),+ "pause": (+ "already paused" in message or "is not running" in message,+ SandboxErrorCodes.SANDBOX_NOT_RUNNING,+ "Sandbox is not in a running state.",+ ),+ "resume": (+ "is not paused" in message or "is not running" in message,+ SandboxErrorCodes.SANDBOX_NOT_PAUSED,+ "Sandbox is not in a paused state.",+ ),+ }+ matches, code, conflict_message = conflicts[operation]+ if not matches:+ return+ if operation != "delete":+ # The state error may have raced with removal; use the usual 404+ # if the sandbox has disappeared since the operation failed.+ self._get_container_by_sandbox_id(sandbox_id)+ raise HTTPException(+ status_code=status.HTTP_409_CONFLICT,+ detail={"code": code, "message": conflict_message},+ ) from exc+def delete_sandbox(self, sandbox_id: str) -> None:try:container = self._get_container_by_sandbox_id(sandbox_id)@@ -1123,6 +1171,7 @@ class DockerSandboxService(DockerDiagnosticsMixin, DockerRuntimeMixin, DockerVolwith self._docker_operation("remove sandbox container", sandbox_id):container.remove(force=True)except DockerException as exc:+ self._raise_if_lifecycle_race(sandbox_id, exc, "delete")raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,detail={@@ -1167,6 +1216,7 @@ class DockerSandboxService(DockerDiagnosticsMixin, DockerRuntimeMixin, DockerVolwith self._docker_operation("pause sandbox container", sandbox_id):container.pause()except DockerException as exc:+ self._raise_if_lifecycle_race(sandbox_id, exc, "pause")raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,detail={@@ -1301,6 +1351,8 @@ class DockerSandboxService(DockerDiagnosticsMixin, DockerRuntimeMixin, DockerVolmessage = f"Failed to resume {failed_component}: {str(exc)}"if rollback_errors:message += f"; rollback failed: {'; '.join(rollback_errors)}"+ elif resuming_main:+ self._raise_if_lifecycle_race(sandbox_id, exc, "resume")raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,detail={diff --git a/server/tests/test_docker_service.py b/server/tests/test_docker_service.pyindex 7a9bb36dd..43f07679a 100644--- a/server/tests/test_docker_service.py+++ b/server/tests/test_docker_service.py@@ -20,7 +20,7 @@ from datetime import datetime, timedelta, timezonefrom typing import Any, castfrom unittest.mock import MagicMock, patch-from docker.errors import DockerException, NotFound as DockerNotFound+from docker.errors import APIError, DockerException, NotFound as DockerNotFoundimport pytestfrom fastapi import HTTPException, statusfrom pydantic import SecretStr, ValidationError@@ -2130,6 +2130,226 @@ def _lifecycle_container(return container[email protected]("failed_operation", ["kill", "remove"])+ "docker_status, message, expected_status",+ [+ (409, "removal of container main-id is already in progress", 409),+ (409, "container main-id is already being removed", 409),+ (404, "No such container: main-id", 404),+ (409, "unrelated daemon conflict", 500),+ (500, "removal of container main-id is already in progress", 500),+ ],+)+def test_delete_sandbox_concurrent_outcomes(+ tmp_path, failed_operation, docker_status, message, expected_status,+):+ service = DockerSandboxService(config=_app_config())+ service._metadata_store = DockerMetadataStore(tmp_path / "metadata")+ expiration = datetime.now(timezone.utc) + timedelta(hours=1)+ service._metadata_store.set_expiration("sandbox-id", expiration)+ main = _lifecycle_container("main-id", running=True, paused=False)+ error_type = DockerNotFound if docker_status == 404 else APIError+ error = error_type(message, response=MagicMock(status_code=docker_status), explanation=message)+ getattr(main, failed_operation).side_effect = error++ with (+ patch.object(service, "_get_container_by_sandbox_id", return_value=main),+ patch.object(service, "_cleanup_egress_sidecar") as cleanup_sidecar,+ patch.object(service, "_remove_expiration_tracking") as remove_tracking,+ patch.object(service, "_cleanup_windows_oem_volume") as cleanup_oem,+ patch.object(service, "_release_ossfs_mounts") as release_mounts,+ patch.object(service, "_cleanup_managed_volumes") as cleanup_volumes,+ pytest.raises(HTTPException) as exc_info,+ ):+ service.delete_sandbox("sandbox-id")++ assert exc_info.value.status_code == expected_status+ if expected_status == 404:+ assert exc_info.value.detail == {+ "code": SandboxErrorCodes.SANDBOX_NOT_FOUND,+ "message": "Sandbox sandbox-id not found.",+ }+ cleanup_sidecar.assert_called_once_with("sandbox-id")+ else:+ assert exc_info.value.detail["code"] == SandboxErrorCodes.SANDBOX_DELETE_FAILED+ assert exc_info.value.detail["message"]+ cleanup_sidecar.assert_not_called()+ if expected_status == 500:+ assert str(error) in exc_info.value.detail["message"]+ if failed_operation == "kill":+ main.remove.assert_not_called()+ remove_tracking.assert_not_called()+ cleanup_oem.assert_not_called()+ release_mounts.assert_not_called()+ cleanup_volumes.assert_not_called()+ assert service._metadata_store.get_expiration("sandbox-id") == expiration.isoformat()++[email protected]("operation", ["pause", "resume"])+ "docker_status, outcome, disappeared, expected_status",+ [+ (409, "state conflict", False, 409),+ (409, "not running", False, 409),+ (409, "not running", True, 404),+ (404, "No such container: main-id", True, 404),+ (409, "unrelated daemon conflict", False, 500),+ (500, "state conflict", False, 500),+ ],+)+def test_pause_resume_sandbox_concurrent_outcomes(+ operation, docker_status, outcome, disappeared, expected_status,+):+ service = DockerSandboxService(config=_app_config())+ resuming = operation == "resume"+ main = _lifecycle_container("main-id", running=True, paused=resuming, egress_expected=True)+ sidecar = _lifecycle_container("sidecar-id", running=True, paused=resuming)+ if outcome == "state conflict":+ message = "container main-id is not paused" if resuming else "container main-id is already paused"+ elif outcome == "not running":+ message = "container main-id is not running"+ else:+ message = outcome+ error_type = DockerNotFound if docker_status == 404 else APIError+ error = error_type(message, response=MagicMock(status_code=docker_status), explanation=message)+ getattr(main, "unpause" if resuming else "pause").side_effect = error+ missing = HTTPException(status_code=404, detail={+ "code": SandboxErrorCodes.SANDBOX_NOT_FOUND,+ "message": "Sandbox sandbox-id not found.",+ })+ lookup_results = [main, missing if disappeared else main]++ with (+ patch.object(service, "_get_container_by_sandbox_id", side_effect=lookup_results),+ patch.object(service, "_get_egress_sidecars", return_value=[sidecar]),+ pytest.raises(HTTPException) as exc_info,+ ):+ getattr(service, f"{operation}_sandbox")("sandbox-id")++ assert exc_info.value.status_code == expected_status+ if expected_status == 404:+ assert exc_info.value.detail == missing.detail+ elif expected_status == 409:+ assert exc_info.value.detail == {+ "code": SandboxErrorCodes.SANDBOX_NOT_PAUSED if resuming else SandboxErrorCodes.SANDBOX_NOT_RUNNING,+ "message": f"Sandbox is not in a {'paused' if resuming else 'running'} state.",+ }+ else:+ assert exc_info.value.detail == {+ "code": SandboxErrorCodes.SANDBOX_RESUME_FAILED if resuming else SandboxErrorCodes.SANDBOX_PAUSE_FAILED,+ "message": f"Failed to {operation} sandbox container: {error}",+ }+ if resuming:+ main.unpause.assert_called_once_with()+ # Keep the existing sidecar rollback even when the main failure maps to 409/404.+ sidecar.unpause.assert_called_once_with()+ sidecar.pause.assert_called_once_with()+ else:+ main.pause.assert_called_once_with()+ sidecar.pause.assert_not_called()+ main.unpause.assert_not_called()++[email protected]("docker_status", [404, 409])+def test_resume_sandbox_race_with_failed_rollback_remains_500(docker_status):+ service = DockerSandboxService(config=_app_config())+ main = _lifecycle_container("main-id", running=True, paused=True, egress_expected=True)+ sidecar = _lifecycle_container("sidecar-id", running=True, paused=True)+ error_type = DockerNotFound if docker_status == 404 else APIError+ main.unpause.side_effect = error_type(+ "resume raced", response=MagicMock(status_code=docker_status),+ explanation="container main-id is not paused",+ )+ sidecar.pause.side_effect = DockerException("rollback failed")++ with (+ patch.object(service, "_get_container_by_sandbox_id", return_value=main),+ patch.object(service, "_get_egress_sidecars", return_value=[sidecar]),+ pytest.raises(HTTPException) as exc_info,+ ):+ service.resume_sandbox("sandbox-id")++ assert exc_info.value.status_code == 500+ assert exc_info.value.detail["code"] == SandboxErrorCodes.SANDBOX_RESUME_FAILED+ assert "rollback failed" in exc_info.value.detail["message"]+ sidecar.pause.assert_called_once_with()++[email protected]("operation", ["pause", "resume"])[email protected]("docker_status", [404, 409])+def test_pause_resume_sidecar_race_preserves_failure_and_rollback(operation, docker_status):+ service = DockerSandboxService(config=_app_config())+ resuming = operation == "resume"+ main = _lifecycle_container("main-id", running=True, paused=resuming, egress_expected=True)+ sidecar = _lifecycle_container("sidecar-id", running=True, paused=resuming)+ error_type = DockerNotFound if docker_status == 404 else APIError+ getattr(sidecar, "unpause" if resuming else "pause").side_effect = error_type(+ "sidecar raced", response=MagicMock(status_code=docker_status),+ explanation="container sidecar-id is not paused" if resuming else "container sidecar-id is already paused",+ )++ with (+ patch.object(service, "_get_container_by_sandbox_id", return_value=main),+ patch.object(service, "_get_egress_sidecars", return_value=[sidecar]),+ pytest.raises(HTTPException) as exc_info,+ ):+ getattr(service, f"{operation}_sandbox")("sandbox-id")++ assert exc_info.value.status_code == 500+ assert exc_info.value.detail["code"] == (+ SandboxErrorCodes.SANDBOX_RESUME_FAILED if resuming else SandboxErrorCodes.SANDBOX_PAUSE_FAILED+ )+ assert "egress sidecar" in exc_info.value.detail["message"]+ if resuming:+ main.unpause.assert_not_called()+ else:+ main.pause.assert_called_once_with()+ main.unpause.assert_called_once_with()++[email protected]("docker_status", [404, 409, 500])+def test_expire_sandbox_removal_race_retries_without_releasing_stale_mounts(tmp_path, docker_status):+ service = DockerSandboxService(config=_app_config())+ service._metadata_store = DockerMetadataStore(tmp_path / "metadata")+ service._metadata_store.set_expiration("sandbox-id", datetime.now(timezone.utc) - timedelta(seconds=1))+ main = _lifecycle_container("main-id", running=True, paused=False)+ main.attrs["Config"]["Labels"][SANDBOX_OSSFS_MOUNTS_LABEL] = '["shared-mount"]'+ error_type = DockerNotFound if docker_status == 404 else APIError+ error = error_type(+ "removal raced", response=MagicMock(status_code=docker_status),+ explanation="removal of container main-id is already in progress",+ )+ main.remove.side_effect = error+ if docker_status == 404:+ main.kill.side_effect = error+ missing = HTTPException(status_code=404, detail={+ "code": SandboxErrorCodes.SANDBOX_NOT_FOUND,+ "message": "Sandbox sandbox-id not found.",+ })++ with (+ patch.object(service, "_get_container_by_sandbox_id", side_effect=[main, missing]),+ patch.object(service, "_schedule_expiration") as schedule,+ patch.object(service, "_cleanup_egress_sidecar") as cleanup_sidecar,+ patch.object(service, "_release_ossfs_mounts") as release_mounts,+ patch.object(service, "_cleanup_managed_volumes") as cleanup_volumes,+ ):+ service._expire_sandbox("sandbox-id")+ schedule.assert_called_once()+ assert schedule.call_args.args[0] == "sandbox-id"+ assert schedule.call_args.kwargs == {"update_expiration": False}+ cleanup_sidecar.assert_not_called()+ cleanup_volumes.assert_not_called()+ assert service._metadata_store.get_expiration("sandbox-id") is not None++ # A concurrent DELETE finished cleanup. Retry must re-read ownership.+ service._expire_sandbox("sandbox-id")+ cleanup_sidecar.assert_called_once_with("sandbox-id")+ release_mounts.assert_not_called()+ assert service._metadata_store.get_expiration("sandbox-id") is None++def test_pause_sandbox_pauses_main_before_egress_sidecar():service = DockerSandboxService(config=_app_config())main = _lifecycle_container("main-id", running=True, paused=False, egress_expected=True)