Skip to content

Secure, Fast, and Extensible Sandbox runtime for AI agents.

fix(server): map concurrent Docker lifecycle conflicts to 404/409

Jackmeson19e97f68cab95
diff --git a/docs/api/index.md b/docs/api/index.md
index 8ccd65a6b..98f76efa4 100644
--- a/docs/api/index.md
+++ b/docs/api/index.md
@@ -53,6 +53,8 @@ Defines the complete lifecycle interfaces for creating, managing, and destroying
**Expiration updates:** `renew-expiration` sets `expiresAt` to the supplied future timestamp, which may be earlier than, equal to, or later than the current expiration. The configured `server.max_sandbox_timeout_seconds` applies only to `timeout` on creation; it does not cap expiration updates. See [Sandbox expiration](/getting-started/configuration#sandbox-expiration).
+**Concurrent Docker lifecycle requests:** Recognized conflicts on the main container return `409 Conflict` when removal is already in progress or the container is no longer in the state required by pause or resume. A missing main container returns `404 Not Found` when reported by the operation or confirmed by the follow-up lookup. For resume, these mappings apply only after any required sidecar rollback succeeds. Pause/resume sidecar failures, failed rollbacks, and unrelated main-container or lookup failures retain their existing `500` responses.
+
**Optional `Sandbox.allocation` response field:**
- Returned only when the runtime confirms the sandbox's current concrete Pool allocation.
- Omitted for unconfirmed allocations, non-Pool sandboxes, and allocations being released.
diff --git a/server/opensandbox_server/services/docker/docker_service.py b/server/opensandbox_server/services/docker/docker_service.py
index cd9664720..15e6b94e7 100644
--- a/server/opensandbox_server/services/docker/docker_service.py
+++ b/server/opensandbox_server/services/docker/docker_service.py
@@ -34,7 +34,7 @@ from threading import Lock, Thread, Timer
from typing import Any, Dict, Optional
import docker
-from docker.errors import DockerException, NotFound as DockerNotFound
+from docker.errors import APIError, DockerException, NotFound as DockerNotFound
from fastapi import HTTPException, status
from opensandbox_server.extensions import (
@@ -1094,6 +1094,54 @@ class DockerSandboxService(DockerDiagnosticsMixin, DockerRuntimeMixin, DockerVol
container = self._get_container_by_sandbox_id(sandbox_id)
return self._container_to_sandbox(container, sandbox_id)
+ def _raise_if_lifecycle_race(
+ self, sandbox_id: str, exc: DockerException, operation: str,
+ ) -> None:
+ """Map known main-container races using the sequential error envelope."""
+ if isinstance(exc, DockerNotFound):
+ if operation == "delete":
+ self._cleanup_egress_sidecar(sandbox_id)
+ raise HTTPException(
+ status_code=status.HTTP_404_NOT_FOUND,
+ detail={
+ "code": SandboxErrorCodes.SANDBOX_NOT_FOUND,
+ "message": f"Sandbox {sandbox_id} not found.",
+ },
+ ) from exc
+ if not isinstance(exc, APIError) or exc.status_code != status.HTTP_409_CONFLICT:
+ return
+
+ message = (exc.explanation or "").lower()
+ conflicts = {
+ "delete": (
+ ("removal of container " in message and "already in progress" in message)
+ or "already being removed" in message,
+ SandboxErrorCodes.SANDBOX_DELETE_FAILED,
+ "Sandbox deletion is already in progress.",
+ ),
+ "pause": (
+ "already paused" in message or "is not running" in message,
+ SandboxErrorCodes.SANDBOX_NOT_RUNNING,
+ "Sandbox is not in a running state.",
+ ),
+ "resume": (
+ "is not paused" in message or "is not running" in message,
+ SandboxErrorCodes.SANDBOX_NOT_PAUSED,
+ "Sandbox is not in a paused state.",
+ ),
+ }
+ matches, code, conflict_message = conflicts[operation]
+ if not matches:
+ return
+ if operation != "delete":
+ # The state error may have raced with removal; use the usual 404
+ # if the sandbox has disappeared since the operation failed.
+ self._get_container_by_sandbox_id(sandbox_id)
+ raise HTTPException(
+ status_code=status.HTTP_409_CONFLICT,
+ detail={"code": code, "message": conflict_message},
+ ) from exc
+
def delete_sandbox(self, sandbox_id: str) -> None:
try:
container = self._get_container_by_sandbox_id(sandbox_id)
@@ -1123,6 +1171,7 @@ class DockerSandboxService(DockerDiagnosticsMixin, DockerRuntimeMixin, DockerVol
with self._docker_operation("remove sandbox container", sandbox_id):
container.remove(force=True)
except DockerException as exc:
+ self._raise_if_lifecycle_race(sandbox_id, exc, "delete")
raise HTTPException(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
detail={
@@ -1167,6 +1216,7 @@ class DockerSandboxService(DockerDiagnosticsMixin, DockerRuntimeMixin, DockerVol
with self._docker_operation("pause sandbox container", sandbox_id):
container.pause()
except DockerException as exc:
+ self._raise_if_lifecycle_race(sandbox_id, exc, "pause")
raise HTTPException(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
detail={
@@ -1301,6 +1351,8 @@ class DockerSandboxService(DockerDiagnosticsMixin, DockerRuntimeMixin, DockerVol
message = f"Failed to resume {failed_component}: {str(exc)}"
if rollback_errors:
message += f"; rollback failed: {'; '.join(rollback_errors)}"
+ elif resuming_main:
+ self._raise_if_lifecycle_race(sandbox_id, exc, "resume")
raise HTTPException(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
detail={
diff --git a/server/tests/test_docker_service.py b/server/tests/test_docker_service.py
index 7a9bb36dd..43f07679a 100644
--- a/server/tests/test_docker_service.py
+++ b/server/tests/test_docker_service.py
@@ -20,7 +20,7 @@ from datetime import datetime, timedelta, timezone
from typing import Any, cast
from unittest.mock import MagicMock, patch
-from docker.errors import DockerException, NotFound as DockerNotFound
+from docker.errors import APIError, DockerException, NotFound as DockerNotFound
import pytest
from fastapi import HTTPException, status
from pydantic import SecretStr, ValidationError
@@ -2130,6 +2130,226 @@ def _lifecycle_container(
return container
[email protected]("failed_operation", ["kill", "remove"])
+ "docker_status, message, expected_status",
+ [
+ (409, "removal of container main-id is already in progress", 409),
+ (409, "container main-id is already being removed", 409),
+ (404, "No such container: main-id", 404),
+ (409, "unrelated daemon conflict", 500),
+ (500, "removal of container main-id is already in progress", 500),
+ ],
+)
+def test_delete_sandbox_concurrent_outcomes(
+ tmp_path, failed_operation, docker_status, message, expected_status,
+):
+ service = DockerSandboxService(config=_app_config())
+ service._metadata_store = DockerMetadataStore(tmp_path / "metadata")
+ expiration = datetime.now(timezone.utc) + timedelta(hours=1)
+ service._metadata_store.set_expiration("sandbox-id", expiration)
+ main = _lifecycle_container("main-id", running=True, paused=False)
+ error_type = DockerNotFound if docker_status == 404 else APIError
+ error = error_type(message, response=MagicMock(status_code=docker_status), explanation=message)
+ getattr(main, failed_operation).side_effect = error
+
+ with (
+ patch.object(service, "_get_container_by_sandbox_id", return_value=main),
+ patch.object(service, "_cleanup_egress_sidecar") as cleanup_sidecar,
+ patch.object(service, "_remove_expiration_tracking") as remove_tracking,
+ patch.object(service, "_cleanup_windows_oem_volume") as cleanup_oem,
+ patch.object(service, "_release_ossfs_mounts") as release_mounts,
+ patch.object(service, "_cleanup_managed_volumes") as cleanup_volumes,
+ pytest.raises(HTTPException) as exc_info,
+ ):
+ service.delete_sandbox("sandbox-id")
+
+ assert exc_info.value.status_code == expected_status
+ if expected_status == 404:
+ assert exc_info.value.detail == {
+ "code": SandboxErrorCodes.SANDBOX_NOT_FOUND,
+ "message": "Sandbox sandbox-id not found.",
+ }
+ cleanup_sidecar.assert_called_once_with("sandbox-id")
+ else:
+ assert exc_info.value.detail["code"] == SandboxErrorCodes.SANDBOX_DELETE_FAILED
+ assert exc_info.value.detail["message"]
+ cleanup_sidecar.assert_not_called()
+ if expected_status == 500:
+ assert str(error) in exc_info.value.detail["message"]
+ if failed_operation == "kill":
+ main.remove.assert_not_called()
+ remove_tracking.assert_not_called()
+ cleanup_oem.assert_not_called()
+ release_mounts.assert_not_called()
+ cleanup_volumes.assert_not_called()
+ assert service._metadata_store.get_expiration("sandbox-id") == expiration.isoformat()
+
+
[email protected]("operation", ["pause", "resume"])
+ "docker_status, outcome, disappeared, expected_status",
+ [
+ (409, "state conflict", False, 409),
+ (409, "not running", False, 409),
+ (409, "not running", True, 404),
+ (404, "No such container: main-id", True, 404),
+ (409, "unrelated daemon conflict", False, 500),
+ (500, "state conflict", False, 500),
+ ],
+)
+def test_pause_resume_sandbox_concurrent_outcomes(
+ operation, docker_status, outcome, disappeared, expected_status,
+):
+ service = DockerSandboxService(config=_app_config())
+ resuming = operation == "resume"
+ main = _lifecycle_container("main-id", running=True, paused=resuming, egress_expected=True)
+ sidecar = _lifecycle_container("sidecar-id", running=True, paused=resuming)
+ if outcome == "state conflict":
+ message = "container main-id is not paused" if resuming else "container main-id is already paused"
+ elif outcome == "not running":
+ message = "container main-id is not running"
+ else:
+ message = outcome
+ error_type = DockerNotFound if docker_status == 404 else APIError
+ error = error_type(message, response=MagicMock(status_code=docker_status), explanation=message)
+ getattr(main, "unpause" if resuming else "pause").side_effect = error
+ missing = HTTPException(status_code=404, detail={
+ "code": SandboxErrorCodes.SANDBOX_NOT_FOUND,
+ "message": "Sandbox sandbox-id not found.",
+ })
+ lookup_results = [main, missing if disappeared else main]
+
+ with (
+ patch.object(service, "_get_container_by_sandbox_id", side_effect=lookup_results),
+ patch.object(service, "_get_egress_sidecars", return_value=[sidecar]),
+ pytest.raises(HTTPException) as exc_info,
+ ):
+ getattr(service, f"{operation}_sandbox")("sandbox-id")
+
+ assert exc_info.value.status_code == expected_status
+ if expected_status == 404:
+ assert exc_info.value.detail == missing.detail
+ elif expected_status == 409:
+ assert exc_info.value.detail == {
+ "code": SandboxErrorCodes.SANDBOX_NOT_PAUSED if resuming else SandboxErrorCodes.SANDBOX_NOT_RUNNING,
+ "message": f"Sandbox is not in a {'paused' if resuming else 'running'} state.",
+ }
+ else:
+ assert exc_info.value.detail == {
+ "code": SandboxErrorCodes.SANDBOX_RESUME_FAILED if resuming else SandboxErrorCodes.SANDBOX_PAUSE_FAILED,
+ "message": f"Failed to {operation} sandbox container: {error}",
+ }
+ if resuming:
+ main.unpause.assert_called_once_with()
+ # Keep the existing sidecar rollback even when the main failure maps to 409/404.
+ sidecar.unpause.assert_called_once_with()
+ sidecar.pause.assert_called_once_with()
+ else:
+ main.pause.assert_called_once_with()
+ sidecar.pause.assert_not_called()
+ main.unpause.assert_not_called()
+
+
[email protected]("docker_status", [404, 409])
+def test_resume_sandbox_race_with_failed_rollback_remains_500(docker_status):
+ service = DockerSandboxService(config=_app_config())
+ main = _lifecycle_container("main-id", running=True, paused=True, egress_expected=True)
+ sidecar = _lifecycle_container("sidecar-id", running=True, paused=True)
+ error_type = DockerNotFound if docker_status == 404 else APIError
+ main.unpause.side_effect = error_type(
+ "resume raced", response=MagicMock(status_code=docker_status),
+ explanation="container main-id is not paused",
+ )
+ sidecar.pause.side_effect = DockerException("rollback failed")
+
+ with (
+ patch.object(service, "_get_container_by_sandbox_id", return_value=main),
+ patch.object(service, "_get_egress_sidecars", return_value=[sidecar]),
+ pytest.raises(HTTPException) as exc_info,
+ ):
+ service.resume_sandbox("sandbox-id")
+
+ assert exc_info.value.status_code == 500
+ assert exc_info.value.detail["code"] == SandboxErrorCodes.SANDBOX_RESUME_FAILED
+ assert "rollback failed" in exc_info.value.detail["message"]
+ sidecar.pause.assert_called_once_with()
+
+
[email protected]("operation", ["pause", "resume"])
[email protected]("docker_status", [404, 409])
+def test_pause_resume_sidecar_race_preserves_failure_and_rollback(operation, docker_status):
+ service = DockerSandboxService(config=_app_config())
+ resuming = operation == "resume"
+ main = _lifecycle_container("main-id", running=True, paused=resuming, egress_expected=True)
+ sidecar = _lifecycle_container("sidecar-id", running=True, paused=resuming)
+ error_type = DockerNotFound if docker_status == 404 else APIError
+ getattr(sidecar, "unpause" if resuming else "pause").side_effect = error_type(
+ "sidecar raced", response=MagicMock(status_code=docker_status),
+ explanation="container sidecar-id is not paused" if resuming else "container sidecar-id is already paused",
+ )
+
+ with (
+ patch.object(service, "_get_container_by_sandbox_id", return_value=main),
+ patch.object(service, "_get_egress_sidecars", return_value=[sidecar]),
+ pytest.raises(HTTPException) as exc_info,
+ ):
+ getattr(service, f"{operation}_sandbox")("sandbox-id")
+
+ assert exc_info.value.status_code == 500
+ assert exc_info.value.detail["code"] == (
+ SandboxErrorCodes.SANDBOX_RESUME_FAILED if resuming else SandboxErrorCodes.SANDBOX_PAUSE_FAILED
+ )
+ assert "egress sidecar" in exc_info.value.detail["message"]
+ if resuming:
+ main.unpause.assert_not_called()
+ else:
+ main.pause.assert_called_once_with()
+ main.unpause.assert_called_once_with()
+
+
[email protected]("docker_status", [404, 409, 500])
+def test_expire_sandbox_removal_race_retries_without_releasing_stale_mounts(tmp_path, docker_status):
+ service = DockerSandboxService(config=_app_config())
+ service._metadata_store = DockerMetadataStore(tmp_path / "metadata")
+ service._metadata_store.set_expiration("sandbox-id", datetime.now(timezone.utc) - timedelta(seconds=1))
+ main = _lifecycle_container("main-id", running=True, paused=False)
+ main.attrs["Config"]["Labels"][SANDBOX_OSSFS_MOUNTS_LABEL] = '["shared-mount"]'
+ error_type = DockerNotFound if docker_status == 404 else APIError
+ error = error_type(
+ "removal raced", response=MagicMock(status_code=docker_status),
+ explanation="removal of container main-id is already in progress",
+ )
+ main.remove.side_effect = error
+ if docker_status == 404:
+ main.kill.side_effect = error
+ missing = HTTPException(status_code=404, detail={
+ "code": SandboxErrorCodes.SANDBOX_NOT_FOUND,
+ "message": "Sandbox sandbox-id not found.",
+ })
+
+ with (
+ patch.object(service, "_get_container_by_sandbox_id", side_effect=[main, missing]),
+ patch.object(service, "_schedule_expiration") as schedule,
+ patch.object(service, "_cleanup_egress_sidecar") as cleanup_sidecar,
+ patch.object(service, "_release_ossfs_mounts") as release_mounts,
+ patch.object(service, "_cleanup_managed_volumes") as cleanup_volumes,
+ ):
+ service._expire_sandbox("sandbox-id")
+ schedule.assert_called_once()
+ assert schedule.call_args.args[0] == "sandbox-id"
+ assert schedule.call_args.kwargs == {"update_expiration": False}
+ cleanup_sidecar.assert_not_called()
+ cleanup_volumes.assert_not_called()
+ assert service._metadata_store.get_expiration("sandbox-id") is not None
+
+ # A concurrent DELETE finished cleanup. Retry must re-read ownership.
+ service._expire_sandbox("sandbox-id")
+ cleanup_sidecar.assert_called_once_with("sandbox-id")
+ release_mounts.assert_not_called()
+ assert service._metadata_store.get_expiration("sandbox-id") is None
+
+
def test_pause_sandbox_pauses_main_before_egress_sidecar():
service = DockerSandboxService(config=_app_config())
main = _lifecycle_container("main-id", running=True, paused=False, egress_expected=True)