Secure, Fast, and Extensible Sandbox runtime for AI agents.
fix(server): keep Docker createdAt stable and record pause/resume transitions
Jackmeson15bd9349e8fab
diff --git a/docs/architecture/control-plane/server.md b/docs/architecture/control-plane/server.mdindex 6c1b6481f..83eb5fab4 100644--- a/docs/architecture/control-plane/server.md+++ b/docs/architecture/control-plane/server.md@@ -39,6 +39,8 @@ Creation returns before the sandbox is running (`Pending`), and clients poll staA sandbox is observed as `Pending` or `Running`, moves through the `Pausing` / `Paused` / `Resuming` / `Stopping` transitions, and ends in `Terminated` (exit code 0) or `Failed` (non-zero). `resume()` is deliberately a pause-state operation, never a general restart: a workload that already exited is `Terminated` or `Failed`, and the honest answer to that is a replacement sandbox, not a silent revive. Pausing on Kubernetes keeps the sandbox ID and restores the root filesystem — with the opt-in QEMU mode, even guest memory. See [Pause & Resume](/guides/pause-resume).+For Docker sandboxes, the creation response's `createdAt` is preserved in the `opensandbox.io/created-at` container label and reused by subsequent reads. Older containers without this label use Docker's creation time. Server-managed pause and resume transitions record the main container's state and timestamp under `~/.opensandbox/metadata/_transition/`; successfully persisted records survive server restarts when that metadata directory is retained. A stored transition timestamp is used only when its state matches the observed container state. Docker's `FinishedAt` remains the timestamp for container termination, including termination outside the server. For sandboxes with a timeout, expiration remains the original request-time clock reading plus that timeout; pausing and resuming do not extend it.+For Kubernetes BatchSandbox workloads, `DELETE /sandboxes/{id}` returns `204` and SDK `kill()` returns when deletion is accepted. Poll `GET /sandboxes/{id}`: it reports `Stopping` while cleanup finishes, then returns `404`. Under normal managed deletion, non-pooled sandboxes and pools using the default `Delete` strategy wait for pods to disappear; pool `Noop` preserves processes, so its `404` does not confirm runtime termination.### Endpoint resolution is part of the contractdiff --git a/server/opensandbox_server/services/constants.py b/server/opensandbox_server/services/constants.pyindex ffa29f3b2..521772bcd 100644--- a/server/opensandbox_server/services/constants.py+++ b/server/opensandbox_server/services/constants.py@@ -21,6 +21,7 @@ RESERVED_LABEL_PREFIX = "opensandbox.io/"SANDBOX_ID_LABEL = "opensandbox.io/id"SANDBOX_TENANT_LABEL = "opensandbox.io/tenant"SANDBOX_EXPIRES_AT_LABEL = "opensandbox.io/expires-at"+SANDBOX_CREATED_AT_LABEL = "opensandbox.io/created-at"SANDBOX_MANUAL_CLEANUP_LABEL = "opensandbox.io/manual-cleanup"SANDBOX_PLATFORM_OS_LABEL = "opensandbox.io/platform-os"SANDBOX_PLATFORM_ARCH_LABEL = "opensandbox.io/platform-arch"@@ -205,6 +206,7 @@ __all__ = ["SANDBOX_ID_LABEL","SANDBOX_TENANT_LABEL","SANDBOX_EXPIRES_AT_LABEL",+ "SANDBOX_CREATED_AT_LABEL","SANDBOX_MANUAL_CLEANUP_LABEL","SANDBOX_PLATFORM_OS_LABEL","SANDBOX_PLATFORM_ARCH_LABEL",diff --git a/server/opensandbox_server/services/docker/docker_service.py b/server/opensandbox_server/services/docker/docker_service.pyindex 15e6b94e7..a64879b49 100644--- a/server/opensandbox_server/services/docker/docker_service.py+++ b/server/opensandbox_server/services/docker/docker_service.py@@ -28,10 +28,12 @@ import loggingimport mathimport osimport time+from _thread import LockTypefrom contextlib import contextmanagerfrom datetime import datetime, timedelta, timezonefrom threading import Lock, Thread, Timerfrom typing import Any, Dict, Optional+from weakref import WeakValueDictionaryimport dockerfrom docker.errors import APIError, DockerException, NotFound as DockerNotFound@@ -95,6 +97,7 @@ from opensandbox_server.services.constants import (OPENSANDBOX_EGRESS_MITMPROXY_SSL_INSECURE,OPENSANDBOX_EGRESS_MITMPROXY_TRANSPARENT,OPENSANDBOX_RUNTIME_MOUNT_PATH,+ SANDBOX_CREATED_AT_LABEL,SANDBOX_EGRESS_AUTH_TOKEN_METADATA_KEY,SANDBOX_EMBEDDING_PROXY_PORT_LABEL,SANDBOX_EXPIRES_AT_LABEL,@@ -191,6 +194,8 @@ class DockerSandboxService(DockerDiagnosticsMixin, DockerRuntimeMixin, DockerVol},)self._expiration_lock = Lock()+ self._lifecycle_locks_guard = Lock()+ self._lifecycle_locks: WeakValueDictionary[str, LockType] = WeakValueDictionary()self._execd_archive_lock = Lock()self._bootstrap_script_lock = Lock()self._sandbox_expirations: Dict[str, datetime] = {}@@ -210,6 +215,26 @@ class DockerSandboxService(DockerDiagnosticsMixin, DockerRuntimeMixin, DockerVolreturn cfgreturn 180+ @contextmanager+ def _sandbox_lifecycle_lock(self, sandbox_id: str):+ # Order runtime mutations and transition records within this service+ # instance only, not across service instances or processes. Keep this+ # separate from the metadata store's non-reentrant file-write lock.+ with self._lifecycle_locks_guard:+ lock = self._lifecycle_locks.get(sandbox_id)+ if lock is None:+ lock = Lock()+ self._lifecycle_locks[sandbox_id] = lock+ # Active and waiting calls retain the lock; idle entries disappear.+ with lock:+ yield++ def _persist_transition(self, sandbox_id: str, state: str) -> None:+ try:+ self._metadata_store.set_transition(sandbox_id, state, datetime.now(timezone.utc))+ except OSError as exc:+ logger.warning(f"Failed to persist {state} transition for sandbox {sandbox_id}: {exc}")+@contextmanagerdef _docker_operation(self, action: str, sandbox_id: Optional[str] = None):op_id = sandbox_id or "shared"@@ -342,6 +367,14 @@ class DockerSandboxService(DockerDiagnosticsMixin, DockerRuntimeMixin, DockerVolfallback_mount_keys: Optional[list[str]] = None,) -> None:"""Timer callback to terminate expired sandboxes."""+ with self._sandbox_lifecycle_lock(sandbox_id):+ self._expire_sandbox_locked(sandbox_id, fallback_mount_keys)++ def _expire_sandbox_locked(+ self,+ sandbox_id: str,+ fallback_mount_keys: Optional[list[str]] = None,+ ) -> None:mount_keys: list[str] = []try:container = self._get_container_by_sandbox_id(sandbox_id)@@ -540,11 +573,14 @@ class DockerSandboxService(DockerDiagnosticsMixin, DockerRuntimeMixin, DockerVolsnapshot_id = labels.get(SANDBOX_SNAPSHOT_ID_LABEL) or Noneimage_spec = None if snapshot_id else ImageSpec(uri=image_uri)- created_at = parse_timestamp(container.attrs.get("Created"))+ created_at = parse_timestamp(+ labels.get(SANDBOX_CREATED_AT_LABEL) or container.attrs.get("Created")+ )+ transitioned_at = self._metadata_store.get_transition_time(resolved_id, state)last_transition_at = (parse_timestamp(finished_at)if finished_at and finished_at != "0001-01-01T00:00:00Z"- else created_at+ else parse_timestamp(transitioned_at) if transitioned_at else created_at)expires_at = self._get_tracked_expiration(resolved_id, labels)@@ -708,6 +744,7 @@ class DockerSandboxService(DockerDiagnosticsMixin, DockerRuntimeMixin, DockerVolsandbox_env, egress_env = split_egress_env(request.env)patched_request = request.model_copy(update={"env": sandbox_env or None})labels, environment = self._build_labels_and_env(sandbox_id, patched_request, expires_at)+ labels[SANDBOX_CREATED_AT_LABEL] = created_at.isoformat()if auto_created_volumes:labels[SANDBOX_MANAGED_VOLUMES_LABEL] = json.dumps(auto_created_volumes, separators=(",", ":"),@@ -1143,6 +1180,10 @@ class DockerSandboxService(DockerDiagnosticsMixin, DockerRuntimeMixin, DockerVol) from excdef delete_sandbox(self, sandbox_id: str) -> None:+ with self._sandbox_lifecycle_lock(sandbox_id):+ self._delete_sandbox_locked(sandbox_id)++ def _delete_sandbox_locked(self, sandbox_id: str) -> None:try:container = self._get_container_by_sandbox_id(sandbox_id)except HTTPException as exc:@@ -1187,6 +1228,10 @@ class DockerSandboxService(DockerDiagnosticsMixin, DockerRuntimeMixin, DockerVolself._metadata_store.delete(sandbox_id)def pause_sandbox(self, sandbox_id: str) -> None:+ with self._sandbox_lifecycle_lock(sandbox_id):+ self._pause_sandbox_locked(sandbox_id)++ def _pause_sandbox_locked(self, sandbox_id: str) -> None:container = self._get_container_by_sandbox_id(sandbox_id)state = container.attrs.get("State", {})if not state.get("Running", False) or state.get("Paused", False):@@ -1224,6 +1269,7 @@ class DockerSandboxService(DockerDiagnosticsMixin, DockerRuntimeMixin, DockerVol"message": f"Failed to pause sandbox container: {str(exc)}",},) from exc+ self._persist_transition(sandbox_id, "Paused")if egress_expected and not sidecars:raise HTTPException(@@ -1269,6 +1315,8 @@ class DockerSandboxService(DockerDiagnosticsMixin, DockerRuntimeMixin, DockerVolf"sandbox={sandbox_id} | failed to rollback sandbox pause: {rollback_exc}")rollback_errors.append(str(rollback_exc))+ else:+ self._persist_transition(sandbox_id, "Running")message = f"Failed to pause egress sidecar: {str(exc)}"if rollback_errors:@@ -1282,6 +1330,10 @@ class DockerSandboxService(DockerDiagnosticsMixin, DockerRuntimeMixin, DockerVol) from excdef resume_sandbox(self, sandbox_id: str) -> None:+ with self._sandbox_lifecycle_lock(sandbox_id):+ self._resume_sandbox_locked(sandbox_id)++ def _resume_sandbox_locked(self, sandbox_id: str) -> None:container = self._get_container_by_sandbox_id(sandbox_id)state = container.attrs.get("State", {})if not state.get("Paused", False):@@ -1360,6 +1412,7 @@ class DockerSandboxService(DockerDiagnosticsMixin, DockerRuntimeMixin, DockerVol"message": message,},) from exc+ self._persist_transition(sandbox_id, "Running")def get_access_renew_extend_seconds(self, sandbox_id: str) -> Optional[int]:try:diff --git a/server/opensandbox_server/services/docker/metadata.py b/server/opensandbox_server/services/docker/metadata.pyindex cfb42219c..5607d868b 100644--- a/server/opensandbox_server/services/docker/metadata.py+++ b/server/opensandbox_server/services/docker/metadata.py@@ -14,8 +14,8 @@"""Per-sandbox file-backed metadata store for Docker.-Docker cannot update labels on running containers, so user metadata patches-and expiration overrides are persisted to individual JSON files under+Docker cannot update labels on running containers, so user metadata patches,+expiration overrides, and lifecycle transitions are persisted to JSON files under``~/.opensandbox/metadata/``. Atomic writes (tmp + rename) guard againsttruncation during crash, and one lock serializes every read-modify-write soconcurrent patches of one sandbox neither fail nor drop each other's keys.@@ -66,7 +66,7 @@ def _extract_user_labels(labels: dict) -> dict:class DockerMetadataStore:- """Per-sandbox file-backed store for user metadata and expiration overrides."""+ """Per-sandbox file-backed store for metadata, expiration, and transitions."""def __init__(self, root: Path | None = None) -> None:self._root = root or DEFAULT_STORE_DIR@@ -132,9 +132,29 @@ class DockerMetadataStore:{"expires_at": expires_at.isoformat()},)+ def get_transition_time(self, sandbox_id: str, state: str) -> str | None:+ payload = self._read_file(self._transition_path(sandbox_id))+ if not isinstance(payload, dict) or payload.get("state") != state:+ return None+ transitioned_at = payload.get("last_transition_at")+ if isinstance(transitioned_at, str) and transitioned_at.strip():+ return transitioned_at+ return None++ def set_transition(self, sandbox_id: str, state: str, transitioned_at: datetime) -> None:+ with self._lock:+ self._write_file(+ self._transition_path(sandbox_id),+ {"state": state, "last_transition_at": transitioned_at.isoformat()},+ )+def delete(self, sandbox_id: str) -> None:with self._lock:- for path in (self._sandbox_path(sandbox_id), self._expiration_path(sandbox_id)):+ for path in (+ self._sandbox_path(sandbox_id),+ self._expiration_path(sandbox_id),+ self._transition_path(sandbox_id),+ ):try:path.unlink(missing_ok=True)except OSError as exc:@@ -146,6 +166,9 @@ class DockerMetadataStore:def _expiration_path(self, sandbox_id: str) -> Path:return self._root / "_expiration" / f"{sandbox_id}.json"+ def _transition_path(self, sandbox_id: str) -> Path:+ return self._root / "_transition" / f"{sandbox_id}.json"+@staticmethoddef _read_file(path: Path) -> dict | None:try:diff --git a/server/tests/test_docker_metadata_store.py b/server/tests/test_docker_metadata_store.pyindex 9f1ecf739..028beb4a5 100644--- a/server/tests/test_docker_metadata_store.py+++ b/server/tests/test_docker_metadata_store.py@@ -243,6 +243,19 @@ class TestDockerMetadataStore:# -- crash recovery ------------------------------------------------------+ def test_transition_is_state_associated_and_separate_from_user_metadata(self, tmp_path: Path):+ store = self._make_store(tmp_path)+ transitioned_at = datetime(2030, 1, 1, tzinfo=timezone.utc)+ store.set_transition("sbx-001", "Paused", transitioned_at)+ store.patch("sbx-001", {}, {"state": "user-value"})++ reconstructed = self._make_store(tmp_path)+ assert reconstructed.get_transition_time("sbx-001", "Paused") == transitioned_at.isoformat()+ assert reconstructed.get_transition_time("sbx-001", "Running") is None+ assert reconstructed.get("sbx-001", {}) == {"state": "user-value"}+ reconstructed.delete("sbx-001")+ assert reconstructed.get_transition_time("sbx-001", "Paused") is None+def test_tmp_file_not_read_on_reload(self, tmp_path: Path):"""Tmp files written during atomic save must not be picked up by get()."""root = tmp_path / "metadata"diff --git a/server/tests/test_docker_service.py b/server/tests/test_docker_service.pyindex 43f07679a..e83fa403f 100644--- a/server/tests/test_docker_service.py+++ b/server/tests/test_docker_service.py@@ -95,6 +95,14 @@ def _app_config() -> AppConfig:ingress=IngressConfig(mode="direct"),)+[email protected](autouse=True)+def isolated_metadata_store(monkeypatch, tmp_path):+ monkeypatch.setattr(+ "opensandbox_server.services.docker.metadata.DEFAULT_STORE_DIR", tmp_path / "metadata"+ )++def test_parse_memory_limit_handles_units():assert parse_memory_limit("512Mi") == 512 * 1024 * 1024assert parse_memory_limit("1G") == 1_000_000_000@@ -2223,11 +2231,13 @@ def test_pause_resume_sandbox_concurrent_outcomes(with (patch.object(service, "_get_container_by_sandbox_id", side_effect=lookup_results),patch.object(service, "_get_egress_sidecars", return_value=[sidecar]),+ patch.object(service._metadata_store, "set_transition") as set_transition,pytest.raises(HTTPException) as exc_info,):getattr(service, f"{operation}_sandbox")("sandbox-id")assert exc_info.value.status_code == expected_status+ set_transition.assert_not_called()if expected_status == 404:assert exc_info.value.detail == missing.detailelif expected_status == 409:@@ -5356,6 +5366,435 @@ def _mock_container(return container[email protected]("timeout", [None, 120])+async def test_create_get_timestamp_consistency_preserves_ttl(timeout):+ requested_at = datetime(2030, 1, 1, tzinfo=timezone.utc)+ created_at = requested_at + timedelta(seconds=2, microseconds=123456)+ container = _mock_container("timestamp-test")+ container.attrs["Created"] = created_at.isoformat()+ docker_client = MagicMock()+ docker_client.containers.list.return_value = []+ docker_client.containers.get.return_value = container++ def create_container(sandbox_id, image, entrypoint, labels, *args, **kwargs):+ container.attrs["Config"]["Labels"] = dict(labels)+ return container++ with (+ patch("docker.from_env", return_value=docker_client),+ patch("opensandbox_server.services.docker.docker_service.datetime", wraps=datetime) as clock,+ ):+ clock.now.return_value = requested_at+ service = DockerSandboxService(config=_app_config())+ request = CreateSandboxRequest(+ image=ImageSpec(uri="python:3.11"), timeout=timeout,+ resourceLimits=ResourceLimits(root={}), entrypoint=["python"],+ )+ with (+ patch.object(service, "generate_sandbox_id", return_value="timestamp-test"),+ patch.object(service, "_ensure_image_available"),+ patch.object(service, "_create_and_start_container", side_effect=create_container),+ patch.object(service, "_schedule_expiration") as schedule,+ ):+ created = await service.create_sandbox(request)++ expected_expiration = requested_at + timedelta(seconds=timeout) if timeout else None+ assert created.expires_at == expected_expiration+ if timeout:+ schedule.assert_called_once_with(created.id, expected_expiration)+ else:+ schedule.assert_not_called()+ clock.now.return_value = requested_at + timedelta(seconds=10)+ for _ in range(2):+ fetched = service.get_sandbox(created.id)+ assert created.created_at == fetched.created_at == requested_at+ assert created.status.last_transition_at == fetched.status.last_transition_at == requested_at+ assert fetched.expires_at == expected_expiration+ reconstructed = DockerSandboxService(config=_app_config()).get_sandbox(created.id)+ assert reconstructed.created_at == requested_at+ assert reconstructed.expires_at == expected_expiration++[email protected]("exit_code", [0, 137])+def test_pause_resume_transition_timestamps_survive_reconstruction(exit_code):+ created_at = datetime(2030, 1, 1, tzinfo=timezone.utc)+ expires_at = created_at + timedelta(hours=1)+ container = _mock_container(+ "timestamp-test", extra_labels={SANDBOX_EXPIRES_AT_LABEL: expires_at.isoformat()}+ )+ container.attrs["Created"] = created_at.isoformat()+ state = container.attrs["State"]+ container.pause.side_effect = lambda: state.update(Paused=True)+ container.unpause.side_effect = lambda: state.update(Paused=False)+ docker_client = MagicMock()+ docker_client.containers.list.return_value = []+ docker_client.containers.get.return_value = container++ with (+ patch("docker.from_env", return_value=docker_client),+ patch("opensandbox_server.services.docker.docker_service.datetime", wraps=datetime) as clock,+ ):+ service = DockerSandboxService(config=_app_config())+ previous_transition = created_at+ for step, expected_state in enumerate(["Paused", "Running", "Paused", "Running"], 1):+ transitioned_at = created_at + timedelta(seconds=step * 10)+ clock.now.return_value = transitioned_at+ if expected_state == "Paused":+ service.pause_sandbox("timestamp-test")+ else:+ service.resume_sandbox("timestamp-test")+ clock.now.return_value += timedelta(seconds=1)+ # Each new service must read the timestamp from disk, not process memory.+ service = DockerSandboxService(config=_app_config())+ for _ in range(2):+ fetched = service.get_sandbox("timestamp-test")+ assert fetched.status.state == expected_state+ assert fetched.status.last_transition_at == transitioned_at > previous_transition+ assert fetched.created_at == created_at+ assert fetched.expires_at == expires_at+ previous_transition = transitioned_at++ finished_at = created_at + timedelta(minutes=1, microseconds=500000)+ state.update(Running=False, Status="exited", ExitCode=exit_code, FinishedAt=finished_at.isoformat())+ service = DockerSandboxService(config=_app_config())+ terminated = service.get_sandbox("timestamp-test")+ assert terminated.status.state == ("Terminated" if exit_code == 0 else "Failed")+ assert terminated.status.last_transition_at == finished_at+ assert terminated.created_at == created_at+ assert terminated.expires_at == expires_at+ service.delete_sandbox("timestamp-test")+ assert not list(service._metadata_store._root.rglob("timestamp-test.json"))++[email protected]("sidecar_missing", [False, True])+def test_pause_failure_timestamp_matches_actual_state(sidecar_missing):+ paused_at = datetime(2030, 1, 1, tzinfo=timezone.utc)+ rolled_back_at = paused_at + timedelta(seconds=1)+ container = _mock_container(+ "timestamp-test", extra_labels={SANDBOX_EGRESS_AUTH_TOKEN_METADATA_KEY: "token"}+ )+ state = container.attrs["State"]+ container.pause.side_effect = lambda: state.update(Paused=True)+ container.unpause.side_effect = lambda: state.update(Paused=False)+ sidecar = _lifecycle_container("sidecar", running=True, paused=False)+ sidecar.pause.side_effect = DockerException("sidecar pause failed")+ service = DockerSandboxService(config=_app_config())++ with (+ patch.object(service, "_get_container_by_sandbox_id", return_value=container),+ patch.object(service, "_get_egress_sidecars", return_value=[] if sidecar_missing else [sidecar]),+ patch("opensandbox_server.services.docker.docker_service.datetime", wraps=datetime) as clock,+ ):+ clock.now.side_effect = [paused_at, rolled_back_at]+ with pytest.raises(HTTPException) as exc_info:+ service.pause_sandbox("timestamp-test")+ assert exc_info.value.status_code == 500+ fetched = service.get_sandbox("timestamp-test")+ assert fetched.status.state == ("Paused" if sidecar_missing else "Running")+ assert fetched.status.last_transition_at == (paused_at if sidecar_missing else rolled_back_at)++[email protected]("operation", ["pause", "resume"])+def test_transition_write_failure_preserves_successful_lifecycle(operation):+ service = DockerSandboxService(config=_app_config())+ resuming = operation == "resume"+ main = _lifecycle_container("main", running=True, paused=resuming, egress_expected=True)+ sidecar = _lifecycle_container("sidecar", running=True, paused=resuming)+ for container in (main, sidecar):+ container.pause.side_effect = lambda c=container: c.attrs["State"].update(Paused=True)+ container.unpause.side_effect = lambda c=container: c.attrs["State"].update(Paused=False)++ with (+ patch.object(service, "_get_container_by_sandbox_id", return_value=main),+ patch.object(service, "_get_egress_sidecars", return_value=[sidecar]),+ patch.object(service._metadata_store, "_write_file", side_effect=OSError("disk full")),+ patch.object(service, "_raise_if_lifecycle_race") as map_race,+ patch("opensandbox_server.services.docker.docker_service.logger.warning") as warning,+ ):+ assert getattr(service, f"{operation}_sandbox")("sandbox-id") is None++ for container in (main, sidecar):+ assert container.attrs["State"] == {"Running": True, "Paused": not resuming}+ getattr(container, "unpause" if resuming else "pause").assert_called_once_with()+ getattr(container, "pause" if resuming else "unpause").assert_not_called()+ map_race.assert_not_called()+ transition_state = "Running" if resuming else "Paused"+ warning.assert_called_once_with(+ f"Failed to persist {transition_state} transition for sandbox sandbox-id: disk full"+ )++[email protected]("failed_state", ["Paused", "Running"])+def test_transition_write_failure_preserves_pause_rollback_error(failed_state):+ service = DockerSandboxService(config=_app_config())+ main = _lifecycle_container("main", running=True, paused=False, egress_expected=True)+ sidecar = _lifecycle_container("sidecar", running=True, paused=False)+ failing_sidecar = _lifecycle_container("failing", running=True, paused=False)+ for container in (main, sidecar):+ container.pause.side_effect = lambda c=container: c.attrs["State"].update(Paused=True)+ container.unpause.side_effect = lambda c=container: c.attrs["State"].update(Paused=False)+ failing_sidecar.pause.side_effect = DockerException("sidecar pause failed")+ write_file = service._metadata_store._write_file++ def fail_transition_write(path, payload):+ if payload["state"] == failed_state:+ raise OSError("disk full")+ write_file(path, payload)++ with (+ patch.object(service, "_get_container_by_sandbox_id", return_value=main),+ patch.object(service, "_get_egress_sidecars", return_value=[sidecar, failing_sidecar]),+ patch.object(service._metadata_store, "_write_file", side_effect=fail_transition_write),+ patch.object(service, "_raise_if_lifecycle_race") as map_race,+ patch("opensandbox_server.services.docker.docker_service.logger.warning") as warning,+ pytest.raises(HTTPException) as exc_info,+ ):+ service.pause_sandbox("sandbox-id")++ assert exc_info.value.status_code == 500+ assert exc_info.value.detail == {+ "code": SandboxErrorCodes.SANDBOX_PAUSE_FAILED,+ "message": "Failed to pause egress sidecar: sidecar pause failed",+ }+ assert isinstance(exc_info.value.__cause__, DockerException)+ for container in (main, sidecar):+ assert container.attrs["State"] == {"Running": True, "Paused": False}+ container.pause.assert_called_once_with()+ container.unpause.assert_called_once_with()+ failing_sidecar.pause.assert_called_once_with()+ map_race.assert_not_called()+ warning.assert_any_call(+ f"Failed to persist {failed_state} transition for sandbox sandbox-id: disk full"+ )+++def test_failed_pause_rollback_does_not_record_running_transition():+ service = DockerSandboxService(config=_app_config())+ main = _lifecycle_container("main", running=True, paused=False, egress_expected=True)+ main.pause.side_effect = lambda: main.attrs["State"].update(Paused=True)+ main.unpause.side_effect = DockerException("main rollback failed")+ sidecar = _lifecycle_container("sidecar", running=True, paused=False)+ sidecar.pause.side_effect = DockerException("sidecar pause failed")++ with (+ patch.object(service, "_get_container_by_sandbox_id", return_value=main),+ patch.object(service, "_get_egress_sidecars", return_value=[sidecar]),+ pytest.raises(HTTPException) as exc_info,+ ):+ service.pause_sandbox("sandbox-id")++ assert exc_info.value.status_code == 500+ assert exc_info.value.detail == {+ "code": SandboxErrorCodes.SANDBOX_PAUSE_FAILED,+ "message": "Failed to pause egress sidecar: sidecar pause failed; rollback failed: main rollback failed",+ }+ assert main.attrs["State"]["Paused"] is True+ assert service._metadata_store.get_transition_time("sandbox-id", "Paused") is not None+ assert service._metadata_store.get_transition_time("sandbox-id", "Running") is None+++def _run_lifecycle_interleaving(first_operation, second_operation, first_blocked, release_first):+ second_checkpoint = threading.Event()+ errors: list[Exception] = []++ class ObservedLock:+ """Observe a waiting contender without sleeps or scheduler timing assumptions."""++ def __init__(self):+ self._lock = threading.Lock()++ def __enter__(self):+ if threading.current_thread() is second_thread and self._lock.locked():+ second_checkpoint.set()+ return self._lock.__enter__()++ def __exit__(self, *args):+ return self._lock.__exit__(*args)++ def run(operation):+ try:+ operation()+ except Exception as exc:+ errors.append(exc)++ def run_second():+ run(second_operation)+ # Without a lifecycle lock, let the contender finish before releasing+ # the first operation to reproduce the stale write deterministically.+ second_checkpoint.set()++ first_thread = threading.Thread(target=run, args=(first_operation,), daemon=True)+ second_thread = threading.Thread(target=run_second, daemon=True)+ with patch("opensandbox_server.services.docker.docker_service.Lock", ObservedLock):+ first_thread.start()+ try:+ assert first_blocked.wait(5), "First operation did not reach the test gate"+ second_thread.start()+ assert second_checkpoint.wait(5), "Contender neither waited for the lock nor finished"+ finally:+ release_first.set()+ first_thread.join(5)+ if second_thread.ident is not None:+ second_thread.join(5)+ assert not first_thread.is_alive() and not second_thread.is_alive()+ assert not errors, errors+++ "operation, concurrent_operation",+ [+ ("pause", "resume"),+ ("pause", "delete"),+ ("resume", "delete"),+ ("pause", "expire"),+ ("resume", "expire"),+ ("pause", "expire_missing"),+ ("resume", "expire_missing"),+ ],+)+def test_delayed_transition_write_is_ordered_with_lifecycle(operation, concurrent_operation):+ service = DockerSandboxService(config=_app_config())+ main = _mock_container("sandbox-id", paused=operation == "resume")+ state = main.attrs["State"]+ main.pause.side_effect = lambda: state.update(Paused=True)+ main.unpause.side_effect = lambda: state.update(Paused=False)+ first_blocked, release_first = threading.Event(), threading.Event()+ events: list[str] = []+ transition_times: dict[str, datetime] = {}+ first_state = "Paused" if operation == "pause" else "Running"+ set_transition = service._metadata_store.set_transition+ delete_metadata = service._metadata_store.delete++ def delayed_transition(sandbox_id, transition_state, transitioned_at):+ transition_times[transition_state] = transitioned_at+ if transition_state == first_state:+ first_blocked.set()+ assert release_first.wait(5)+ set_transition(sandbox_id, transition_state, transitioned_at)+ events.append(f"persist {transition_state}")++ def lookup(sandbox_id):+ if concurrent_operation == "expire_missing" and first_blocked.is_set():+ raise HTTPException(status_code=404, detail={})+ return main++ def cleanup(sandbox_id):+ delete_metadata(sandbox_id)+ events.append("cleanup")++ second_operation = (+ service._expire_sandbox if concurrent_operation.startswith("expire")+ else getattr(service, f"{concurrent_operation}_sandbox")+ )+ with (+ patch.object(service, "_get_container_by_sandbox_id", side_effect=lookup),+ patch.object(service, "_get_egress_sidecars", return_value=[]),+ patch.object(service, "_cleanup_egress_sidecar"),+ patch.object(service._metadata_store, "set_transition", side_effect=delayed_transition),+ patch.object(service._metadata_store, "delete", side_effect=cleanup),+ ):+ _run_lifecycle_interleaving(+ lambda: getattr(service, f"{operation}_sandbox")("sandbox-id"),+ lambda: second_operation("sandbox-id"),+ first_blocked, release_first,+ )+ if concurrent_operation == "resume":+ assert events == ["persist Paused", "persist Running"]+ fetched = service.get_sandbox("sandbox-id")+ assert fetched.status.state == "Running"+ assert fetched.status.last_transition_at == transition_times["Running"]+ else:+ assert events == [f"persist {first_state}", "cleanup"]+ assert not list(service._metadata_store._root.rglob("sandbox-id.json"))+ if concurrent_operation == "expire_missing":+ main.remove.assert_not_called()+ else:+ main.remove.assert_called_once_with(force=True)++[email protected]("cleanup_operation", ["delete", "expire"])[email protected]("operation", ["pause", "resume"])+def test_transition_waits_for_cleanup_before_container_lookup(cleanup_operation, operation):+ service = DockerSandboxService(config=_app_config())+ main = _mock_container("sandbox-id", paused=operation == "resume")+ first_blocked, release_first, removed = (+ threading.Event(), threading.Event(), threading.Event()+ )+ outcomes: list[int | None] = []++ def delayed_kill():+ first_blocked.set()+ assert release_first.wait(5)+ main.attrs["State"]["Running"] = False++ main.kill.side_effect = delayed_kill+ main.remove.side_effect = lambda **kwargs: removed.set()++ def lookup(sandbox_id):+ if removed.is_set():+ raise HTTPException(status_code=404, detail={})+ return main++ def transition():+ try:+ getattr(service, f"{operation}_sandbox")("sandbox-id")+ except HTTPException as exc:+ outcomes.append(exc.status_code)+ else:+ outcomes.append(None)++ cleanup = service.delete_sandbox if cleanup_operation == "delete" else service._expire_sandbox+ with (+ patch.object(service, "_get_container_by_sandbox_id", side_effect=lookup),+ patch.object(service, "_get_egress_sidecars", return_value=[]),+ patch.object(service, "_cleanup_egress_sidecar"),+ patch.object(service._metadata_store, "set_transition") as set_transition,+ ):+ _run_lifecycle_interleaving(+ lambda: cleanup("sandbox-id"), transition, first_blocked, release_first,+ )++ assert outcomes == [404]+ main.pause.assert_not_called()+ main.unpause.assert_not_called()+ set_transition.assert_not_called()+ assert not list(service._metadata_store._root.rglob("sandbox-id.json"))+++def test_delayed_transition_does_not_block_another_sandbox():+ service = DockerSandboxService(config=_app_config())+ containers = {sandbox_id: _mock_container(sandbox_id) for sandbox_id in ("first", "second")}+ for container in containers.values():+ container.pause.side_effect = lambda c=container: c.attrs["State"].update(Paused=True)+ first_blocked, release_first = threading.Event(), threading.Event()+ events: list[str] = []+ set_transition = service._metadata_store.set_transition++ def delayed_transition(sandbox_id, state, transitioned_at):+ if sandbox_id == "first":+ first_blocked.set()+ assert release_first.wait(5)+ set_transition(sandbox_id, state, transitioned_at)+ events.append(sandbox_id)++ with (+ patch.object(service, "_get_container_by_sandbox_id", side_effect=containers.__getitem__),+ patch.object(service, "_get_egress_sidecars", return_value=[]),+ patch.object(service._metadata_store, "set_transition", side_effect=delayed_transition),+ ):+ _run_lifecycle_interleaving(+ lambda: service.pause_sandbox("first"),+ lambda: service.pause_sandbox("second"),+ first_blocked, release_first,+ )++ assert events == ["second", "first"]+ for sandbox_id, container in containers.items():+ assert container.attrs["State"]["Paused"] is True+ assert service._metadata_store.get_transition_time(sandbox_id, "Paused") is not None++def _wire_list_mocks(mock_client: MagicMock, containers: list[MagicMock]) -> None:"""Wire ``api.containers`` (summary) + ``containers.get`` (inspect) sothat list_sandboxes' two-step discovery returns the given containers."""