Skip to content

Secure, Fast, and Extensible Sandbox runtime for AI agents.

fix(server): keep Docker createdAt stable and record pause/resume transitions

Jackmeson15bd9349e8fab
diff --git a/docs/architecture/control-plane/server.md b/docs/architecture/control-plane/server.md
index 6c1b6481f..83eb5fab4 100644
--- a/docs/architecture/control-plane/server.md
+++ b/docs/architecture/control-plane/server.md
@@ -39,6 +39,8 @@ Creation returns before the sandbox is running (`Pending`), and clients poll sta
A sandbox is observed as `Pending` or `Running`, moves through the `Pausing` / `Paused` / `Resuming` / `Stopping` transitions, and ends in `Terminated` (exit code 0) or `Failed` (non-zero). `resume()` is deliberately a pause-state operation, never a general restart: a workload that already exited is `Terminated` or `Failed`, and the honest answer to that is a replacement sandbox, not a silent revive. Pausing on Kubernetes keeps the sandbox ID and restores the root filesystem — with the opt-in QEMU mode, even guest memory. See [Pause & Resume](/guides/pause-resume).
+For Docker sandboxes, the creation response's `createdAt` is preserved in the `opensandbox.io/created-at` container label and reused by subsequent reads. Older containers without this label use Docker's creation time. Server-managed pause and resume transitions record the main container's state and timestamp under `~/.opensandbox/metadata/_transition/`; successfully persisted records survive server restarts when that metadata directory is retained. A stored transition timestamp is used only when its state matches the observed container state. Docker's `FinishedAt` remains the timestamp for container termination, including termination outside the server. For sandboxes with a timeout, expiration remains the original request-time clock reading plus that timeout; pausing and resuming do not extend it.
+
For Kubernetes BatchSandbox workloads, `DELETE /sandboxes/{id}` returns `204` and SDK `kill()` returns when deletion is accepted. Poll `GET /sandboxes/{id}`: it reports `Stopping` while cleanup finishes, then returns `404`. Under normal managed deletion, non-pooled sandboxes and pools using the default `Delete` strategy wait for pods to disappear; pool `Noop` preserves processes, so its `404` does not confirm runtime termination.
### Endpoint resolution is part of the contract
diff --git a/server/opensandbox_server/services/constants.py b/server/opensandbox_server/services/constants.py
index ffa29f3b2..521772bcd 100644
--- a/server/opensandbox_server/services/constants.py
+++ b/server/opensandbox_server/services/constants.py
@@ -21,6 +21,7 @@ RESERVED_LABEL_PREFIX = "opensandbox.io/"
SANDBOX_ID_LABEL = "opensandbox.io/id"
SANDBOX_TENANT_LABEL = "opensandbox.io/tenant"
SANDBOX_EXPIRES_AT_LABEL = "opensandbox.io/expires-at"
+SANDBOX_CREATED_AT_LABEL = "opensandbox.io/created-at"
SANDBOX_MANUAL_CLEANUP_LABEL = "opensandbox.io/manual-cleanup"
SANDBOX_PLATFORM_OS_LABEL = "opensandbox.io/platform-os"
SANDBOX_PLATFORM_ARCH_LABEL = "opensandbox.io/platform-arch"
@@ -205,6 +206,7 @@ __all__ = [
"SANDBOX_ID_LABEL",
"SANDBOX_TENANT_LABEL",
"SANDBOX_EXPIRES_AT_LABEL",
+ "SANDBOX_CREATED_AT_LABEL",
"SANDBOX_MANUAL_CLEANUP_LABEL",
"SANDBOX_PLATFORM_OS_LABEL",
"SANDBOX_PLATFORM_ARCH_LABEL",
diff --git a/server/opensandbox_server/services/docker/docker_service.py b/server/opensandbox_server/services/docker/docker_service.py
index 15e6b94e7..a64879b49 100644
--- a/server/opensandbox_server/services/docker/docker_service.py
+++ b/server/opensandbox_server/services/docker/docker_service.py
@@ -28,10 +28,12 @@ import logging
import math
import os
import time
+from _thread import LockType
from contextlib import contextmanager
from datetime import datetime, timedelta, timezone
from threading import Lock, Thread, Timer
from typing import Any, Dict, Optional
+from weakref import WeakValueDictionary
import docker
from docker.errors import APIError, DockerException, NotFound as DockerNotFound
@@ -95,6 +97,7 @@ from opensandbox_server.services.constants import (
OPENSANDBOX_EGRESS_MITMPROXY_SSL_INSECURE,
OPENSANDBOX_EGRESS_MITMPROXY_TRANSPARENT,
OPENSANDBOX_RUNTIME_MOUNT_PATH,
+ SANDBOX_CREATED_AT_LABEL,
SANDBOX_EGRESS_AUTH_TOKEN_METADATA_KEY,
SANDBOX_EMBEDDING_PROXY_PORT_LABEL,
SANDBOX_EXPIRES_AT_LABEL,
@@ -191,6 +194,8 @@ class DockerSandboxService(DockerDiagnosticsMixin, DockerRuntimeMixin, DockerVol
},
)
self._expiration_lock = Lock()
+ self._lifecycle_locks_guard = Lock()
+ self._lifecycle_locks: WeakValueDictionary[str, LockType] = WeakValueDictionary()
self._execd_archive_lock = Lock()
self._bootstrap_script_lock = Lock()
self._sandbox_expirations: Dict[str, datetime] = {}
@@ -210,6 +215,26 @@ class DockerSandboxService(DockerDiagnosticsMixin, DockerRuntimeMixin, DockerVol
return cfg
return 180
+ @contextmanager
+ def _sandbox_lifecycle_lock(self, sandbox_id: str):
+ # Order runtime mutations and transition records within this service
+ # instance only, not across service instances or processes. Keep this
+ # separate from the metadata store's non-reentrant file-write lock.
+ with self._lifecycle_locks_guard:
+ lock = self._lifecycle_locks.get(sandbox_id)
+ if lock is None:
+ lock = Lock()
+ self._lifecycle_locks[sandbox_id] = lock
+ # Active and waiting calls retain the lock; idle entries disappear.
+ with lock:
+ yield
+
+ def _persist_transition(self, sandbox_id: str, state: str) -> None:
+ try:
+ self._metadata_store.set_transition(sandbox_id, state, datetime.now(timezone.utc))
+ except OSError as exc:
+ logger.warning(f"Failed to persist {state} transition for sandbox {sandbox_id}: {exc}")
+
@contextmanager
def _docker_operation(self, action: str, sandbox_id: Optional[str] = None):
op_id = sandbox_id or "shared"
@@ -342,6 +367,14 @@ class DockerSandboxService(DockerDiagnosticsMixin, DockerRuntimeMixin, DockerVol
fallback_mount_keys: Optional[list[str]] = None,
) -> None:
"""Timer callback to terminate expired sandboxes."""
+ with self._sandbox_lifecycle_lock(sandbox_id):
+ self._expire_sandbox_locked(sandbox_id, fallback_mount_keys)
+
+ def _expire_sandbox_locked(
+ self,
+ sandbox_id: str,
+ fallback_mount_keys: Optional[list[str]] = None,
+ ) -> None:
mount_keys: list[str] = []
try:
container = self._get_container_by_sandbox_id(sandbox_id)
@@ -540,11 +573,14 @@ class DockerSandboxService(DockerDiagnosticsMixin, DockerRuntimeMixin, DockerVol
snapshot_id = labels.get(SANDBOX_SNAPSHOT_ID_LABEL) or None
image_spec = None if snapshot_id else ImageSpec(uri=image_uri)
- created_at = parse_timestamp(container.attrs.get("Created"))
+ created_at = parse_timestamp(
+ labels.get(SANDBOX_CREATED_AT_LABEL) or container.attrs.get("Created")
+ )
+ transitioned_at = self._metadata_store.get_transition_time(resolved_id, state)
last_transition_at = (
parse_timestamp(finished_at)
if finished_at and finished_at != "0001-01-01T00:00:00Z"
- else created_at
+ else parse_timestamp(transitioned_at) if transitioned_at else created_at
)
expires_at = self._get_tracked_expiration(resolved_id, labels)
@@ -708,6 +744,7 @@ class DockerSandboxService(DockerDiagnosticsMixin, DockerRuntimeMixin, DockerVol
sandbox_env, egress_env = split_egress_env(request.env)
patched_request = request.model_copy(update={"env": sandbox_env or None})
labels, environment = self._build_labels_and_env(sandbox_id, patched_request, expires_at)
+ labels[SANDBOX_CREATED_AT_LABEL] = created_at.isoformat()
if auto_created_volumes:
labels[SANDBOX_MANAGED_VOLUMES_LABEL] = json.dumps(
auto_created_volumes, separators=(",", ":"),
@@ -1143,6 +1180,10 @@ class DockerSandboxService(DockerDiagnosticsMixin, DockerRuntimeMixin, DockerVol
) from exc
def delete_sandbox(self, sandbox_id: str) -> None:
+ with self._sandbox_lifecycle_lock(sandbox_id):
+ self._delete_sandbox_locked(sandbox_id)
+
+ def _delete_sandbox_locked(self, sandbox_id: str) -> None:
try:
container = self._get_container_by_sandbox_id(sandbox_id)
except HTTPException as exc:
@@ -1187,6 +1228,10 @@ class DockerSandboxService(DockerDiagnosticsMixin, DockerRuntimeMixin, DockerVol
self._metadata_store.delete(sandbox_id)
def pause_sandbox(self, sandbox_id: str) -> None:
+ with self._sandbox_lifecycle_lock(sandbox_id):
+ self._pause_sandbox_locked(sandbox_id)
+
+ def _pause_sandbox_locked(self, sandbox_id: str) -> None:
container = self._get_container_by_sandbox_id(sandbox_id)
state = container.attrs.get("State", {})
if not state.get("Running", False) or state.get("Paused", False):
@@ -1224,6 +1269,7 @@ class DockerSandboxService(DockerDiagnosticsMixin, DockerRuntimeMixin, DockerVol
"message": f"Failed to pause sandbox container: {str(exc)}",
},
) from exc
+ self._persist_transition(sandbox_id, "Paused")
if egress_expected and not sidecars:
raise HTTPException(
@@ -1269,6 +1315,8 @@ class DockerSandboxService(DockerDiagnosticsMixin, DockerRuntimeMixin, DockerVol
f"sandbox={sandbox_id} | failed to rollback sandbox pause: {rollback_exc}"
)
rollback_errors.append(str(rollback_exc))
+ else:
+ self._persist_transition(sandbox_id, "Running")
message = f"Failed to pause egress sidecar: {str(exc)}"
if rollback_errors:
@@ -1282,6 +1330,10 @@ class DockerSandboxService(DockerDiagnosticsMixin, DockerRuntimeMixin, DockerVol
) from exc
def resume_sandbox(self, sandbox_id: str) -> None:
+ with self._sandbox_lifecycle_lock(sandbox_id):
+ self._resume_sandbox_locked(sandbox_id)
+
+ def _resume_sandbox_locked(self, sandbox_id: str) -> None:
container = self._get_container_by_sandbox_id(sandbox_id)
state = container.attrs.get("State", {})
if not state.get("Paused", False):
@@ -1360,6 +1412,7 @@ class DockerSandboxService(DockerDiagnosticsMixin, DockerRuntimeMixin, DockerVol
"message": message,
},
) from exc
+ self._persist_transition(sandbox_id, "Running")
def get_access_renew_extend_seconds(self, sandbox_id: str) -> Optional[int]:
try:
diff --git a/server/opensandbox_server/services/docker/metadata.py b/server/opensandbox_server/services/docker/metadata.py
index cfb42219c..5607d868b 100644
--- a/server/opensandbox_server/services/docker/metadata.py
+++ b/server/opensandbox_server/services/docker/metadata.py
@@ -14,8 +14,8 @@
"""Per-sandbox file-backed metadata store for Docker.
-Docker cannot update labels on running containers, so user metadata patches
-and expiration overrides are persisted to individual JSON files under
+Docker cannot update labels on running containers, so user metadata patches,
+expiration overrides, and lifecycle transitions are persisted to JSON files under
``~/.opensandbox/metadata/``. Atomic writes (tmp + rename) guard against
truncation during crash, and one lock serializes every read-modify-write so
concurrent patches of one sandbox neither fail nor drop each other's keys.
@@ -66,7 +66,7 @@ def _extract_user_labels(labels: dict) -> dict:
class DockerMetadataStore:
- """Per-sandbox file-backed store for user metadata and expiration overrides."""
+ """Per-sandbox file-backed store for metadata, expiration, and transitions."""
def __init__(self, root: Path | None = None) -> None:
self._root = root or DEFAULT_STORE_DIR
@@ -132,9 +132,29 @@ class DockerMetadataStore:
{"expires_at": expires_at.isoformat()},
)
+ def get_transition_time(self, sandbox_id: str, state: str) -> str | None:
+ payload = self._read_file(self._transition_path(sandbox_id))
+ if not isinstance(payload, dict) or payload.get("state") != state:
+ return None
+ transitioned_at = payload.get("last_transition_at")
+ if isinstance(transitioned_at, str) and transitioned_at.strip():
+ return transitioned_at
+ return None
+
+ def set_transition(self, sandbox_id: str, state: str, transitioned_at: datetime) -> None:
+ with self._lock:
+ self._write_file(
+ self._transition_path(sandbox_id),
+ {"state": state, "last_transition_at": transitioned_at.isoformat()},
+ )
+
def delete(self, sandbox_id: str) -> None:
with self._lock:
- for path in (self._sandbox_path(sandbox_id), self._expiration_path(sandbox_id)):
+ for path in (
+ self._sandbox_path(sandbox_id),
+ self._expiration_path(sandbox_id),
+ self._transition_path(sandbox_id),
+ ):
try:
path.unlink(missing_ok=True)
except OSError as exc:
@@ -146,6 +166,9 @@ class DockerMetadataStore:
def _expiration_path(self, sandbox_id: str) -> Path:
return self._root / "_expiration" / f"{sandbox_id}.json"
+ def _transition_path(self, sandbox_id: str) -> Path:
+ return self._root / "_transition" / f"{sandbox_id}.json"
+
@staticmethod
def _read_file(path: Path) -> dict | None:
try:
diff --git a/server/tests/test_docker_metadata_store.py b/server/tests/test_docker_metadata_store.py
index 9f1ecf739..028beb4a5 100644
--- a/server/tests/test_docker_metadata_store.py
+++ b/server/tests/test_docker_metadata_store.py
@@ -243,6 +243,19 @@ class TestDockerMetadataStore:
# -- crash recovery ------------------------------------------------------
+ def test_transition_is_state_associated_and_separate_from_user_metadata(self, tmp_path: Path):
+ store = self._make_store(tmp_path)
+ transitioned_at = datetime(2030, 1, 1, tzinfo=timezone.utc)
+ store.set_transition("sbx-001", "Paused", transitioned_at)
+ store.patch("sbx-001", {}, {"state": "user-value"})
+
+ reconstructed = self._make_store(tmp_path)
+ assert reconstructed.get_transition_time("sbx-001", "Paused") == transitioned_at.isoformat()
+ assert reconstructed.get_transition_time("sbx-001", "Running") is None
+ assert reconstructed.get("sbx-001", {}) == {"state": "user-value"}
+ reconstructed.delete("sbx-001")
+ assert reconstructed.get_transition_time("sbx-001", "Paused") is None
+
def test_tmp_file_not_read_on_reload(self, tmp_path: Path):
"""Tmp files written during atomic save must not be picked up by get()."""
root = tmp_path / "metadata"
diff --git a/server/tests/test_docker_service.py b/server/tests/test_docker_service.py
index 43f07679a..e83fa403f 100644
--- a/server/tests/test_docker_service.py
+++ b/server/tests/test_docker_service.py
@@ -95,6 +95,14 @@ def _app_config() -> AppConfig:
ingress=IngressConfig(mode="direct"),
)
+
[email protected](autouse=True)
+def isolated_metadata_store(monkeypatch, tmp_path):
+ monkeypatch.setattr(
+ "opensandbox_server.services.docker.metadata.DEFAULT_STORE_DIR", tmp_path / "metadata"
+ )
+
+
def test_parse_memory_limit_handles_units():
assert parse_memory_limit("512Mi") == 512 * 1024 * 1024
assert parse_memory_limit("1G") == 1_000_000_000
@@ -2223,11 +2231,13 @@ def test_pause_resume_sandbox_concurrent_outcomes(
with (
patch.object(service, "_get_container_by_sandbox_id", side_effect=lookup_results),
patch.object(service, "_get_egress_sidecars", return_value=[sidecar]),
+ patch.object(service._metadata_store, "set_transition") as set_transition,
pytest.raises(HTTPException) as exc_info,
):
getattr(service, f"{operation}_sandbox")("sandbox-id")
assert exc_info.value.status_code == expected_status
+ set_transition.assert_not_called()
if expected_status == 404:
assert exc_info.value.detail == missing.detail
elif expected_status == 409:
@@ -5356,6 +5366,435 @@ def _mock_container(
return container
[email protected]("timeout", [None, 120])
+async def test_create_get_timestamp_consistency_preserves_ttl(timeout):
+ requested_at = datetime(2030, 1, 1, tzinfo=timezone.utc)
+ created_at = requested_at + timedelta(seconds=2, microseconds=123456)
+ container = _mock_container("timestamp-test")
+ container.attrs["Created"] = created_at.isoformat()
+ docker_client = MagicMock()
+ docker_client.containers.list.return_value = []
+ docker_client.containers.get.return_value = container
+
+ def create_container(sandbox_id, image, entrypoint, labels, *args, **kwargs):
+ container.attrs["Config"]["Labels"] = dict(labels)
+ return container
+
+ with (
+ patch("docker.from_env", return_value=docker_client),
+ patch("opensandbox_server.services.docker.docker_service.datetime", wraps=datetime) as clock,
+ ):
+ clock.now.return_value = requested_at
+ service = DockerSandboxService(config=_app_config())
+ request = CreateSandboxRequest(
+ image=ImageSpec(uri="python:3.11"), timeout=timeout,
+ resourceLimits=ResourceLimits(root={}), entrypoint=["python"],
+ )
+ with (
+ patch.object(service, "generate_sandbox_id", return_value="timestamp-test"),
+ patch.object(service, "_ensure_image_available"),
+ patch.object(service, "_create_and_start_container", side_effect=create_container),
+ patch.object(service, "_schedule_expiration") as schedule,
+ ):
+ created = await service.create_sandbox(request)
+
+ expected_expiration = requested_at + timedelta(seconds=timeout) if timeout else None
+ assert created.expires_at == expected_expiration
+ if timeout:
+ schedule.assert_called_once_with(created.id, expected_expiration)
+ else:
+ schedule.assert_not_called()
+ clock.now.return_value = requested_at + timedelta(seconds=10)
+ for _ in range(2):
+ fetched = service.get_sandbox(created.id)
+ assert created.created_at == fetched.created_at == requested_at
+ assert created.status.last_transition_at == fetched.status.last_transition_at == requested_at
+ assert fetched.expires_at == expected_expiration
+ reconstructed = DockerSandboxService(config=_app_config()).get_sandbox(created.id)
+ assert reconstructed.created_at == requested_at
+ assert reconstructed.expires_at == expected_expiration
+
+
[email protected]("exit_code", [0, 137])
+def test_pause_resume_transition_timestamps_survive_reconstruction(exit_code):
+ created_at = datetime(2030, 1, 1, tzinfo=timezone.utc)
+ expires_at = created_at + timedelta(hours=1)
+ container = _mock_container(
+ "timestamp-test", extra_labels={SANDBOX_EXPIRES_AT_LABEL: expires_at.isoformat()}
+ )
+ container.attrs["Created"] = created_at.isoformat()
+ state = container.attrs["State"]
+ container.pause.side_effect = lambda: state.update(Paused=True)
+ container.unpause.side_effect = lambda: state.update(Paused=False)
+ docker_client = MagicMock()
+ docker_client.containers.list.return_value = []
+ docker_client.containers.get.return_value = container
+
+ with (
+ patch("docker.from_env", return_value=docker_client),
+ patch("opensandbox_server.services.docker.docker_service.datetime", wraps=datetime) as clock,
+ ):
+ service = DockerSandboxService(config=_app_config())
+ previous_transition = created_at
+ for step, expected_state in enumerate(["Paused", "Running", "Paused", "Running"], 1):
+ transitioned_at = created_at + timedelta(seconds=step * 10)
+ clock.now.return_value = transitioned_at
+ if expected_state == "Paused":
+ service.pause_sandbox("timestamp-test")
+ else:
+ service.resume_sandbox("timestamp-test")
+ clock.now.return_value += timedelta(seconds=1)
+ # Each new service must read the timestamp from disk, not process memory.
+ service = DockerSandboxService(config=_app_config())
+ for _ in range(2):
+ fetched = service.get_sandbox("timestamp-test")
+ assert fetched.status.state == expected_state
+ assert fetched.status.last_transition_at == transitioned_at > previous_transition
+ assert fetched.created_at == created_at
+ assert fetched.expires_at == expires_at
+ previous_transition = transitioned_at
+
+ finished_at = created_at + timedelta(minutes=1, microseconds=500000)
+ state.update(Running=False, Status="exited", ExitCode=exit_code, FinishedAt=finished_at.isoformat())
+ service = DockerSandboxService(config=_app_config())
+ terminated = service.get_sandbox("timestamp-test")
+ assert terminated.status.state == ("Terminated" if exit_code == 0 else "Failed")
+ assert terminated.status.last_transition_at == finished_at
+ assert terminated.created_at == created_at
+ assert terminated.expires_at == expires_at
+ service.delete_sandbox("timestamp-test")
+ assert not list(service._metadata_store._root.rglob("timestamp-test.json"))
+
+
[email protected]("sidecar_missing", [False, True])
+def test_pause_failure_timestamp_matches_actual_state(sidecar_missing):
+ paused_at = datetime(2030, 1, 1, tzinfo=timezone.utc)
+ rolled_back_at = paused_at + timedelta(seconds=1)
+ container = _mock_container(
+ "timestamp-test", extra_labels={SANDBOX_EGRESS_AUTH_TOKEN_METADATA_KEY: "token"}
+ )
+ state = container.attrs["State"]
+ container.pause.side_effect = lambda: state.update(Paused=True)
+ container.unpause.side_effect = lambda: state.update(Paused=False)
+ sidecar = _lifecycle_container("sidecar", running=True, paused=False)
+ sidecar.pause.side_effect = DockerException("sidecar pause failed")
+ service = DockerSandboxService(config=_app_config())
+
+ with (
+ patch.object(service, "_get_container_by_sandbox_id", return_value=container),
+ patch.object(service, "_get_egress_sidecars", return_value=[] if sidecar_missing else [sidecar]),
+ patch("opensandbox_server.services.docker.docker_service.datetime", wraps=datetime) as clock,
+ ):
+ clock.now.side_effect = [paused_at, rolled_back_at]
+ with pytest.raises(HTTPException) as exc_info:
+ service.pause_sandbox("timestamp-test")
+ assert exc_info.value.status_code == 500
+ fetched = service.get_sandbox("timestamp-test")
+ assert fetched.status.state == ("Paused" if sidecar_missing else "Running")
+ assert fetched.status.last_transition_at == (paused_at if sidecar_missing else rolled_back_at)
+
+
[email protected]("operation", ["pause", "resume"])
+def test_transition_write_failure_preserves_successful_lifecycle(operation):
+ service = DockerSandboxService(config=_app_config())
+ resuming = operation == "resume"
+ main = _lifecycle_container("main", running=True, paused=resuming, egress_expected=True)
+ sidecar = _lifecycle_container("sidecar", running=True, paused=resuming)
+ for container in (main, sidecar):
+ container.pause.side_effect = lambda c=container: c.attrs["State"].update(Paused=True)
+ container.unpause.side_effect = lambda c=container: c.attrs["State"].update(Paused=False)
+
+ with (
+ patch.object(service, "_get_container_by_sandbox_id", return_value=main),
+ patch.object(service, "_get_egress_sidecars", return_value=[sidecar]),
+ patch.object(service._metadata_store, "_write_file", side_effect=OSError("disk full")),
+ patch.object(service, "_raise_if_lifecycle_race") as map_race,
+ patch("opensandbox_server.services.docker.docker_service.logger.warning") as warning,
+ ):
+ assert getattr(service, f"{operation}_sandbox")("sandbox-id") is None
+
+ for container in (main, sidecar):
+ assert container.attrs["State"] == {"Running": True, "Paused": not resuming}
+ getattr(container, "unpause" if resuming else "pause").assert_called_once_with()
+ getattr(container, "pause" if resuming else "unpause").assert_not_called()
+ map_race.assert_not_called()
+ transition_state = "Running" if resuming else "Paused"
+ warning.assert_called_once_with(
+ f"Failed to persist {transition_state} transition for sandbox sandbox-id: disk full"
+ )
+
+
[email protected]("failed_state", ["Paused", "Running"])
+def test_transition_write_failure_preserves_pause_rollback_error(failed_state):
+ service = DockerSandboxService(config=_app_config())
+ main = _lifecycle_container("main", running=True, paused=False, egress_expected=True)
+ sidecar = _lifecycle_container("sidecar", running=True, paused=False)
+ failing_sidecar = _lifecycle_container("failing", running=True, paused=False)
+ for container in (main, sidecar):
+ container.pause.side_effect = lambda c=container: c.attrs["State"].update(Paused=True)
+ container.unpause.side_effect = lambda c=container: c.attrs["State"].update(Paused=False)
+ failing_sidecar.pause.side_effect = DockerException("sidecar pause failed")
+ write_file = service._metadata_store._write_file
+
+ def fail_transition_write(path, payload):
+ if payload["state"] == failed_state:
+ raise OSError("disk full")
+ write_file(path, payload)
+
+ with (
+ patch.object(service, "_get_container_by_sandbox_id", return_value=main),
+ patch.object(service, "_get_egress_sidecars", return_value=[sidecar, failing_sidecar]),
+ patch.object(service._metadata_store, "_write_file", side_effect=fail_transition_write),
+ patch.object(service, "_raise_if_lifecycle_race") as map_race,
+ patch("opensandbox_server.services.docker.docker_service.logger.warning") as warning,
+ pytest.raises(HTTPException) as exc_info,
+ ):
+ service.pause_sandbox("sandbox-id")
+
+ assert exc_info.value.status_code == 500
+ assert exc_info.value.detail == {
+ "code": SandboxErrorCodes.SANDBOX_PAUSE_FAILED,
+ "message": "Failed to pause egress sidecar: sidecar pause failed",
+ }
+ assert isinstance(exc_info.value.__cause__, DockerException)
+ for container in (main, sidecar):
+ assert container.attrs["State"] == {"Running": True, "Paused": False}
+ container.pause.assert_called_once_with()
+ container.unpause.assert_called_once_with()
+ failing_sidecar.pause.assert_called_once_with()
+ map_race.assert_not_called()
+ warning.assert_any_call(
+ f"Failed to persist {failed_state} transition for sandbox sandbox-id: disk full"
+ )
+
+
+def test_failed_pause_rollback_does_not_record_running_transition():
+ service = DockerSandboxService(config=_app_config())
+ main = _lifecycle_container("main", running=True, paused=False, egress_expected=True)
+ main.pause.side_effect = lambda: main.attrs["State"].update(Paused=True)
+ main.unpause.side_effect = DockerException("main rollback failed")
+ sidecar = _lifecycle_container("sidecar", running=True, paused=False)
+ sidecar.pause.side_effect = DockerException("sidecar pause failed")
+
+ with (
+ patch.object(service, "_get_container_by_sandbox_id", return_value=main),
+ patch.object(service, "_get_egress_sidecars", return_value=[sidecar]),
+ pytest.raises(HTTPException) as exc_info,
+ ):
+ service.pause_sandbox("sandbox-id")
+
+ assert exc_info.value.status_code == 500
+ assert exc_info.value.detail == {
+ "code": SandboxErrorCodes.SANDBOX_PAUSE_FAILED,
+ "message": "Failed to pause egress sidecar: sidecar pause failed; rollback failed: main rollback failed",
+ }
+ assert main.attrs["State"]["Paused"] is True
+ assert service._metadata_store.get_transition_time("sandbox-id", "Paused") is not None
+ assert service._metadata_store.get_transition_time("sandbox-id", "Running") is None
+
+
+def _run_lifecycle_interleaving(first_operation, second_operation, first_blocked, release_first):
+ second_checkpoint = threading.Event()
+ errors: list[Exception] = []
+
+ class ObservedLock:
+ """Observe a waiting contender without sleeps or scheduler timing assumptions."""
+
+ def __init__(self):
+ self._lock = threading.Lock()
+
+ def __enter__(self):
+ if threading.current_thread() is second_thread and self._lock.locked():
+ second_checkpoint.set()
+ return self._lock.__enter__()
+
+ def __exit__(self, *args):
+ return self._lock.__exit__(*args)
+
+ def run(operation):
+ try:
+ operation()
+ except Exception as exc:
+ errors.append(exc)
+
+ def run_second():
+ run(second_operation)
+ # Without a lifecycle lock, let the contender finish before releasing
+ # the first operation to reproduce the stale write deterministically.
+ second_checkpoint.set()
+
+ first_thread = threading.Thread(target=run, args=(first_operation,), daemon=True)
+ second_thread = threading.Thread(target=run_second, daemon=True)
+ with patch("opensandbox_server.services.docker.docker_service.Lock", ObservedLock):
+ first_thread.start()
+ try:
+ assert first_blocked.wait(5), "First operation did not reach the test gate"
+ second_thread.start()
+ assert second_checkpoint.wait(5), "Contender neither waited for the lock nor finished"
+ finally:
+ release_first.set()
+ first_thread.join(5)
+ if second_thread.ident is not None:
+ second_thread.join(5)
+ assert not first_thread.is_alive() and not second_thread.is_alive()
+ assert not errors, errors
+
+
+ "operation, concurrent_operation",
+ [
+ ("pause", "resume"),
+ ("pause", "delete"),
+ ("resume", "delete"),
+ ("pause", "expire"),
+ ("resume", "expire"),
+ ("pause", "expire_missing"),
+ ("resume", "expire_missing"),
+ ],
+)
+def test_delayed_transition_write_is_ordered_with_lifecycle(operation, concurrent_operation):
+ service = DockerSandboxService(config=_app_config())
+ main = _mock_container("sandbox-id", paused=operation == "resume")
+ state = main.attrs["State"]
+ main.pause.side_effect = lambda: state.update(Paused=True)
+ main.unpause.side_effect = lambda: state.update(Paused=False)
+ first_blocked, release_first = threading.Event(), threading.Event()
+ events: list[str] = []
+ transition_times: dict[str, datetime] = {}
+ first_state = "Paused" if operation == "pause" else "Running"
+ set_transition = service._metadata_store.set_transition
+ delete_metadata = service._metadata_store.delete
+
+ def delayed_transition(sandbox_id, transition_state, transitioned_at):
+ transition_times[transition_state] = transitioned_at
+ if transition_state == first_state:
+ first_blocked.set()
+ assert release_first.wait(5)
+ set_transition(sandbox_id, transition_state, transitioned_at)
+ events.append(f"persist {transition_state}")
+
+ def lookup(sandbox_id):
+ if concurrent_operation == "expire_missing" and first_blocked.is_set():
+ raise HTTPException(status_code=404, detail={})
+ return main
+
+ def cleanup(sandbox_id):
+ delete_metadata(sandbox_id)
+ events.append("cleanup")
+
+ second_operation = (
+ service._expire_sandbox if concurrent_operation.startswith("expire")
+ else getattr(service, f"{concurrent_operation}_sandbox")
+ )
+ with (
+ patch.object(service, "_get_container_by_sandbox_id", side_effect=lookup),
+ patch.object(service, "_get_egress_sidecars", return_value=[]),
+ patch.object(service, "_cleanup_egress_sidecar"),
+ patch.object(service._metadata_store, "set_transition", side_effect=delayed_transition),
+ patch.object(service._metadata_store, "delete", side_effect=cleanup),
+ ):
+ _run_lifecycle_interleaving(
+ lambda: getattr(service, f"{operation}_sandbox")("sandbox-id"),
+ lambda: second_operation("sandbox-id"),
+ first_blocked, release_first,
+ )
+ if concurrent_operation == "resume":
+ assert events == ["persist Paused", "persist Running"]
+ fetched = service.get_sandbox("sandbox-id")
+ assert fetched.status.state == "Running"
+ assert fetched.status.last_transition_at == transition_times["Running"]
+ else:
+ assert events == [f"persist {first_state}", "cleanup"]
+ assert not list(service._metadata_store._root.rglob("sandbox-id.json"))
+ if concurrent_operation == "expire_missing":
+ main.remove.assert_not_called()
+ else:
+ main.remove.assert_called_once_with(force=True)
+
+
[email protected]("cleanup_operation", ["delete", "expire"])
[email protected]("operation", ["pause", "resume"])
+def test_transition_waits_for_cleanup_before_container_lookup(cleanup_operation, operation):
+ service = DockerSandboxService(config=_app_config())
+ main = _mock_container("sandbox-id", paused=operation == "resume")
+ first_blocked, release_first, removed = (
+ threading.Event(), threading.Event(), threading.Event()
+ )
+ outcomes: list[int | None] = []
+
+ def delayed_kill():
+ first_blocked.set()
+ assert release_first.wait(5)
+ main.attrs["State"]["Running"] = False
+
+ main.kill.side_effect = delayed_kill
+ main.remove.side_effect = lambda **kwargs: removed.set()
+
+ def lookup(sandbox_id):
+ if removed.is_set():
+ raise HTTPException(status_code=404, detail={})
+ return main
+
+ def transition():
+ try:
+ getattr(service, f"{operation}_sandbox")("sandbox-id")
+ except HTTPException as exc:
+ outcomes.append(exc.status_code)
+ else:
+ outcomes.append(None)
+
+ cleanup = service.delete_sandbox if cleanup_operation == "delete" else service._expire_sandbox
+ with (
+ patch.object(service, "_get_container_by_sandbox_id", side_effect=lookup),
+ patch.object(service, "_get_egress_sidecars", return_value=[]),
+ patch.object(service, "_cleanup_egress_sidecar"),
+ patch.object(service._metadata_store, "set_transition") as set_transition,
+ ):
+ _run_lifecycle_interleaving(
+ lambda: cleanup("sandbox-id"), transition, first_blocked, release_first,
+ )
+
+ assert outcomes == [404]
+ main.pause.assert_not_called()
+ main.unpause.assert_not_called()
+ set_transition.assert_not_called()
+ assert not list(service._metadata_store._root.rglob("sandbox-id.json"))
+
+
+def test_delayed_transition_does_not_block_another_sandbox():
+ service = DockerSandboxService(config=_app_config())
+ containers = {sandbox_id: _mock_container(sandbox_id) for sandbox_id in ("first", "second")}
+ for container in containers.values():
+ container.pause.side_effect = lambda c=container: c.attrs["State"].update(Paused=True)
+ first_blocked, release_first = threading.Event(), threading.Event()
+ events: list[str] = []
+ set_transition = service._metadata_store.set_transition
+
+ def delayed_transition(sandbox_id, state, transitioned_at):
+ if sandbox_id == "first":
+ first_blocked.set()
+ assert release_first.wait(5)
+ set_transition(sandbox_id, state, transitioned_at)
+ events.append(sandbox_id)
+
+ with (
+ patch.object(service, "_get_container_by_sandbox_id", side_effect=containers.__getitem__),
+ patch.object(service, "_get_egress_sidecars", return_value=[]),
+ patch.object(service._metadata_store, "set_transition", side_effect=delayed_transition),
+ ):
+ _run_lifecycle_interleaving(
+ lambda: service.pause_sandbox("first"),
+ lambda: service.pause_sandbox("second"),
+ first_blocked, release_first,
+ )
+
+ assert events == ["second", "first"]
+ for sandbox_id, container in containers.items():
+ assert container.attrs["State"]["Paused"] is True
+ assert service._metadata_store.get_transition_time(sandbox_id, "Paused") is not None
+
+
def _wire_list_mocks(mock_client: MagicMock, containers: list[MagicMock]) -> None:
"""Wire ``api.containers`` (summary) + ``containers.get`` (inspect) so
that list_sandboxes' two-step discovery returns the given containers."""