Secure, Fast, and Extensible Sandbox runtime for AI agents.
# Copyright 2025 The OpenSandbox Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Docker-based implementation of SandboxService.
This module provides a Docker implementation of the sandbox service interface,
using Docker containers for sandbox lifecycle management.
"""
from __future__ import annotations
import asyncio
import inspect
import json
import logging
import math
import os
import time
from _thread import LockType
from contextlib import contextmanager
from datetime import datetime, timedelta, timezone
from threading import Lock, Thread, Timer
from typing import Any, Dict, Optional
from weakref import WeakValueDictionary
import docker
from docker.errors import APIError, DockerException, NotFound as DockerNotFound
from fastapi import HTTPException, status
from opensandbox_server.extensions import (
ACCESS_RENEW_EXTEND_SECONDS_METADATA_KEY,
BOOTSTRAP_EXECD_ISOLATION_KEY,
ISOLATION_UPPER_MOUNT_PATH,
extract_extensions_from_mapping,
)
from opensandbox_server.api.schema import (
CreateSandboxRequest,
CreateSandboxResponse,
ImageSpec,
ListSandboxesRequest,
ListSandboxesResponse,
PaginationInfo,
PatchSandboxMetadataRequest,
RenewSandboxExpirationRequest,
RenewSandboxExpirationResponse,
Sandbox,
SandboxStatus,
PlatformSpec,
)
from opensandbox_server.config import AppConfig, get_config
from opensandbox_server.services.docker.docker_diagnostics import (
DockerDiagnosticsMixin,
_container_lifecycle_state,
)
from opensandbox_server.services.docker.runtime import (
DockerRuntimeMixin,
)
from opensandbox_server.services.docker.volumes import DockerVolumesMixin
from opensandbox_server.services.docker.networking import (
EGRESS_SIDECAR_LABEL,
HOST_NETWORK_MODE,
DockerNetworkingMixin,
)
from opensandbox_server.services.docker.container_ops import DockerContainerOpsMixin
from opensandbox_server.services.docker.metadata import DockerMetadataStore
from opensandbox_server.services.docker.port_allocator import (
MAX_PORT_PUBLISH_ATTEMPTS,
allocate_port_bindings,
is_port_publish_error,
normalize_port_bindings,
release_port_bindings,
)
from opensandbox_server.services.windows_common import (
inject_windows_resource_limits_env,
inject_windows_user_ports,
is_windows_platform,
validate_windows_resource_limits,
)
from opensandbox_server.services.docker.windows_profile import (
apply_windows_runtime_host_config_defaults,
validate_windows_runtime_prerequisites,
)
from opensandbox_server.services.extension_service import ExtensionService
from opensandbox_server.services.constants import (
OPENSANDBOX_EGRESS_MITMPROXY_SSL_INSECURE,
OPENSANDBOX_EGRESS_MITMPROXY_TRANSPARENT,
OPENSANDBOX_RUNTIME_MOUNT_PATH,
SANDBOX_CREATED_AT_LABEL,
SANDBOX_EGRESS_AUTH_TOKEN_METADATA_KEY,
SANDBOX_EMBEDDING_PROXY_PORT_LABEL,
SANDBOX_EXPIRES_AT_LABEL,
SANDBOX_HTTP_PORT_LABEL,
SANDBOX_ID_LABEL,
SANDBOX_MANAGED_VOLUMES_LABEL,
SANDBOX_MANUAL_CLEANUP_LABEL,
SANDBOX_OSSFS_MOUNTS_LABEL,
SANDBOX_SNAPSHOT_ID_LABEL,
SandboxErrorCodes,
)
from opensandbox_server.services.endpoint_auth import (
generate_egress_token,
)
from opensandbox_server.services.helpers import (
matches_filter,
parse_timestamp,
split_egress_env,
validate_upstream_proxy_request,
)
from opensandbox_server.services.docker.ossfs_mixin import OSSFSMixin
from opensandbox_server.services.sandbox_service import SandboxService
from opensandbox_server.services.runtime_resolver import SecureRuntimeResolver
from opensandbox_server.services.snapshot_restore import resolve_sandbox_image_from_request
from opensandbox_server.services.validators import (
calculate_expiration_or_raise,
ensure_entrypoint,
ensure_future_expiration,
ensure_metadata_labels,
ensure_platform_valid,
ensure_timeout_within_limit,
)
logger = logging.getLogger(__name__)
class DockerSandboxService(DockerDiagnosticsMixin, DockerRuntimeMixin, DockerVolumesMixin, DockerNetworkingMixin, DockerContainerOpsMixin, OSSFSMixin, SandboxService, ExtensionService):
def __init__(self, config: Optional[AppConfig] = None):
"""
Initializes Docker service from environment variables.
Note: Connection is not verified at initialization time.
Connection errors will be raised when Docker operations are performed.
"""
self.app_config = config or get_config()
runtime_config = self.app_config.runtime
if runtime_config.type != "docker":
raise ValueError("DockerSandboxService requires runtime.type = 'docker'.")
self.execd_image = runtime_config.execd_image
self.network_mode = (self.app_config.docker.network_mode or HOST_NETWORK_MODE).lower()
self._execd_archive_cache: Dict[str, bytes] = {}
self._bootstrap_script_cache: Dict[str, bytes] = {}
self._bwrap_archive_cache: Dict[str, bytes] = {}
self._session_gate_archive_cache: Dict[str, bytes] = {}
self._launcher_archive_cache: Dict[str, bytes] = {}
self._windows_profile_cache: Dict[str, bytes] = {}
self._daemon_platform: Optional[PlatformSpec] = None
self._metadata_store = DockerMetadataStore()
self._api_timeout = self._resolve_api_timeout()
try:
client_kwargs = {}
try:
signature = inspect.signature(docker.from_env)
if "timeout" in signature.parameters:
client_kwargs["timeout"] = self._api_timeout
except (ValueError, TypeError):
logger.debug(
"Unable to introspect docker.from_env signature; using default parameters."
)
self.docker_client = docker.from_env(**client_kwargs)
if not client_kwargs:
try:
self.docker_client.api.timeout = self._api_timeout
except AttributeError:
logger.debug("Docker client API does not expose timeout attribute.")
logger.info("Docker service initialized from environment")
except Exception as e: # noqa: BLE001
# Common failure mode on macOS/dev machines: Docker daemon not running or socket path wrong.
hint = ""
msg = str(e)
if isinstance(e, FileNotFoundError) or "No such file or directory" in msg:
docker_host = os.environ.get("DOCKER_HOST", "")
hint = (
" Docker daemon seems unavailable (unix socket not found). "
"Make sure Docker Desktop (or Colima/Rancher Desktop) is running. "
"If you use Colima on macOS, you may need to set "
"DOCKER_HOST=unix://${HOME}/.colima/default/docker.sock before starting the server. "
f"(current DOCKER_HOST='{docker_host}')"
)
raise HTTPException(
status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
detail={
"code": SandboxErrorCodes.DOCKER_INITIALIZATION_ERROR,
"message": f"Failed to initialize Docker service: {str(e)}.{hint}",
},
)
self._expiration_lock = Lock()
self._lifecycle_locks_guard = Lock()
self._lifecycle_locks: WeakValueDictionary[str, LockType] = WeakValueDictionary()
self._execd_archive_lock = Lock()
self._sandbox_expirations: Dict[str, datetime] = {}
self._expiration_timers: Dict[str, Timer] = {}
self._ossfs_mount_lock = Lock()
self._ossfs_mount_ref_counts: Dict[str, int] = {}
self._restore_existing_sandboxes()
# Initialize secure runtime resolver
self.resolver = SecureRuntimeResolver(self.app_config)
self.docker_runtime = self.resolver.get_docker_runtime()
def _resolve_api_timeout(self) -> int:
"""Docker API timeout in seconds: [docker].api_timeout if set, else default 180."""
cfg = self.app_config.docker.api_timeout
if cfg is not None and cfg >= 1:
return cfg
return 180
@contextmanager
def _sandbox_lifecycle_lock(self, sandbox_id: str):
# Order runtime mutations and transition records within this service
# instance only, not across service instances or processes. Keep this
# separate from the metadata store's non-reentrant file-write lock.
with self._lifecycle_locks_guard:
lock = self._lifecycle_locks.get(sandbox_id)
if lock is None:
lock = Lock()
self._lifecycle_locks[sandbox_id] = lock
# Active and waiting calls retain the lock; idle entries disappear.
with lock:
yield
def _persist_transition(self, sandbox_id: str, state: str) -> None:
try:
self._metadata_store.set_transition(sandbox_id, state, datetime.now(timezone.utc))
except OSError as exc:
logger.warning(f"Failed to persist {state} transition for sandbox {sandbox_id}: {exc}")
@contextmanager
def _docker_operation(self, action: str, sandbox_id: Optional[str] = None):
op_id = sandbox_id or "shared"
start = time.perf_counter()
try:
yield
except Exception as exc:
elapsed_ms = (time.perf_counter() - start) * 1000
logger.warning(
f"sandbox={op_id} | action={action} | duration={elapsed_ms:.2f} | error={exc}"
)
raise
else:
elapsed_ms = (time.perf_counter() - start) * 1000
logger.info(
f"sandbox={op_id} | action={action} | duration={elapsed_ms:.2f}"
)
def _get_container_by_sandbox_id(self, sandbox_id: str):
label_selector = f"{SANDBOX_ID_LABEL}={sandbox_id}"
try:
try:
container = self.docker_client.containers.get(f"sandbox-{sandbox_id}")
except DockerNotFound:
container = None
if container is not None:
labels = container.attrs.get("Config", {}).get("Labels") or {}
if labels.get(SANDBOX_ID_LABEL) == sandbox_id:
return container
# Preserve lookup for managed containers with a nonstandard name.
containers = self.docker_client.containers.list(
all=True, filters={"label": label_selector}
)
except DockerNotFound as exc:
# Container disappeared between list-summary and inspect; treat as
# not found rather than surfacing a 500. See list_sandboxes for
# the full-table variant of this fix.
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail={
"code": SandboxErrorCodes.SANDBOX_NOT_FOUND,
"message": f"Sandbox {sandbox_id} not found.",
},
) from exc
except DockerException as exc:
raise HTTPException(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
detail={
"code": SandboxErrorCodes.CONTAINER_QUERY_FAILED,
"message": f"Failed to query sandbox containers: {str(exc)}",
},
) from exc
if not containers:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail={
"code": SandboxErrorCodes.SANDBOX_NOT_FOUND,
"message": f"Sandbox {sandbox_id} not found.",
},
)
return containers[0]
def _get_egress_sidecars(self, sandbox_id: str) -> list[Any]:
return self.docker_client.containers.list(
all=True, filters={"label": f"{EGRESS_SIDECAR_LABEL}={sandbox_id}"}
)
def _schedule_expiration(
self,
sandbox_id: str,
expires_at: datetime,
*,
update_expiration: bool = True,
**expire_kwargs,
) -> None:
# Delay might already be negative if the timer should fire immediately
delay = max(0.0, (expires_at - datetime.now(timezone.utc)).total_seconds())
timer = Timer(
delay,
self._expire_sandbox,
args=(sandbox_id,),
kwargs=expire_kwargs or None,
)
timer.daemon = True
with self._expiration_lock:
# Replace existing timer (if any) so renew operations take effect immediately
existing = self._expiration_timers.pop(sandbox_id, None)
if existing:
existing.cancel()
if update_expiration:
self._sandbox_expirations[sandbox_id] = expires_at
self._expiration_timers[sandbox_id] = timer
timer.start()
def _remove_expiration_tracking(self, sandbox_id: str) -> None:
with self._expiration_lock:
timer = self._expiration_timers.pop(sandbox_id, None)
if timer:
timer.cancel()
self._sandbox_expirations.pop(sandbox_id, None)
@staticmethod
def _has_manual_cleanup(labels: Dict[str, str]) -> bool:
return labels.get(SANDBOX_MANUAL_CLEANUP_LABEL, "").lower() == "true"
def _get_tracked_expiration(
self,
sandbox_id: str,
labels: Dict[str, str],
) -> Optional[datetime]:
with self._expiration_lock:
tracked = self._sandbox_expirations.get(sandbox_id)
if tracked:
return tracked
persisted = self._metadata_store.get_expiration(sandbox_id)
if persisted:
return parse_timestamp(persisted)
label_value = labels.get(SANDBOX_EXPIRES_AT_LABEL)
if label_value:
return parse_timestamp(label_value)
return None
def _expire_sandbox(
self,
sandbox_id: str,
fallback_mount_keys: Optional[list[str]] = None,
) -> None:
"""Timer callback to terminate expired sandboxes."""
with self._sandbox_lifecycle_lock(sandbox_id):
self._expire_sandbox_locked(sandbox_id, fallback_mount_keys)
def _expire_sandbox_locked(
self,
sandbox_id: str,
fallback_mount_keys: Optional[list[str]] = None,
) -> None:
mount_keys: list[str] = []
try:
container = self._get_container_by_sandbox_id(sandbox_id)
except HTTPException as exc:
if exc.status_code == status.HTTP_404_NOT_FOUND:
self._cleanup_egress_sidecar(sandbox_id)
self._remove_expiration_tracking(sandbox_id)
self._cleanup_windows_oem_volume(sandbox_id, None)
if fallback_mount_keys:
self._release_ossfs_mounts(fallback_mount_keys)
self._metadata_store.delete(sandbox_id)
else:
with self._expiration_lock:
current_expires = self._sandbox_expirations.get(sandbox_id)
now = datetime.now(timezone.utc)
if current_expires and current_expires > now:
logger.info(
f"Sandbox {sandbox_id} expiration was renewed; skipping retry."
)
else:
logger.warning(
f"Failed to fetch sandbox {sandbox_id} for expiration: {exc.detail} — "
"scheduling retry in 30s"
)
retry_at = now + timedelta(seconds=30)
self._schedule_expiration(
sandbox_id,
retry_at,
update_expiration=False,
fallback_mount_keys=fallback_mount_keys,
)
return
labels = container.attrs.get("Config", {}).get("Labels") or {}
current_expires = self._get_tracked_expiration(sandbox_id, labels)
if current_expires and current_expires > datetime.now(timezone.utc):
self._schedule_expiration(sandbox_id, current_expires, update_expiration=False)
logger.info(
f"Sandbox {sandbox_id} was renewed "
f"(expires {current_expires}); aborting expiration."
)
return
mount_keys_raw = labels.get(SANDBOX_OSSFS_MOUNTS_LABEL, "[]")
try:
parsed_mount_keys = json.loads(mount_keys_raw)
if isinstance(parsed_mount_keys, list):
mount_keys = [key for key in parsed_mount_keys if isinstance(key, str) and key]
except (TypeError, json.JSONDecodeError):
mount_keys = []
try:
state = container.attrs.get("State", {})
if state.get("Running", False):
container.kill()
except DockerException as exc:
logger.warning(f"Failed to stop expired sandbox {sandbox_id}: {exc}")
try:
container.remove(force=True)
except DockerException as exc:
logger.warning(f"Failed to remove expired sandbox {sandbox_id}: {exc}")
# Re-read ownership on retry: a concurrent DELETE may already
# have released the mount references captured by this callback.
self._schedule_expiration(
sandbox_id, datetime.now(timezone.utc) + timedelta(seconds=30),
update_expiration=False,
)
return
managed_volumes_raw = labels.get(SANDBOX_MANAGED_VOLUMES_LABEL, "[]")
try:
managed_volumes: list[str] = json.loads(managed_volumes_raw)
except (TypeError, json.JSONDecodeError):
managed_volumes = []
self._remove_expiration_tracking(sandbox_id)
# Ensure sidecar is also cleaned up on expiration
self._cleanup_egress_sidecar(sandbox_id)
self._cleanup_windows_oem_volume(sandbox_id, labels)
self._release_ossfs_mounts(mount_keys)
self._cleanup_managed_volumes(sandbox_id, managed_volumes)
self._metadata_store.delete(sandbox_id)
def _restore_existing_sandboxes(self) -> None:
"""On startup, rebuild expiration timers for containers already running."""
try:
containers = self.docker_client.containers.list(all=True)
except DockerException as exc:
logger.warning(f"Failed to restore existing sandboxes: {exc}")
return
restored = 0
seen_sidecars: set[str] = set()
restored_mount_refs: dict[str, int] = {}
expired_entries: list[tuple[str, list[str]]] = []
now = datetime.now(timezone.utc)
def _parse_and_accumulate_mount_refs(labels: dict) -> list[str]:
mount_keys_raw = labels.get(SANDBOX_OSSFS_MOUNTS_LABEL, "[]")
try:
parsed = json.loads(mount_keys_raw)
except (TypeError, json.JSONDecodeError):
parsed = []
keys: list[str] = []
if isinstance(parsed, list):
for key in parsed:
if isinstance(key, str) and key:
keys.append(key)
restored_mount_refs[key] = restored_mount_refs.get(key, 0) + 1
return keys
# Pass 1: collect ref counts for ALL sandbox containers (alive + expired)
# and schedule timers for alive ones. Expired sandboxes are deferred to
# pass 2 so that ref counts are fully populated before any release.
for container in containers:
labels = container.attrs.get("Config", {}).get("Labels") or {}
sidecar_for = labels.get(EGRESS_SIDECAR_LABEL)
if sidecar_for:
seen_sidecars.add(sidecar_for)
continue
sandbox_id = labels.get(SANDBOX_ID_LABEL)
if not sandbox_id:
continue
mount_keys = _parse_and_accumulate_mount_refs(labels)
expires_at = self._get_tracked_expiration(sandbox_id, labels)
if expires_at is None:
if self._has_manual_cleanup(labels):
restored += 1
continue
logger.warning(
f"Sandbox {sandbox_id} missing expires-at label; "
"skipping expiration scheduling."
)
continue
if expires_at <= now:
logger.info(f"Sandbox {sandbox_id} already expired; terminating now.")
expired_entries.append((sandbox_id, mount_keys))
continue
self._schedule_expiration(sandbox_id, expires_at)
restored += 1
# Populate ref counts before expiring anything so _release_ossfs_mount
# can properly decrement and unmount.
with self._ossfs_mount_lock:
self._ossfs_mount_ref_counts = restored_mount_refs
# Pass 2: expire deferred sandboxes (ref counts are now available).
# Cached mount keys are passed as fallback so that mounts are still
# released even if the container vanishes between pass 1 and pass 2.
for sandbox_id, cached_mount_keys in expired_entries:
self._expire_sandbox(sandbox_id, fallback_mount_keys=cached_mount_keys)
# Cleanup orphan sidecars (no matching sandbox container)
for orphan_id in seen_sidecars:
try:
self._get_container_by_sandbox_id(orphan_id)
except HTTPException as exc:
if exc.status_code == status.HTTP_404_NOT_FOUND:
self._cleanup_egress_sidecar(orphan_id)
else:
logger.warning(
f"Failed to check sandbox {orphan_id} for orphan sidecar cleanup: {exc}"
)
if restored:
logger.info(f"Restored expiration timers for {restored} sandbox(es).")
def _container_to_sandbox(self, container, sandbox_id: Optional[str] = None) -> Sandbox:
labels = container.attrs.get("Config", {}).get("Labels") or {}
resolved_id = sandbox_id or labels.get(SANDBOX_ID_LABEL)
if not resolved_id:
raise HTTPException(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
detail={
"code": SandboxErrorCodes.SANDBOX_NOT_FOUND,
"message": "Container missing sandbox ID label.",
},
)
status_section = container.attrs.get("State", {})
finished_at = status_section.get("FinishedAt")
state, reason, message = _container_lifecycle_state(container)
metadata = self._metadata_store.get(resolved_id, labels)
entrypoint = container.attrs.get("Config", {}).get("Cmd") or []
if isinstance(entrypoint, str):
entrypoint = [entrypoint]
# The image reference the container was created with. Do not look the
# image up via container.image: it may have been removed since.
image_uri = container.attrs.get("Config", {}).get("Image")
snapshot_id = labels.get(SANDBOX_SNAPSHOT_ID_LABEL) or None
image_spec = None if snapshot_id else ImageSpec(uri=image_uri)
created_at = parse_timestamp(
labels.get(SANDBOX_CREATED_AT_LABEL) or container.attrs.get("Created")
)
transitioned_at = self._metadata_store.get_transition_time(resolved_id, state)
last_transition_at = (
parse_timestamp(finished_at)
if finished_at and finished_at != "0001-01-01T00:00:00Z"
else parse_timestamp(transitioned_at) if transitioned_at else created_at
)
expires_at = self._get_tracked_expiration(resolved_id, labels)
status_info = SandboxStatus(
state=state,
reason=reason,
message=message,
last_transition_at=last_transition_at,
)
platform_spec = self._resolve_platform_for_container(container, labels)
return Sandbox(
id=resolved_id,
image=image_spec,
snapshotId=snapshot_id,
platform=platform_spec,
status=status_info,
metadata=metadata,
extensions=extract_extensions_from_mapping(labels),
entrypoint=entrypoint,
expiresAt=expires_at,
createdAt=created_at,
)
def _prepare_creation_context(
self,
request: CreateSandboxRequest,
) -> tuple[str, datetime, Optional[datetime]]:
sandbox_id = self.generate_sandbox_id()
created_at = datetime.now(timezone.utc)
expires_at = None
if request.timeout is not None:
expires_at = calculate_expiration_or_raise(created_at, request.timeout)
return sandbox_id, created_at, expires_at
async def create_sandbox(self, request: CreateSandboxRequest) -> CreateSandboxResponse:
if request.lifecycle is not None:
# renewalPolicy (OSEP-0009) is control-plane state, persisted as a
# label; hooks and idlePolicy stay unsupported on docker.
unsupported: list[str] = []
if request.lifecycle.has_hooks:
unsupported.append("lifecycle.preStart/periodic hooks")
if request.lifecycle.idle_policy is not None:
unsupported.append("lifecycle.idlePolicy")
if unsupported:
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail={
"code": SandboxErrorCodes.INVALID_PARAMETER,
"message": (
f"{', '.join(unsupported)} not supported by the Docker "
"provider; only lifecycle.renewalPolicy (OSEP-0009) is "
"accepted on the lifecycle object."
),
},
)
if (request.extensions or {}).get("poolRef", "").strip():
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail={
"code": "SANDBOX::UNSUPPORTED_POOL_REF",
"message": "poolRef is not supported by the Docker provider. Use Kubernetes BatchSandbox provider instead.",
},
)
request = await resolve_sandbox_image_from_request(request)
ensure_entrypoint(request.entrypoint or [])
ensure_metadata_labels(request.metadata)
ensure_platform_valid(request.platform)
ensure_timeout_within_limit(
request.timeout,
self.app_config.server.max_sandbox_timeout_seconds,
)
self._ensure_secure_access_support(request)
self._ensure_network_policy_support(request)
resource_limits = self._prepare_resource_limits(request)
self._validate_volume_definitions(request)
# Docker API calls block; keep them off the event loop.
await asyncio.to_thread(self._validate_network_exists)
try:
sandbox_env, egress_env = split_egress_env(request.env)
except ValueError as e:
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail={
"code": SandboxErrorCodes.INVALID_PARAMETER,
"message": str(e),
},
) from e
sandbox_id, created_at, expires_at = self._prepare_creation_context(request)
loop = asyncio.get_running_loop()
future: asyncio.Future[CreateSandboxResponse] = loop.create_future()
def _run() -> None:
auto_created_volumes: list[str] = []
try:
# Volume validation calls the Docker API and may create
# volumes, so it runs here, off the event loop, where a failed
# create removes them below.
pvc_inspect_cache, auto_created_volumes = self._validate_volumes(request)
result = self._provision_sandbox(
sandbox_id, request, created_at, expires_at, resource_limits,
pvc_inspect_cache, auto_created_volumes,
sandbox_env=sandbox_env, egress_env=egress_env,
)
loop.call_soon_threadsafe(future.set_result, result)
except BaseException as exc:
# _provision_sandbox only removes these when it fails inside its
# main try block. Earlier failures (the image pull, request
# checks) would leak them, and delete_sandbox never runs for a
# sandbox that was never created. Removal is idempotent, and a
# cleanup error must not keep exc from reaching the caller.
try:
self._cleanup_managed_volumes(sandbox_id, auto_created_volumes)
except Exception as cleanup_exc: # noqa: BLE001
logger.warning(
f"sandbox={sandbox_id} | failed to clean up volumes "
f"after create failure: {cleanup_exc}"
)
try:
loop.call_soon_threadsafe(future.set_exception, exc)
except RuntimeError:
pass
Thread(target=_run, daemon=True).start()
try:
return await future
except ValueError as e:
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail={
"code": SandboxErrorCodes.INVALID_PARAMETER,
"message": str(e),
},
) from e
def _prepare_resource_limits(
self, request: CreateSandboxRequest
) -> tuple[Optional[int], Optional[int], Optional[int]]:
"""Validate platform resource limits and return outer Docker limits."""
if is_windows_platform(request.platform):
validate_windows_resource_limits(
(request.resource_limits.root if request.resource_limits else None) or {}
)
return None, None, None
return self._resolve_resource_limits(request)
def _provision_sandbox(
self,
sandbox_id: str,
request: CreateSandboxRequest,
created_at: datetime,
expires_at: Optional[datetime],
resource_limits: tuple[Optional[int], Optional[int], Optional[int]],
pvc_inspect_cache: Optional[dict[str, dict]] = None,
auto_created_volumes: Optional[list[str]] = None,
sandbox_env: Optional[Dict[str, Optional[str]]] = None,
egress_env: Optional[Dict[str, Optional[str]]] = None,
) -> CreateSandboxResponse:
if sandbox_env is None and egress_env is None:
sandbox_env, egress_env = split_egress_env(request.env)
patched_request = request.model_copy(update={"env": sandbox_env or None})
labels, environment = self._build_labels_and_env(sandbox_id, patched_request, expires_at)
labels[SANDBOX_CREATED_AT_LABEL] = created_at.isoformat()
if auto_created_volumes:
labels[SANDBOX_MANAGED_VOLUMES_LABEL] = json.dumps(
auto_created_volumes, separators=(",", ":"),
)
image_uri, auth_config = self._resolve_image_auth(request, sandbox_id)
mem_limit, nano_cpus, gpu_count = resource_limits
egress_token: Optional[str] = None
requested_windows_profile = is_windows_platform(request.platform)
credential_proxy_enabled = bool(
request.credential_proxy and request.credential_proxy.enabled
)
if credential_proxy_enabled and egress_env.get(OPENSANDBOX_EGRESS_MITMPROXY_SSL_INSECURE):
raise ValueError(
f"'{OPENSANDBOX_EGRESS_MITMPROXY_SSL_INSECURE}' cannot be set when credential proxy is enabled"
)
validate_upstream_proxy_request(
self.app_config.egress,
has_network_policy=bool(request.network_policy),
credential_proxy_enabled=credential_proxy_enabled,
egress_env=egress_env or {},
)
if egress_env and not request.network_policy:
dropped_keys = sorted(egress_env.keys())
logger.warning(
f"Sandbox {sandbox_id} has OPENSANDBOX_EGRESS_ env vars {dropped_keys} "
"but no networkPolicy; these variables will be ignored because no "
"egress sidecar is created"
)
egress_env = {}
if requested_windows_profile:
validate_windows_runtime_prerequisites()
# Prepare OSSFS mounts first so binds can reference mounted host paths.
ossfs_mount_keys = self._prepare_ossfs_mounts(request.volumes)
if ossfs_mount_keys:
labels[SANDBOX_OSSFS_MOUNTS_LABEL] = json.dumps(
ossfs_mount_keys,
separators=(",", ":"),
)
sidecar_container = None
reserved_port_bindings: dict[str, tuple[str, int]] = {}
runtime_volume_name: Optional[str] = None
try:
# For dockur/windows profile, resourceLimits are translated to
# guest envs (RAM_SIZE/CPU_CORES/DISK_SIZE). Avoid applying
# container cgroup memory/cpu limits to the outer Linux container,
# which can OOM-kill QEMU during installation/runtime. GPU
# passthrough is likewise suppressed: the Windows guest runs inside
# QEMU and would not see GPUs exposed to the outer container.
effective_mem_limit = None if requested_windows_profile else mem_limit
effective_nano_cpus = None if requested_windows_profile else nano_cpus
effective_gpu_count = None if requested_windows_profile else gpu_count
# pvc_inspect_cache carries Docker volume inspect data from the
# validation phase, avoiding a redundant API call.
volume_binds = self._build_volume_binds(request.volumes, pvc_inspect_cache)
host_config_kwargs: Dict[str, Any]
exposed_ports: Optional[list[str]] = ["44772", "8080"]
if requested_windows_profile:
# dockur/windows exposes RDP and noVNC/web UI on these ports.
# https://github.com/dockur/windows/blob/master/Dockerfile
exposed_ports.extend(["3389/tcp", "3389/udp", "8006/tcp"])
container_exposed_ports: Optional[list[str]] = exposed_ports
if request.network_policy:
runtime_volume_name = f"opensandbox-runtime-{sandbox_id}"
with self._docker_operation(
"create egress runtime volume", sandbox_id
):
self.docker_client.volumes.create(
name=runtime_volume_name,
labels={SANDBOX_MANAGED_VOLUMES_LABEL: "server"},
)
auto_created_volumes = list(auto_created_volumes or [])
auto_created_volumes.append(runtime_volume_name)
labels[SANDBOX_MANAGED_VOLUMES_LABEL] = json.dumps(
auto_created_volumes,
separators=(",", ":"),
)
if credential_proxy_enabled:
environment = [
entry
for entry in environment
if not entry.startswith(f"{OPENSANDBOX_EGRESS_MITMPROXY_TRANSPARENT}=")
]
environment.append(f"{OPENSANDBOX_EGRESS_MITMPROXY_TRANSPARENT}=true")
egress_token = generate_egress_token()
labels[SANDBOX_EGRESS_AUTH_TOKEN_METADATA_KEY] = egress_token
sidecar_port_bindings = allocate_port_bindings(
[*exposed_ports, "18080"],
min_port=self.app_config.docker.port_range_min,
max_port=self.app_config.docker.port_range_max,
publish_host=self.app_config.docker.publish_host,
)
reserved_port_bindings = sidecar_port_bindings
host_execd_port = sidecar_port_bindings["44772"][1]
host_http_port = sidecar_port_bindings["8080"][1]
egress_api_binding = sidecar_port_bindings.get("18080")
extra_sidecar_port_bindings = {
port: binding
for port, binding in sidecar_port_bindings.items()
if port not in {"44772", "8080"}
}
sidecar_container = self._start_egress_sidecar(
sandbox_id=sandbox_id,
network_policy=request.network_policy,
egress_token=egress_token,
host_execd_port=host_execd_port,
host_http_port=host_http_port,
extra_port_bindings=extra_sidecar_port_bindings,
egress_api_host_port=(
egress_api_binding[1] if egress_api_binding is not None else None
),
runtime_volume_name=runtime_volume_name,
credential_proxy_enabled=credential_proxy_enabled,
extra_env=egress_env or None,
)
labels[SANDBOX_EMBEDDING_PROXY_PORT_LABEL] = str(host_execd_port)
labels[SANDBOX_HTTP_PORT_LABEL] = str(host_http_port)
host_config_kwargs = self._base_host_config_kwargs(
effective_mem_limit, effective_nano_cpus, f"container:{sidecar_container.id}",
gpu_count=effective_gpu_count,
)
# Container network namespace is shared with sidecar. Docker rejects
# exposing ports on the main container in "container:<id>" mode.
container_exposed_ports = None
# Drop NET_ADMIN for the main container; only the sidecar should keep it
cap_drop = set(host_config_kwargs.get("cap_drop") or [])
cap_drop.add("NET_ADMIN")
if cap_drop:
host_config_kwargs["cap_drop"] = list(cap_drop)
else:
host_config_kwargs = self._base_host_config_kwargs(
effective_mem_limit, effective_nano_cpus, self.network_mode,
gpu_count=effective_gpu_count,
)
if self.network_mode != HOST_NETWORK_MODE:
port_bindings = allocate_port_bindings(
exposed_ports,
min_port=self.app_config.docker.port_range_min,
max_port=self.app_config.docker.port_range_max,
publish_host=self.app_config.docker.publish_host,
)
reserved_port_bindings = port_bindings
host_execd_port = port_bindings["44772"][1]
host_http_port = port_bindings["8080"][1]
host_config_kwargs["port_bindings"] = normalize_port_bindings(port_bindings)
labels[SANDBOX_EMBEDDING_PROXY_PORT_LABEL] = str(host_execd_port)
labels[SANDBOX_HTTP_PORT_LABEL] = str(host_http_port)
else:
exposed_ports = None
if runtime_volume_name:
runtime_mount_suffix = f":{OPENSANDBOX_RUNTIME_MOUNT_PATH}:"
runtime_mount_end = f":{OPENSANDBOX_RUNTIME_MOUNT_PATH}"
has_runtime_mount = any(
runtime_mount_suffix in bind or bind.endswith(runtime_mount_end)
for bind in volume_binds
)
if not has_runtime_mount:
volume_binds.append(
f"{runtime_volume_name}:{OPENSANDBOX_RUNTIME_MOUNT_PATH}:rw"
)
# Config-level binds (docker.sandbox_binds) apply to every sandbox
# and come first.
all_binds = list(self.app_config.docker.sandbox_binds or []) + (volume_binds or [])
if all_binds:
host_config_kwargs["binds"] = all_binds
if requested_windows_profile:
host_config_kwargs = apply_windows_runtime_host_config_defaults(
host_config_kwargs,
sandbox_id,
)
environment = inject_windows_resource_limits_env(
environment,
(request.resource_limits.root if request.resource_limits else None) or {},
)
environment = inject_windows_user_ports(environment, exposed_ports)
# Inject the container permissions required by bwrap isolation.
# bwrap needs pivot_root/mount, including a fresh procfs, which are blocked
# by Docker's default AppArmor, seccomp, and protected-system-path settings.
isolation_requested = (
(request.extensions or {}).get(BOOTSTRAP_EXECD_ISOLATION_KEY)
== "enable"
)
if isolation_requested:
cap_add = set(host_config_kwargs.get("cap_add") or [])
cap_add.add("SYS_ADMIN")
host_config_kwargs["cap_add"] = sorted(cap_add)
security_opt = list(host_config_kwargs.get("security_opt") or [])
security_opt = [s for s in security_opt if not s.startswith(("apparmor=", "seccomp="))]
security_opt.append("apparmor=unconfined")
security_opt.append("seccomp=unconfined")
host_config_kwargs["security_opt"] = security_opt
tmpfs = dict(host_config_kwargs.get("tmpfs") or {})
tmpfs[ISOLATION_UPPER_MOUNT_PATH] = ""
host_config_kwargs["tmpfs"] = tmpfs
logger.warning(
f"sandbox {sandbox_id}: granting CAP_SYS_ADMIN + "
"apparmor/seccomp/system paths unconfined + tmpfs for bwrap isolation "
"(bootstrap.execd.isolation=enable)"
)
if self.network_mode != HOST_NETWORK_MODE and request.network_policy is None:
for attempt in range(MAX_PORT_PUBLISH_ATTEMPTS):
try:
created_container = self._create_and_start_container(
sandbox_id,
image_uri,
request.entrypoint,
labels,
environment,
host_config_kwargs,
container_exposed_ports,
request.platform,
unconfine_system_paths=isolation_requested,
)
break
except Exception as exc:
if (
attempt + 1 < MAX_PORT_PUBLISH_ATTEMPTS
and is_port_publish_error(exc)
):
logger.warning(
f"sandbox {sandbox_id}: container start failed due to "
f"host port conflict: {exc}. Retrying with fresh "
f"ports (attempt {attempt + 1}/{MAX_PORT_PUBLISH_ATTEMPTS})..."
)
if reserved_port_bindings:
release_port_bindings(reserved_port_bindings)
reserved_port_bindings = {}
port_bindings = allocate_port_bindings(
exposed_ports,
min_port=self.app_config.docker.port_range_min,
max_port=self.app_config.docker.port_range_max,
publish_host=self.app_config.docker.publish_host,
)
reserved_port_bindings = port_bindings
host_execd_port = port_bindings["44772"][1]
host_http_port = port_bindings["8080"][1]
host_config_kwargs["port_bindings"] = normalize_port_bindings(port_bindings)
labels[SANDBOX_EMBEDDING_PROXY_PORT_LABEL] = str(host_execd_port)
labels[SANDBOX_HTTP_PORT_LABEL] = str(host_http_port)
continue
raise
else:
created_container = self._create_and_start_container(
sandbox_id,
image_uri,
request.entrypoint,
labels,
environment,
host_config_kwargs,
container_exposed_ports,
request.platform,
unconfine_system_paths=isolation_requested,
)
except Exception:
if sidecar_container is not None:
try:
sidecar_container.remove(force=True)
except DockerException as cleanup_exc:
logger.warning(
f"Failed to cleanup egress sidecar for sandbox {sandbox_id}: {cleanup_exc}"
)
self._release_ossfs_mounts(ossfs_mount_keys)
self._cleanup_managed_volumes(sandbox_id, auto_created_volumes or [])
raise
finally:
if reserved_port_bindings:
release_port_bindings(reserved_port_bindings)
status_info = SandboxStatus(
state="Running",
reason="CONTAINER_RUNNING",
message="Sandbox container started successfully.",
last_transition_at=created_at,
)
if expires_at is not None:
self._schedule_expiration(sandbox_id, expires_at)
effective_platform = self._resolve_platform_for_container(created_container, labels)
return CreateSandboxResponse(
id=sandbox_id,
status=status_info,
metadata=request.metadata,
extensions=extract_extensions_from_mapping(labels),
platform=effective_platform or request.platform,
expiresAt=expires_at,
createdAt=created_at,
entrypoint=request.entrypoint,
)
def list_sandboxes(self, request: ListSandboxesRequest) -> ListSandboxesResponse:
"""List sandboxes with optional filtering and pagination.
Discovers container IDs via ``docker_client.api.containers`` and then
inspects each one via ``containers.get(id)``. This gives an explicit
per-container failure boundary: if a container is removed between the
list summary and its inspect, that entry is skipped instead of
failing the whole listing with 500. The high-level
``docker_client.containers.list(filters=...)`` inlines the same
second-stage inspect but does not expose that boundary.
"""
try:
summaries = self.docker_client.api.containers(
all=True,
filters={"label": [SANDBOX_ID_LABEL]},
trunc=False,
)
except DockerException as exc:
raise HTTPException(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
detail={
"code": SandboxErrorCodes.CONTAINER_QUERY_FAILED,
"message": f"Failed to query sandbox containers: {str(exc)}",
},
) from exc
sandboxes_by_id: dict[str, Sandbox] = {}
for summary in summaries:
container_id = summary.get("Id")
summary_labels = summary.get("Labels") or {}
sandbox_id = summary_labels.get(SANDBOX_ID_LABEL)
if not container_id or not sandbox_id:
continue
try:
container = self.docker_client.containers.get(container_id)
except DockerNotFound:
# Removed between list and inspect; skip so concurrent
# deletions do not fail the whole listing.
continue
except DockerException as exc:
raise HTTPException(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
detail={
"code": SandboxErrorCodes.CONTAINER_QUERY_FAILED,
"message": f"Failed to query sandbox containers: {str(exc)}",
},
) from exc
sandbox_obj = self._container_to_sandbox(container, sandbox_id)
if matches_filter(sandbox_obj, request.filter):
sandboxes_by_id[sandbox_id] = sandbox_obj
sandboxes: list[Sandbox] = list(sandboxes_by_id.values())
sandboxes.sort(key=lambda s: s.created_at or datetime.min, reverse=True)
if request.pagination:
page = request.pagination.page
page_size = request.pagination.page_size
else:
page = 1
page_size = 20
total_items = len(sandboxes)
total_pages = math.ceil(total_items / page_size) if total_items else 0
start_index = (page - 1) * page_size
end_index = start_index + page_size
items = sandboxes[start_index:end_index]
has_next_page = page < total_pages
pagination_info = PaginationInfo(
page=page,
page_size=page_size,
total_items=total_items,
total_pages=total_pages,
has_next_page=has_next_page,
)
return ListSandboxesResponse(items=items, pagination=pagination_info)
def get_sandbox(self, sandbox_id: str) -> Sandbox:
container = self._get_container_by_sandbox_id(sandbox_id)
return self._container_to_sandbox(container, sandbox_id)
def _raise_if_lifecycle_race(
self, sandbox_id: str, exc: DockerException, operation: str,
) -> None:
"""Map known main-container races using the sequential error envelope."""
if isinstance(exc, DockerNotFound):
if operation == "delete":
self._cleanup_egress_sidecar(sandbox_id)
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail={
"code": SandboxErrorCodes.SANDBOX_NOT_FOUND,
"message": f"Sandbox {sandbox_id} not found.",
},
) from exc
if not isinstance(exc, APIError) or exc.status_code != status.HTTP_409_CONFLICT:
return
message = (exc.explanation or "").lower()
conflicts = {
"delete": (
("removal of container " in message and "already in progress" in message)
or "already being removed" in message,
SandboxErrorCodes.SANDBOX_DELETE_FAILED,
"Sandbox deletion is already in progress.",
),
"pause": (
"already paused" in message or "is not running" in message,
SandboxErrorCodes.SANDBOX_NOT_RUNNING,
"Sandbox is not in a running state.",
),
"resume": (
"is not paused" in message or "is not running" in message,
SandboxErrorCodes.SANDBOX_NOT_PAUSED,
"Sandbox is not in a paused state.",
),
}
matches, code, conflict_message = conflicts[operation]
if not matches:
return
if operation != "delete":
# The state error may have raced with removal; use the usual 404
# if the sandbox has disappeared since the operation failed.
self._get_container_by_sandbox_id(sandbox_id)
raise HTTPException(
status_code=status.HTTP_409_CONFLICT,
detail={"code": code, "message": conflict_message},
) from exc
def delete_sandbox(self, sandbox_id: str) -> None:
with self._sandbox_lifecycle_lock(sandbox_id):
self._delete_sandbox_locked(sandbox_id)
def _delete_sandbox_locked(self, sandbox_id: str) -> None:
try:
container = self._get_container_by_sandbox_id(sandbox_id)
except HTTPException as exc:
if exc.status_code == status.HTTP_404_NOT_FOUND:
self._cleanup_egress_sidecar(sandbox_id)
raise
labels = container.attrs.get("Config", {}).get("Labels") or {}
mount_keys_raw = labels.get(SANDBOX_OSSFS_MOUNTS_LABEL, "[]")
try:
mount_keys: list[str] = json.loads(mount_keys_raw)
except (TypeError, json.JSONDecodeError):
mount_keys = []
managed_volumes_raw = labels.get(SANDBOX_MANAGED_VOLUMES_LABEL, "[]")
try:
managed_volumes: list[str] = json.loads(managed_volumes_raw)
except (TypeError, json.JSONDecodeError):
managed_volumes = []
try:
try:
with self._docker_operation("kill sandbox container", sandbox_id):
container.kill()
except DockerException as exc:
# Ignore error if container is already stopped
if "is not running" not in str(exc).lower():
raise
with self._docker_operation("remove sandbox container", sandbox_id):
container.remove(force=True)
except DockerException as exc:
self._raise_if_lifecycle_race(sandbox_id, exc, "delete")
raise HTTPException(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
detail={
"code": SandboxErrorCodes.SANDBOX_DELETE_FAILED,
"message": f"Failed to delete sandbox container: {str(exc)}",
},
) from exc
self._remove_expiration_tracking(sandbox_id)
self._cleanup_egress_sidecar(sandbox_id)
self._cleanup_windows_oem_volume(sandbox_id, labels)
self._release_ossfs_mounts(mount_keys)
self._cleanup_managed_volumes(sandbox_id, managed_volumes)
self._metadata_store.delete(sandbox_id)
def pause_sandbox(self, sandbox_id: str) -> None:
with self._sandbox_lifecycle_lock(sandbox_id):
self._pause_sandbox_locked(sandbox_id)
def _pause_sandbox_locked(self, sandbox_id: str) -> None:
container = self._get_container_by_sandbox_id(sandbox_id)
state = container.attrs.get("State", {})
if not state.get("Running", False) or state.get("Paused", False):
raise HTTPException(
status_code=status.HTTP_409_CONFLICT,
detail={
"code": SandboxErrorCodes.SANDBOX_NOT_RUNNING,
"message": "Sandbox is not in a running state.",
},
)
labels = container.attrs.get("Config", {}).get("Labels") or {}
egress_expected = bool(labels.get(SANDBOX_EGRESS_AUTH_TOKEN_METADATA_KEY))
try:
with self._docker_operation("query egress sidecar", sandbox_id):
sidecars = self._get_egress_sidecars(sandbox_id)
except DockerException as exc:
raise HTTPException(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
detail={
"code": SandboxErrorCodes.SANDBOX_PAUSE_FAILED,
"message": f"Failed to query egress sidecar: {str(exc)}",
},
) from exc
try:
with self._docker_operation("pause sandbox container", sandbox_id):
container.pause()
except DockerException as exc:
self._raise_if_lifecycle_race(sandbox_id, exc, "pause")
raise HTTPException(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
detail={
"code": SandboxErrorCodes.SANDBOX_PAUSE_FAILED,
"message": f"Failed to pause sandbox container: {str(exc)}",
},
) from exc
self._persist_transition(sandbox_id, "Paused")
if egress_expected and not sidecars:
raise HTTPException(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
detail={
"code": SandboxErrorCodes.SANDBOX_PAUSE_FAILED,
"message": (
"Sandbox container was paused, but the expected egress sidecar was not found."
),
},
)
paused_sidecars: list[Any] = []
try:
for sidecar in sidecars:
sidecar_state = sidecar.attrs.get("State", {})
if sidecar_state.get("Paused", False):
continue
if not sidecar_state.get("Running", False):
raise DockerException(
f"Egress sidecar {sidecar.id} is not in a running state."
)
with self._docker_operation("pause egress sidecar", sandbox_id):
sidecar.pause()
paused_sidecars.append(sidecar)
except DockerException as exc:
rollback_errors: list[str] = []
for paused_sidecar in reversed(paused_sidecars):
try:
with self._docker_operation("rollback egress sidecar pause", sandbox_id):
paused_sidecar.unpause()
except DockerException as rollback_exc:
logger.warning(
f"sandbox={sandbox_id} | failed to rollback egress "
f"sidecar pause: {rollback_exc}"
)
rollback_errors.append(str(rollback_exc))
try:
with self._docker_operation("rollback sandbox pause", sandbox_id):
container.unpause()
except DockerException as rollback_exc:
logger.warning(
f"sandbox={sandbox_id} | failed to rollback sandbox pause: {rollback_exc}"
)
rollback_errors.append(str(rollback_exc))
else:
self._persist_transition(sandbox_id, "Running")
message = f"Failed to pause egress sidecar: {str(exc)}"
if rollback_errors:
message += f"; rollback failed: {'; '.join(rollback_errors)}"
raise HTTPException(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
detail={
"code": SandboxErrorCodes.SANDBOX_PAUSE_FAILED,
"message": message,
},
) from exc
def resume_sandbox(self, sandbox_id: str) -> None:
with self._sandbox_lifecycle_lock(sandbox_id):
self._resume_sandbox_locked(sandbox_id)
def _resume_sandbox_locked(self, sandbox_id: str) -> None:
container = self._get_container_by_sandbox_id(sandbox_id)
state = container.attrs.get("State", {})
if not state.get("Paused", False):
raise HTTPException(
status_code=status.HTTP_409_CONFLICT,
detail={
"code": SandboxErrorCodes.SANDBOX_NOT_PAUSED,
"message": "Sandbox is not in a paused state.",
},
)
labels = container.attrs.get("Config", {}).get("Labels") or {}
egress_expected = bool(labels.get(SANDBOX_EGRESS_AUTH_TOKEN_METADATA_KEY))
try:
with self._docker_operation("query egress sidecar", sandbox_id):
sidecars = self._get_egress_sidecars(sandbox_id)
except DockerException as exc:
raise HTTPException(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
detail={
"code": SandboxErrorCodes.SANDBOX_RESUME_FAILED,
"message": f"Failed to query egress sidecar: {str(exc)}",
},
) from exc
if egress_expected and not sidecars:
raise HTTPException(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
detail={
"code": SandboxErrorCodes.SANDBOX_RESUME_FAILED,
"message": "The expected egress sidecar was not found; sandbox remains paused.",
},
)
resumed_sidecars: list[Any] = []
resuming_main = False
try:
for sidecar in sidecars:
sidecar_state = sidecar.attrs.get("State", {})
if not sidecar_state.get("Paused", False):
if sidecar_state.get("Running", False):
continue
raise DockerException(
f"Egress sidecar {sidecar.id} is not in a paused state."
)
with self._docker_operation("resume egress sidecar", sandbox_id):
sidecar.unpause()
resumed_sidecars.append(sidecar)
resuming_main = True
with self._docker_operation("resume sandbox container", sandbox_id):
container.unpause()
except DockerException as exc:
rollback_errors: list[str] = []
for resumed_sidecar in reversed(resumed_sidecars):
try:
with self._docker_operation("rollback egress sidecar resume", sandbox_id):
resumed_sidecar.pause()
except DockerException as rollback_exc:
logger.warning(
f"sandbox={sandbox_id} | failed to rollback egress "
f"sidecar resume: {rollback_exc}"
)
rollback_errors.append(str(rollback_exc))
failed_component = "sandbox container" if resuming_main else "egress sidecar"
message = f"Failed to resume {failed_component}: {str(exc)}"
if rollback_errors:
message += f"; rollback failed: {'; '.join(rollback_errors)}"
elif resuming_main:
self._raise_if_lifecycle_race(sandbox_id, exc, "resume")
raise HTTPException(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
detail={
"code": SandboxErrorCodes.SANDBOX_RESUME_FAILED,
"message": message,
},
) from exc
self._persist_transition(sandbox_id, "Running")
def get_access_renew_extend_seconds(self, sandbox_id: str) -> Optional[int]:
try:
container = self._get_container_by_sandbox_id(sandbox_id)
except HTTPException:
return None
labels = container.attrs.get("Config", {}).get("Labels") or {}
raw = labels.get(ACCESS_RENEW_EXTEND_SECONDS_METADATA_KEY)
if raw is None or not str(raw).strip():
return None
try:
return int(str(raw).strip())
except ValueError:
return None
def renew_expiration(
self,
sandbox_id: str,
request: RenewSandboxExpirationRequest,
) -> RenewSandboxExpirationResponse:
container = self._get_container_by_sandbox_id(sandbox_id)
new_expiration = ensure_future_expiration(request.expires_at)
labels = container.attrs.get("Config", {}).get("Labels") or {}
if self._has_manual_cleanup(labels):
raise HTTPException(
status_code=status.HTTP_409_CONFLICT,
detail={
"code": SandboxErrorCodes.INVALID_EXPIRATION,
"message": f"Sandbox {sandbox_id} does not have automatic expiration enabled.",
},
)
if self._get_tracked_expiration(sandbox_id, labels) is None:
raise HTTPException(
status_code=status.HTTP_409_CONFLICT,
detail={
"code": SandboxErrorCodes.INVALID_EXPIRATION,
"message": (
f"Sandbox {sandbox_id} is missing expiration metadata and cannot be renewed safely."
),
},
)
# Persist the new timeout in memory; the file-backed override also keeps renewals correct across restarts
self._schedule_expiration(sandbox_id, new_expiration)
try:
self._metadata_store.set_expiration(sandbox_id, new_expiration)
except OSError as exc:
logger.warning(f"Failed to persist expiration override for sandbox {sandbox_id}: {exc}")
labels[SANDBOX_EXPIRES_AT_LABEL] = new_expiration.isoformat()
try:
with self._docker_operation("update sandbox labels", sandbox_id):
self._update_container_labels(container, labels)
except (DockerException, TypeError) as exc:
logger.warning(f"Failed to refresh labels for sandbox {sandbox_id}: {exc}")
return RenewSandboxExpirationResponse(expires_at=new_expiration)
def patch_sandbox_metadata(self, sandbox_id: str, patch: PatchSandboxMetadataRequest) -> Sandbox:
"""Patch sandbox metadata via JSON Merge Patch (RFC 7396). Docker cannot update labels on running containers, so metadata is persisted to file."""
from opensandbox_server.services.validators import ensure_metadata_labels
container = self._get_container_by_sandbox_id(sandbox_id)
labels = dict(container.attrs.get("Config", {}).get("Labels") or {})
for key in patch:
if SandboxService._is_system_label(key):
raise HTTPException(
status_code=400,
detail={
"code": "INVALID_METADATA_LABEL",
"message": f"Metadata key '{key}' is reserved (opensandbox.io/ prefix).",
},
)
patch_additions = {k: str(v) for k, v in patch.items() if v is not None}
if patch_additions:
ensure_metadata_labels(patch_additions)
self._metadata_store.patch(sandbox_id, labels, patch)
return self._container_to_sandbox(container, sandbox_id)