Skip to content

Secure, Fast, and Extensible Sandbox runtime for AI agents.

65.5 KB
Raw Blame
# Copyright 2025 The OpenSandbox Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
Docker-based implementation of SandboxService.

This module provides a Docker implementation of the sandbox service interface,
using Docker containers for sandbox lifecycle management.
"""

from __future__ import annotations

import asyncio
import inspect
import json
import logging
import math
import os
import time
from _thread import LockType
from contextlib import contextmanager
from datetime import datetime, timedelta, timezone
from threading import Lock, Thread, Timer
from typing import Any, Dict, Optional
from weakref import WeakValueDictionary

import docker
from docker.errors import APIError, DockerException, NotFound as DockerNotFound
from fastapi import HTTPException, status

from opensandbox_server.extensions import (
    ACCESS_RENEW_EXTEND_SECONDS_METADATA_KEY,
    BOOTSTRAP_EXECD_ISOLATION_KEY,
    ISOLATION_UPPER_MOUNT_PATH,
    extract_extensions_from_mapping,
)
from opensandbox_server.api.schema import (
    CreateSandboxRequest,
    CreateSandboxResponse,
    ImageSpec,
    ListSandboxesRequest,
    ListSandboxesResponse,
    PaginationInfo,
    PatchSandboxMetadataRequest,
    RenewSandboxExpirationRequest,
    RenewSandboxExpirationResponse,
    Sandbox,
    SandboxStatus,
    PlatformSpec,
)
from opensandbox_server.config import AppConfig, get_config
from opensandbox_server.services.docker.docker_diagnostics import (
    DockerDiagnosticsMixin,
    _container_lifecycle_state,
)
from opensandbox_server.services.docker.runtime import (
    DockerRuntimeMixin,
)
from opensandbox_server.services.docker.volumes import DockerVolumesMixin
from opensandbox_server.services.docker.networking import (
    EGRESS_SIDECAR_LABEL,
    HOST_NETWORK_MODE,
    DockerNetworkingMixin,
)
from opensandbox_server.services.docker.container_ops import DockerContainerOpsMixin
from opensandbox_server.services.docker.metadata import DockerMetadataStore
from opensandbox_server.services.docker.port_allocator import (
    MAX_PORT_PUBLISH_ATTEMPTS,
    allocate_port_bindings,
    is_port_publish_error,
    normalize_port_bindings,
    release_port_bindings,
)
from opensandbox_server.services.windows_common import (
    inject_windows_resource_limits_env,
    inject_windows_user_ports,
    is_windows_platform,
    validate_windows_resource_limits,
)
from opensandbox_server.services.docker.windows_profile import (
    apply_windows_runtime_host_config_defaults,
    validate_windows_runtime_prerequisites,
)
from opensandbox_server.services.extension_service import ExtensionService
from opensandbox_server.services.constants import (
    OPENSANDBOX_EGRESS_MITMPROXY_SSL_INSECURE,
    OPENSANDBOX_EGRESS_MITMPROXY_TRANSPARENT,
    OPENSANDBOX_RUNTIME_MOUNT_PATH,
    SANDBOX_CREATED_AT_LABEL,
    SANDBOX_EGRESS_AUTH_TOKEN_METADATA_KEY,
    SANDBOX_EMBEDDING_PROXY_PORT_LABEL,
    SANDBOX_EXPIRES_AT_LABEL,
    SANDBOX_HTTP_PORT_LABEL,
    SANDBOX_ID_LABEL,
    SANDBOX_MANAGED_VOLUMES_LABEL,
    SANDBOX_MANUAL_CLEANUP_LABEL,
    SANDBOX_OSSFS_MOUNTS_LABEL,
    SANDBOX_SNAPSHOT_ID_LABEL,
    SandboxErrorCodes,
)
from opensandbox_server.services.endpoint_auth import (
    generate_egress_token,
)
from opensandbox_server.services.helpers import (
    matches_filter,
    parse_timestamp,
    split_egress_env,
    validate_upstream_proxy_request,
)
from opensandbox_server.services.docker.ossfs_mixin import OSSFSMixin
from opensandbox_server.services.sandbox_service import SandboxService
from opensandbox_server.services.runtime_resolver import SecureRuntimeResolver
from opensandbox_server.services.snapshot_restore import resolve_sandbox_image_from_request
from opensandbox_server.services.validators import (
    calculate_expiration_or_raise,
    ensure_entrypoint,
    ensure_future_expiration,
    ensure_metadata_labels,
    ensure_platform_valid,
    ensure_timeout_within_limit,
)
logger = logging.getLogger(__name__)

class DockerSandboxService(DockerDiagnosticsMixin, DockerRuntimeMixin, DockerVolumesMixin, DockerNetworkingMixin, DockerContainerOpsMixin, OSSFSMixin, SandboxService, ExtensionService):
    def __init__(self, config: Optional[AppConfig] = None):
        """
        Initializes Docker service from environment variables.

        Note: Connection is not verified at initialization time.
        Connection errors will be raised when Docker operations are performed.
        """
        self.app_config = config or get_config()
        runtime_config = self.app_config.runtime
        if runtime_config.type != "docker":
            raise ValueError("DockerSandboxService requires runtime.type = 'docker'.")

        self.execd_image = runtime_config.execd_image
        self.network_mode = (self.app_config.docker.network_mode or HOST_NETWORK_MODE).lower()
        self._execd_archive_cache: Dict[str, bytes] = {}
        self._bootstrap_script_cache: Dict[str, bytes] = {}
        self._bwrap_archive_cache: Dict[str, bytes] = {}
        self._session_gate_archive_cache: Dict[str, bytes] = {}
        self._launcher_archive_cache: Dict[str, bytes] = {}
        self._windows_profile_cache: Dict[str, bytes] = {}
        self._daemon_platform: Optional[PlatformSpec] = None
        self._metadata_store = DockerMetadataStore()
        self._api_timeout = self._resolve_api_timeout()
        try:
            client_kwargs = {}
            try:
                signature = inspect.signature(docker.from_env)
                if "timeout" in signature.parameters:
                    client_kwargs["timeout"] = self._api_timeout
            except (ValueError, TypeError):
                logger.debug(
                    "Unable to introspect docker.from_env signature; using default parameters."
                )
            self.docker_client = docker.from_env(**client_kwargs)
            if not client_kwargs:
                try:
                    self.docker_client.api.timeout = self._api_timeout
                except AttributeError:
                    logger.debug("Docker client API does not expose timeout attribute.")
            logger.info("Docker service initialized from environment")
        except Exception as e:  # noqa: BLE001
            # Common failure mode on macOS/dev machines: Docker daemon not running or socket path wrong.
            hint = ""
            msg = str(e)
            if isinstance(e, FileNotFoundError) or "No such file or directory" in msg:
                docker_host = os.environ.get("DOCKER_HOST", "")
                hint = (
                    " Docker daemon seems unavailable (unix socket not found). "
                    "Make sure Docker Desktop (or Colima/Rancher Desktop) is running. "
                    "If you use Colima on macOS, you may need to set "
                    "DOCKER_HOST=unix://${HOME}/.colima/default/docker.sock before starting the server. "
                    f"(current DOCKER_HOST='{docker_host}')"
                )
            raise HTTPException(
                status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
                detail={
                    "code": SandboxErrorCodes.DOCKER_INITIALIZATION_ERROR,
                    "message": f"Failed to initialize Docker service: {str(e)}.{hint}",
                },
            )
        self._expiration_lock = Lock()
        self._lifecycle_locks_guard = Lock()
        self._lifecycle_locks: WeakValueDictionary[str, LockType] = WeakValueDictionary()
        self._execd_archive_lock = Lock()
        self._sandbox_expirations: Dict[str, datetime] = {}
        self._expiration_timers: Dict[str, Timer] = {}
        self._ossfs_mount_lock = Lock()
        self._ossfs_mount_ref_counts: Dict[str, int] = {}
        self._restore_existing_sandboxes()

        # Initialize secure runtime resolver
        self.resolver = SecureRuntimeResolver(self.app_config)
        self.docker_runtime = self.resolver.get_docker_runtime()

    def _resolve_api_timeout(self) -> int:
        """Docker API timeout in seconds: [docker].api_timeout if set, else default 180."""
        cfg = self.app_config.docker.api_timeout
        if cfg is not None and cfg >= 1:
            return cfg
        return 180

    @contextmanager
    def _sandbox_lifecycle_lock(self, sandbox_id: str):
        # Order runtime mutations and transition records within this service
        # instance only, not across service instances or processes. Keep this
        # separate from the metadata store's non-reentrant file-write lock.
        with self._lifecycle_locks_guard:
            lock = self._lifecycle_locks.get(sandbox_id)
            if lock is None:
                lock = Lock()
                self._lifecycle_locks[sandbox_id] = lock
        # Active and waiting calls retain the lock; idle entries disappear.
        with lock:
            yield

    def _persist_transition(self, sandbox_id: str, state: str) -> None:
        try:
            self._metadata_store.set_transition(sandbox_id, state, datetime.now(timezone.utc))
        except OSError as exc:
            logger.warning(f"Failed to persist {state} transition for sandbox {sandbox_id}: {exc}")

    @contextmanager
    def _docker_operation(self, action: str, sandbox_id: Optional[str] = None):
        op_id = sandbox_id or "shared"
        start = time.perf_counter()
        try:
            yield
        except Exception as exc:
            elapsed_ms = (time.perf_counter() - start) * 1000
            logger.warning(
                f"sandbox={op_id} | action={action} | duration={elapsed_ms:.2f} | error={exc}"
            )
            raise
        else:
            elapsed_ms = (time.perf_counter() - start) * 1000
            logger.info(
                f"sandbox={op_id} | action={action} | duration={elapsed_ms:.2f}"
            )

    def _get_container_by_sandbox_id(self, sandbox_id: str):
        label_selector = f"{SANDBOX_ID_LABEL}={sandbox_id}"
        try:
            try:
                container = self.docker_client.containers.get(f"sandbox-{sandbox_id}")
            except DockerNotFound:
                container = None

            if container is not None:
                labels = container.attrs.get("Config", {}).get("Labels") or {}
                if labels.get(SANDBOX_ID_LABEL) == sandbox_id:
                    return container

            # Preserve lookup for managed containers with a nonstandard name.
            containers = self.docker_client.containers.list(
                all=True, filters={"label": label_selector}
            )
        except DockerNotFound as exc:
            # Container disappeared between list-summary and inspect; treat as
            # not found rather than surfacing a 500. See list_sandboxes for
            # the full-table variant of this fix.
            raise HTTPException(
                status_code=status.HTTP_404_NOT_FOUND,
                detail={
                    "code": SandboxErrorCodes.SANDBOX_NOT_FOUND,
                    "message": f"Sandbox {sandbox_id} not found.",
                },
            ) from exc
        except DockerException as exc:
            raise HTTPException(
                status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
                detail={
                    "code": SandboxErrorCodes.CONTAINER_QUERY_FAILED,
                    "message": f"Failed to query sandbox containers: {str(exc)}",
                },
            ) from exc

        if not containers:
            raise HTTPException(
                status_code=status.HTTP_404_NOT_FOUND,
                detail={
                    "code": SandboxErrorCodes.SANDBOX_NOT_FOUND,
                    "message": f"Sandbox {sandbox_id} not found.",
                },
            )

        return containers[0]

    def _get_egress_sidecars(self, sandbox_id: str) -> list[Any]:
        return self.docker_client.containers.list(
            all=True, filters={"label": f"{EGRESS_SIDECAR_LABEL}={sandbox_id}"}
        )

    def _schedule_expiration(
        self,
        sandbox_id: str,
        expires_at: datetime,
        *,
        update_expiration: bool = True,
        **expire_kwargs,
    ) -> None:
        # Delay might already be negative if the timer should fire immediately
        delay = max(0.0, (expires_at - datetime.now(timezone.utc)).total_seconds())
        timer = Timer(
            delay,
            self._expire_sandbox,
            args=(sandbox_id,),
            kwargs=expire_kwargs or None,
        )
        timer.daemon = True
        with self._expiration_lock:
            # Replace existing timer (if any) so renew operations take effect immediately
            existing = self._expiration_timers.pop(sandbox_id, None)
            if existing:
                existing.cancel()
            if update_expiration:
                self._sandbox_expirations[sandbox_id] = expires_at
            self._expiration_timers[sandbox_id] = timer
        timer.start()

    def _remove_expiration_tracking(self, sandbox_id: str) -> None:
        with self._expiration_lock:
            timer = self._expiration_timers.pop(sandbox_id, None)
            if timer:
                timer.cancel()
            self._sandbox_expirations.pop(sandbox_id, None)

    @staticmethod
    def _has_manual_cleanup(labels: Dict[str, str]) -> bool:
        return labels.get(SANDBOX_MANUAL_CLEANUP_LABEL, "").lower() == "true"

    def _get_tracked_expiration(
        self,
        sandbox_id: str,
        labels: Dict[str, str],
    ) -> Optional[datetime]:
        with self._expiration_lock:
            tracked = self._sandbox_expirations.get(sandbox_id)
        if tracked:
            return tracked
        persisted = self._metadata_store.get_expiration(sandbox_id)
        if persisted:
            return parse_timestamp(persisted)
        label_value = labels.get(SANDBOX_EXPIRES_AT_LABEL)
        if label_value:
            return parse_timestamp(label_value)
        return None

    def _expire_sandbox(
        self,
        sandbox_id: str,
        fallback_mount_keys: Optional[list[str]] = None,
    ) -> None:
        """Timer callback to terminate expired sandboxes."""
        with self._sandbox_lifecycle_lock(sandbox_id):
            self._expire_sandbox_locked(sandbox_id, fallback_mount_keys)

    def _expire_sandbox_locked(
        self,
        sandbox_id: str,
        fallback_mount_keys: Optional[list[str]] = None,
    ) -> None:
        mount_keys: list[str] = []
        try:
            container = self._get_container_by_sandbox_id(sandbox_id)
        except HTTPException as exc:
            if exc.status_code == status.HTTP_404_NOT_FOUND:
                self._cleanup_egress_sidecar(sandbox_id)
                self._remove_expiration_tracking(sandbox_id)
                self._cleanup_windows_oem_volume(sandbox_id, None)
                if fallback_mount_keys:
                    self._release_ossfs_mounts(fallback_mount_keys)
                self._metadata_store.delete(sandbox_id)
            else:
                with self._expiration_lock:
                    current_expires = self._sandbox_expirations.get(sandbox_id)
                now = datetime.now(timezone.utc)
                if current_expires and current_expires > now:
                    logger.info(
                        f"Sandbox {sandbox_id} expiration was renewed; skipping retry."
                    )
                else:
                    logger.warning(
                        f"Failed to fetch sandbox {sandbox_id} for expiration: {exc.detail} — "
                        "scheduling retry in 30s"
                    )
                    retry_at = now + timedelta(seconds=30)
                    self._schedule_expiration(
                        sandbox_id,
                        retry_at,
                        update_expiration=False,
                        fallback_mount_keys=fallback_mount_keys,
                    )
            return

        labels = container.attrs.get("Config", {}).get("Labels") or {}
        current_expires = self._get_tracked_expiration(sandbox_id, labels)
        if current_expires and current_expires > datetime.now(timezone.utc):
            self._schedule_expiration(sandbox_id, current_expires, update_expiration=False)
            logger.info(
                f"Sandbox {sandbox_id} was renewed "
                f"(expires {current_expires}); aborting expiration."
            )
            return

        mount_keys_raw = labels.get(SANDBOX_OSSFS_MOUNTS_LABEL, "[]")
        try:
            parsed_mount_keys = json.loads(mount_keys_raw)
            if isinstance(parsed_mount_keys, list):
                mount_keys = [key for key in parsed_mount_keys if isinstance(key, str) and key]
        except (TypeError, json.JSONDecodeError):
            mount_keys = []

        try:
            state = container.attrs.get("State", {})
            if state.get("Running", False):
                container.kill()
        except DockerException as exc:
            logger.warning(f"Failed to stop expired sandbox {sandbox_id}: {exc}")

        try:
            container.remove(force=True)
        except DockerException as exc:
            logger.warning(f"Failed to remove expired sandbox {sandbox_id}: {exc}")
            # Re-read ownership on retry: a concurrent DELETE may already
            # have released the mount references captured by this callback.
            self._schedule_expiration(
                sandbox_id, datetime.now(timezone.utc) + timedelta(seconds=30),
                update_expiration=False,
            )
            return

        managed_volumes_raw = labels.get(SANDBOX_MANAGED_VOLUMES_LABEL, "[]")
        try:
            managed_volumes: list[str] = json.loads(managed_volumes_raw)
        except (TypeError, json.JSONDecodeError):
            managed_volumes = []

        self._remove_expiration_tracking(sandbox_id)
        # Ensure sidecar is also cleaned up on expiration
        self._cleanup_egress_sidecar(sandbox_id)
        self._cleanup_windows_oem_volume(sandbox_id, labels)
        self._release_ossfs_mounts(mount_keys)
        self._cleanup_managed_volumes(sandbox_id, managed_volumes)
        self._metadata_store.delete(sandbox_id)

    def _restore_existing_sandboxes(self) -> None:
        """On startup, rebuild expiration timers for containers already running."""
        try:
            containers = self.docker_client.containers.list(all=True)
        except DockerException as exc:
            logger.warning(f"Failed to restore existing sandboxes: {exc}")
            return

        restored = 0
        seen_sidecars: set[str] = set()
        restored_mount_refs: dict[str, int] = {}
        expired_entries: list[tuple[str, list[str]]] = []
        now = datetime.now(timezone.utc)

        def _parse_and_accumulate_mount_refs(labels: dict) -> list[str]:
            mount_keys_raw = labels.get(SANDBOX_OSSFS_MOUNTS_LABEL, "[]")
            try:
                parsed = json.loads(mount_keys_raw)
            except (TypeError, json.JSONDecodeError):
                parsed = []
            keys: list[str] = []
            if isinstance(parsed, list):
                for key in parsed:
                    if isinstance(key, str) and key:
                        keys.append(key)
                        restored_mount_refs[key] = restored_mount_refs.get(key, 0) + 1
            return keys

        # Pass 1: collect ref counts for ALL sandbox containers (alive + expired)
        # and schedule timers for alive ones.  Expired sandboxes are deferred to
        # pass 2 so that ref counts are fully populated before any release.
        for container in containers:
            labels = container.attrs.get("Config", {}).get("Labels") or {}
            sidecar_for = labels.get(EGRESS_SIDECAR_LABEL)
            if sidecar_for:
                seen_sidecars.add(sidecar_for)
                continue

            sandbox_id = labels.get(SANDBOX_ID_LABEL)
            if not sandbox_id:
                continue

            mount_keys = _parse_and_accumulate_mount_refs(labels)

            expires_at = self._get_tracked_expiration(sandbox_id, labels)
            if expires_at is None:
                if self._has_manual_cleanup(labels):
                    restored += 1
                    continue
                logger.warning(
                    f"Sandbox {sandbox_id} missing expires-at label; "
                    "skipping expiration scheduling."
                )
                continue

            if expires_at <= now:
                logger.info(f"Sandbox {sandbox_id} already expired; terminating now.")
                expired_entries.append((sandbox_id, mount_keys))
                continue

            self._schedule_expiration(sandbox_id, expires_at)
            restored += 1

        # Populate ref counts before expiring anything so _release_ossfs_mount
        # can properly decrement and unmount.
        with self._ossfs_mount_lock:
            self._ossfs_mount_ref_counts = restored_mount_refs

        # Pass 2: expire deferred sandboxes (ref counts are now available).
        # Cached mount keys are passed as fallback so that mounts are still
        # released even if the container vanishes between pass 1 and pass 2.
        for sandbox_id, cached_mount_keys in expired_entries:
            self._expire_sandbox(sandbox_id, fallback_mount_keys=cached_mount_keys)

        # Cleanup orphan sidecars (no matching sandbox container)
        for orphan_id in seen_sidecars:
            try:
                self._get_container_by_sandbox_id(orphan_id)
            except HTTPException as exc:
                if exc.status_code == status.HTTP_404_NOT_FOUND:
                    self._cleanup_egress_sidecar(orphan_id)
                else:
                    logger.warning(
                        f"Failed to check sandbox {orphan_id} for orphan sidecar cleanup: {exc}"
                    )

        if restored:
            logger.info(f"Restored expiration timers for {restored} sandbox(es).")

    def _container_to_sandbox(self, container, sandbox_id: Optional[str] = None) -> Sandbox:
        labels = container.attrs.get("Config", {}).get("Labels") or {}
        resolved_id = sandbox_id or labels.get(SANDBOX_ID_LABEL)
        if not resolved_id:
            raise HTTPException(
                status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
                detail={
                    "code": SandboxErrorCodes.SANDBOX_NOT_FOUND,
                    "message": "Container missing sandbox ID label.",
                },
            )

        status_section = container.attrs.get("State", {})
        finished_at = status_section.get("FinishedAt")
        state, reason, message = _container_lifecycle_state(container)

        metadata = self._metadata_store.get(resolved_id, labels)
        entrypoint = container.attrs.get("Config", {}).get("Cmd") or []
        if isinstance(entrypoint, str):
            entrypoint = [entrypoint]
        # The image reference the container was created with. Do not look the
        # image up via container.image: it may have been removed since.
        image_uri = container.attrs.get("Config", {}).get("Image")
        snapshot_id = labels.get(SANDBOX_SNAPSHOT_ID_LABEL) or None
        image_spec = None if snapshot_id else ImageSpec(uri=image_uri)

        created_at = parse_timestamp(
            labels.get(SANDBOX_CREATED_AT_LABEL) or container.attrs.get("Created")
        )
        transitioned_at = self._metadata_store.get_transition_time(resolved_id, state)
        last_transition_at = (
            parse_timestamp(finished_at)
            if finished_at and finished_at != "0001-01-01T00:00:00Z"
            else parse_timestamp(transitioned_at) if transitioned_at else created_at
        )
        expires_at = self._get_tracked_expiration(resolved_id, labels)

        status_info = SandboxStatus(
            state=state,
            reason=reason,
            message=message,
            last_transition_at=last_transition_at,
        )
        platform_spec = self._resolve_platform_for_container(container, labels)

        return Sandbox(
            id=resolved_id,
            image=image_spec,
            snapshotId=snapshot_id,
            platform=platform_spec,
            status=status_info,
            metadata=metadata,
            extensions=extract_extensions_from_mapping(labels),
            entrypoint=entrypoint,
            expiresAt=expires_at,
            createdAt=created_at,
        )

    def _prepare_creation_context(
        self,
        request: CreateSandboxRequest,
    ) -> tuple[str, datetime, Optional[datetime]]:
        sandbox_id = self.generate_sandbox_id()
        created_at = datetime.now(timezone.utc)
        expires_at = None
        if request.timeout is not None:
            expires_at = calculate_expiration_or_raise(created_at, request.timeout)
        return sandbox_id, created_at, expires_at

    async def create_sandbox(self, request: CreateSandboxRequest) -> CreateSandboxResponse:
        if request.lifecycle is not None:
            # renewalPolicy (OSEP-0009) is control-plane state, persisted as a
            # label; hooks and idlePolicy stay unsupported on docker.
            unsupported: list[str] = []
            if request.lifecycle.has_hooks:
                unsupported.append("lifecycle.preStart/periodic hooks")
            if request.lifecycle.idle_policy is not None:
                unsupported.append("lifecycle.idlePolicy")
            if unsupported:
                raise HTTPException(
                    status_code=status.HTTP_400_BAD_REQUEST,
                    detail={
                        "code": SandboxErrorCodes.INVALID_PARAMETER,
                        "message": (
                            f"{', '.join(unsupported)} not supported by the Docker "
                            "provider; only lifecycle.renewalPolicy (OSEP-0009) is "
                            "accepted on the lifecycle object."
                        ),
                    },
                )
        if (request.extensions or {}).get("poolRef", "").strip():
            raise HTTPException(
                status_code=status.HTTP_400_BAD_REQUEST,
                detail={
                    "code": "SANDBOX::UNSUPPORTED_POOL_REF",
                    "message": "poolRef is not supported by the Docker provider. Use Kubernetes BatchSandbox provider instead.",
                },
            )
        request = await resolve_sandbox_image_from_request(request)
        ensure_entrypoint(request.entrypoint or [])
        ensure_metadata_labels(request.metadata)
        ensure_platform_valid(request.platform)
        ensure_timeout_within_limit(
            request.timeout,
            self.app_config.server.max_sandbox_timeout_seconds,
        )
        self._ensure_secure_access_support(request)
        self._ensure_network_policy_support(request)
        resource_limits = self._prepare_resource_limits(request)
        self._validate_volume_definitions(request)
        # Docker API calls block; keep them off the event loop.
        await asyncio.to_thread(self._validate_network_exists)

        try:
            sandbox_env, egress_env = split_egress_env(request.env)
        except ValueError as e:
            raise HTTPException(
                status_code=status.HTTP_400_BAD_REQUEST,
                detail={
                    "code": SandboxErrorCodes.INVALID_PARAMETER,
                    "message": str(e),
                },
            ) from e

        sandbox_id, created_at, expires_at = self._prepare_creation_context(request)
        loop = asyncio.get_running_loop()
        future: asyncio.Future[CreateSandboxResponse] = loop.create_future()

        def _run() -> None:
            auto_created_volumes: list[str] = []
            try:
                # Volume validation calls the Docker API and may create
                # volumes, so it runs here, off the event loop, where a failed
                # create removes them below.
                pvc_inspect_cache, auto_created_volumes = self._validate_volumes(request)
                result = self._provision_sandbox(
                    sandbox_id, request, created_at, expires_at, resource_limits,
                    pvc_inspect_cache, auto_created_volumes,
                    sandbox_env=sandbox_env, egress_env=egress_env,
                )
                loop.call_soon_threadsafe(future.set_result, result)
            except BaseException as exc:
                # _provision_sandbox only removes these when it fails inside its
                # main try block. Earlier failures (the image pull, request
                # checks) would leak them, and delete_sandbox never runs for a
                # sandbox that was never created. Removal is idempotent, and a
                # cleanup error must not keep exc from reaching the caller.
                try:
                    self._cleanup_managed_volumes(sandbox_id, auto_created_volumes)
                except Exception as cleanup_exc:  # noqa: BLE001
                    logger.warning(
                        f"sandbox={sandbox_id} | failed to clean up volumes "
                        f"after create failure: {cleanup_exc}"
                    )
                try:
                    loop.call_soon_threadsafe(future.set_exception, exc)
                except RuntimeError:
                    pass

        Thread(target=_run, daemon=True).start()
        try:
            return await future
        except ValueError as e:
            raise HTTPException(
                status_code=status.HTTP_400_BAD_REQUEST,
                detail={
                    "code": SandboxErrorCodes.INVALID_PARAMETER,
                    "message": str(e),
                },
            ) from e

    def _prepare_resource_limits(
        self, request: CreateSandboxRequest
    ) -> tuple[Optional[int], Optional[int], Optional[int]]:
        """Validate platform resource limits and return outer Docker limits."""
        if is_windows_platform(request.platform):
            validate_windows_resource_limits(
                (request.resource_limits.root if request.resource_limits else None) or {}
            )
            return None, None, None
        return self._resolve_resource_limits(request)

    def _provision_sandbox(
        self,
        sandbox_id: str,
        request: CreateSandboxRequest,
        created_at: datetime,
        expires_at: Optional[datetime],
        resource_limits: tuple[Optional[int], Optional[int], Optional[int]],
        pvc_inspect_cache: Optional[dict[str, dict]] = None,
        auto_created_volumes: Optional[list[str]] = None,
        sandbox_env: Optional[Dict[str, Optional[str]]] = None,
        egress_env: Optional[Dict[str, Optional[str]]] = None,
    ) -> CreateSandboxResponse:
        if sandbox_env is None and egress_env is None:
            sandbox_env, egress_env = split_egress_env(request.env)
        patched_request = request.model_copy(update={"env": sandbox_env or None})
        labels, environment = self._build_labels_and_env(sandbox_id, patched_request, expires_at)
        labels[SANDBOX_CREATED_AT_LABEL] = created_at.isoformat()
        if auto_created_volumes:
            labels[SANDBOX_MANAGED_VOLUMES_LABEL] = json.dumps(
                auto_created_volumes, separators=(",", ":"),
            )
        image_uri, auth_config = self._resolve_image_auth(request, sandbox_id)
        mem_limit, nano_cpus, gpu_count = resource_limits
        egress_token: Optional[str] = None
        requested_windows_profile = is_windows_platform(request.platform)

        credential_proxy_enabled = bool(
            request.credential_proxy and request.credential_proxy.enabled
        )

        if credential_proxy_enabled and egress_env.get(OPENSANDBOX_EGRESS_MITMPROXY_SSL_INSECURE):
            raise ValueError(
                f"'{OPENSANDBOX_EGRESS_MITMPROXY_SSL_INSECURE}' cannot be set when credential proxy is enabled"
            )
        validate_upstream_proxy_request(
            self.app_config.egress,
            has_network_policy=bool(request.network_policy),
            credential_proxy_enabled=credential_proxy_enabled,
            egress_env=egress_env or {},
        )

        if egress_env and not request.network_policy:
            dropped_keys = sorted(egress_env.keys())
            logger.warning(
                f"Sandbox {sandbox_id} has OPENSANDBOX_EGRESS_ env vars {dropped_keys} "
                "but no networkPolicy; these variables will be ignored because no "
                "egress sidecar is created"
            )
            egress_env = {}

        if requested_windows_profile:
            validate_windows_runtime_prerequisites()

        # Prepare OSSFS mounts first so binds can reference mounted host paths.
        ossfs_mount_keys = self._prepare_ossfs_mounts(request.volumes)
        if ossfs_mount_keys:
            labels[SANDBOX_OSSFS_MOUNTS_LABEL] = json.dumps(
                ossfs_mount_keys,
                separators=(",", ":"),
            )

        sidecar_container = None
        reserved_port_bindings: dict[str, tuple[str, int]] = {}
        runtime_volume_name: Optional[str] = None
        try:
            # For dockur/windows profile, resourceLimits are translated to
            # guest envs (RAM_SIZE/CPU_CORES/DISK_SIZE). Avoid applying
            # container cgroup memory/cpu limits to the outer Linux container,
            # which can OOM-kill QEMU during installation/runtime. GPU
            # passthrough is likewise suppressed: the Windows guest runs inside
            # QEMU and would not see GPUs exposed to the outer container.
            effective_mem_limit = None if requested_windows_profile else mem_limit
            effective_nano_cpus = None if requested_windows_profile else nano_cpus
            effective_gpu_count = None if requested_windows_profile else gpu_count

            # pvc_inspect_cache carries Docker volume inspect data from the
            # validation phase, avoiding a redundant API call.
            volume_binds = self._build_volume_binds(request.volumes, pvc_inspect_cache)

            host_config_kwargs: Dict[str, Any]
            exposed_ports: Optional[list[str]] = ["44772", "8080"]
            if requested_windows_profile:
                # dockur/windows exposes RDP and noVNC/web UI on these ports.
                # https://github.com/dockur/windows/blob/master/Dockerfile
                exposed_ports.extend(["3389/tcp", "3389/udp", "8006/tcp"])
            container_exposed_ports: Optional[list[str]] = exposed_ports

            if request.network_policy:
                runtime_volume_name = f"opensandbox-runtime-{sandbox_id}"
                with self._docker_operation(
                    "create egress runtime volume", sandbox_id
                ):
                    self.docker_client.volumes.create(
                        name=runtime_volume_name,
                        labels={SANDBOX_MANAGED_VOLUMES_LABEL: "server"},
                    )
                auto_created_volumes = list(auto_created_volumes or [])
                auto_created_volumes.append(runtime_volume_name)
                labels[SANDBOX_MANAGED_VOLUMES_LABEL] = json.dumps(
                    auto_created_volumes,
                    separators=(",", ":"),
                )
                if credential_proxy_enabled:
                    environment = [
                        entry
                        for entry in environment
                        if not entry.startswith(f"{OPENSANDBOX_EGRESS_MITMPROXY_TRANSPARENT}=")
                    ]
                    environment.append(f"{OPENSANDBOX_EGRESS_MITMPROXY_TRANSPARENT}=true")

                egress_token = generate_egress_token()
                labels[SANDBOX_EGRESS_AUTH_TOKEN_METADATA_KEY] = egress_token
                sidecar_port_bindings = allocate_port_bindings(
                    [*exposed_ports, "18080"],
                    min_port=self.app_config.docker.port_range_min,
                    max_port=self.app_config.docker.port_range_max,
                    publish_host=self.app_config.docker.publish_host,
                )
                reserved_port_bindings = sidecar_port_bindings
                host_execd_port = sidecar_port_bindings["44772"][1]
                host_http_port = sidecar_port_bindings["8080"][1]
                egress_api_binding = sidecar_port_bindings.get("18080")
                extra_sidecar_port_bindings = {
                    port: binding
                    for port, binding in sidecar_port_bindings.items()
                    if port not in {"44772", "8080"}
                }
                sidecar_container = self._start_egress_sidecar(
                    sandbox_id=sandbox_id,
                    network_policy=request.network_policy,
                    egress_token=egress_token,
                    host_execd_port=host_execd_port,
                    host_http_port=host_http_port,
                    extra_port_bindings=extra_sidecar_port_bindings,
                    egress_api_host_port=(
                        egress_api_binding[1] if egress_api_binding is not None else None
                    ),
                    runtime_volume_name=runtime_volume_name,
                    credential_proxy_enabled=credential_proxy_enabled,
                    extra_env=egress_env or None,
                )
                labels[SANDBOX_EMBEDDING_PROXY_PORT_LABEL] = str(host_execd_port)
                labels[SANDBOX_HTTP_PORT_LABEL] = str(host_http_port)
                host_config_kwargs = self._base_host_config_kwargs(
                    effective_mem_limit, effective_nano_cpus, f"container:{sidecar_container.id}",
                    gpu_count=effective_gpu_count,
                )
                # Container network namespace is shared with sidecar. Docker rejects
                # exposing ports on the main container in "container:<id>" mode.
                container_exposed_ports = None
                # Drop NET_ADMIN for the main container; only the sidecar should keep it
                cap_drop = set(host_config_kwargs.get("cap_drop") or [])
                cap_drop.add("NET_ADMIN")
                if cap_drop:
                    host_config_kwargs["cap_drop"] = list(cap_drop)
            else:
                host_config_kwargs = self._base_host_config_kwargs(
                    effective_mem_limit, effective_nano_cpus, self.network_mode,
                    gpu_count=effective_gpu_count,
                )
                if self.network_mode != HOST_NETWORK_MODE:
                    port_bindings = allocate_port_bindings(
                        exposed_ports,
                        min_port=self.app_config.docker.port_range_min,
                        max_port=self.app_config.docker.port_range_max,
                        publish_host=self.app_config.docker.publish_host,
                    )
                    reserved_port_bindings = port_bindings
                    host_execd_port = port_bindings["44772"][1]
                    host_http_port = port_bindings["8080"][1]
                    host_config_kwargs["port_bindings"] = normalize_port_bindings(port_bindings)
                    labels[SANDBOX_EMBEDDING_PROXY_PORT_LABEL] = str(host_execd_port)
                    labels[SANDBOX_HTTP_PORT_LABEL] = str(host_http_port)
                else:
                    exposed_ports = None

            if runtime_volume_name:
                runtime_mount_suffix = f":{OPENSANDBOX_RUNTIME_MOUNT_PATH}:"
                runtime_mount_end = f":{OPENSANDBOX_RUNTIME_MOUNT_PATH}"
                has_runtime_mount = any(
                    runtime_mount_suffix in bind or bind.endswith(runtime_mount_end)
                    for bind in volume_binds
                )
                if not has_runtime_mount:
                    volume_binds.append(
                        f"{runtime_volume_name}:{OPENSANDBOX_RUNTIME_MOUNT_PATH}:rw"
                    )
            # Config-level binds (docker.sandbox_binds) apply to every sandbox
            # and come first.
            all_binds = list(self.app_config.docker.sandbox_binds or []) + (volume_binds or [])
            if all_binds:
                host_config_kwargs["binds"] = all_binds
            if requested_windows_profile:
                host_config_kwargs = apply_windows_runtime_host_config_defaults(
                    host_config_kwargs,
                    sandbox_id,
                )
                environment = inject_windows_resource_limits_env(
                    environment,
                    (request.resource_limits.root if request.resource_limits else None) or {},
                )
                environment = inject_windows_user_ports(environment, exposed_ports)

            # Inject the container permissions required by bwrap isolation.
            # bwrap needs pivot_root/mount, including a fresh procfs, which are blocked
            # by Docker's default AppArmor, seccomp, and protected-system-path settings.
            isolation_requested = (
                (request.extensions or {}).get(BOOTSTRAP_EXECD_ISOLATION_KEY)
                == "enable"
            )
            if isolation_requested:
                cap_add = set(host_config_kwargs.get("cap_add") or [])
                cap_add.add("SYS_ADMIN")
                host_config_kwargs["cap_add"] = sorted(cap_add)
                security_opt = list(host_config_kwargs.get("security_opt") or [])
                security_opt = [s for s in security_opt if not s.startswith(("apparmor=", "seccomp="))]
                security_opt.append("apparmor=unconfined")
                security_opt.append("seccomp=unconfined")
                host_config_kwargs["security_opt"] = security_opt
                tmpfs = dict(host_config_kwargs.get("tmpfs") or {})
                tmpfs[ISOLATION_UPPER_MOUNT_PATH] = ""
                host_config_kwargs["tmpfs"] = tmpfs
                logger.warning(
                    f"sandbox {sandbox_id}: granting CAP_SYS_ADMIN + "
                    "apparmor/seccomp/system paths unconfined + tmpfs for bwrap isolation "
                    "(bootstrap.execd.isolation=enable)"
                )

            if self.network_mode != HOST_NETWORK_MODE and request.network_policy is None:
                for attempt in range(MAX_PORT_PUBLISH_ATTEMPTS):
                    try:
                        created_container = self._create_and_start_container(
                            sandbox_id,
                            image_uri,
                            request.entrypoint,
                            labels,
                            environment,
                            host_config_kwargs,
                            container_exposed_ports,
                            request.platform,
                            unconfine_system_paths=isolation_requested,
                        )
                        break
                    except Exception as exc:
                        if (
                            attempt + 1 < MAX_PORT_PUBLISH_ATTEMPTS
                            and is_port_publish_error(exc)
                        ):
                            logger.warning(
                                f"sandbox {sandbox_id}: container start failed due to "
                                f"host port conflict: {exc}. Retrying with fresh "
                                f"ports (attempt {attempt + 1}/{MAX_PORT_PUBLISH_ATTEMPTS})..."
                            )
                            if reserved_port_bindings:
                                release_port_bindings(reserved_port_bindings)
                                reserved_port_bindings = {}

                            port_bindings = allocate_port_bindings(
                                exposed_ports,
                                min_port=self.app_config.docker.port_range_min,
                                max_port=self.app_config.docker.port_range_max,
                                publish_host=self.app_config.docker.publish_host,
                            )
                            reserved_port_bindings = port_bindings
                            host_execd_port = port_bindings["44772"][1]
                            host_http_port = port_bindings["8080"][1]
                            host_config_kwargs["port_bindings"] = normalize_port_bindings(port_bindings)
                            labels[SANDBOX_EMBEDDING_PROXY_PORT_LABEL] = str(host_execd_port)
                            labels[SANDBOX_HTTP_PORT_LABEL] = str(host_http_port)
                            continue
                        raise
            else:
                created_container = self._create_and_start_container(
                    sandbox_id,
                    image_uri,
                    request.entrypoint,
                    labels,
                    environment,
                    host_config_kwargs,
                    container_exposed_ports,
                    request.platform,
                    unconfine_system_paths=isolation_requested,
                )
        except Exception:
            if sidecar_container is not None:
                try:
                    sidecar_container.remove(force=True)
                except DockerException as cleanup_exc:
                    logger.warning(
                        f"Failed to cleanup egress sidecar for sandbox {sandbox_id}: {cleanup_exc}"
                    )
            self._release_ossfs_mounts(ossfs_mount_keys)
            self._cleanup_managed_volumes(sandbox_id, auto_created_volumes or [])
            raise
        finally:
            if reserved_port_bindings:
                release_port_bindings(reserved_port_bindings)

        status_info = SandboxStatus(
            state="Running",
            reason="CONTAINER_RUNNING",
            message="Sandbox container started successfully.",
            last_transition_at=created_at,
        )

        if expires_at is not None:
            self._schedule_expiration(sandbox_id, expires_at)

        effective_platform = self._resolve_platform_for_container(created_container, labels)
        return CreateSandboxResponse(
            id=sandbox_id,
            status=status_info,
            metadata=request.metadata,
            extensions=extract_extensions_from_mapping(labels),
            platform=effective_platform or request.platform,
            expiresAt=expires_at,
            createdAt=created_at,
            entrypoint=request.entrypoint,
        )

    def list_sandboxes(self, request: ListSandboxesRequest) -> ListSandboxesResponse:
        """List sandboxes with optional filtering and pagination.

        Discovers container IDs via ``docker_client.api.containers`` and then
        inspects each one via ``containers.get(id)``. This gives an explicit
        per-container failure boundary: if a container is removed between the
        list summary and its inspect, that entry is skipped instead of
        failing the whole listing with 500. The high-level
        ``docker_client.containers.list(filters=...)`` inlines the same
        second-stage inspect but does not expose that boundary.
        """
        try:
            summaries = self.docker_client.api.containers(
                all=True,
                filters={"label": [SANDBOX_ID_LABEL]},
                trunc=False,
            )
        except DockerException as exc:
            raise HTTPException(
                status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
                detail={
                    "code": SandboxErrorCodes.CONTAINER_QUERY_FAILED,
                    "message": f"Failed to query sandbox containers: {str(exc)}",
                },
            ) from exc

        sandboxes_by_id: dict[str, Sandbox] = {}
        for summary in summaries:
            container_id = summary.get("Id")
            summary_labels = summary.get("Labels") or {}
            sandbox_id = summary_labels.get(SANDBOX_ID_LABEL)
            if not container_id or not sandbox_id:
                continue
            try:
                container = self.docker_client.containers.get(container_id)
            except DockerNotFound:
                # Removed between list and inspect; skip so concurrent
                # deletions do not fail the whole listing.
                continue
            except DockerException as exc:
                raise HTTPException(
                    status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
                    detail={
                        "code": SandboxErrorCodes.CONTAINER_QUERY_FAILED,
                        "message": f"Failed to query sandbox containers: {str(exc)}",
                    },
                ) from exc
            sandbox_obj = self._container_to_sandbox(container, sandbox_id)
            if matches_filter(sandbox_obj, request.filter):
                sandboxes_by_id[sandbox_id] = sandbox_obj

        sandboxes: list[Sandbox] = list(sandboxes_by_id.values())

        sandboxes.sort(key=lambda s: s.created_at or datetime.min, reverse=True)

        if request.pagination:
            page = request.pagination.page
            page_size = request.pagination.page_size
        else:
            page = 1
            page_size = 20

        total_items = len(sandboxes)
        total_pages = math.ceil(total_items / page_size) if total_items else 0
        start_index = (page - 1) * page_size
        end_index = start_index + page_size
        items = sandboxes[start_index:end_index]
        has_next_page = page < total_pages

        pagination_info = PaginationInfo(
            page=page,
            page_size=page_size,
            total_items=total_items,
            total_pages=total_pages,
            has_next_page=has_next_page,
        )

        return ListSandboxesResponse(items=items, pagination=pagination_info)

    def get_sandbox(self, sandbox_id: str) -> Sandbox:
        container = self._get_container_by_sandbox_id(sandbox_id)
        return self._container_to_sandbox(container, sandbox_id)

    def _raise_if_lifecycle_race(
        self, sandbox_id: str, exc: DockerException, operation: str,
    ) -> None:
        """Map known main-container races using the sequential error envelope."""
        if isinstance(exc, DockerNotFound):
            if operation == "delete":
                self._cleanup_egress_sidecar(sandbox_id)
            raise HTTPException(
                status_code=status.HTTP_404_NOT_FOUND,
                detail={
                    "code": SandboxErrorCodes.SANDBOX_NOT_FOUND,
                    "message": f"Sandbox {sandbox_id} not found.",
                },
            ) from exc
        if not isinstance(exc, APIError) or exc.status_code != status.HTTP_409_CONFLICT:
            return

        message = (exc.explanation or "").lower()
        conflicts = {
            "delete": (
                ("removal of container " in message and "already in progress" in message)
                or "already being removed" in message,
                SandboxErrorCodes.SANDBOX_DELETE_FAILED,
                "Sandbox deletion is already in progress.",
            ),
            "pause": (
                "already paused" in message or "is not running" in message,
                SandboxErrorCodes.SANDBOX_NOT_RUNNING,
                "Sandbox is not in a running state.",
            ),
            "resume": (
                "is not paused" in message or "is not running" in message,
                SandboxErrorCodes.SANDBOX_NOT_PAUSED,
                "Sandbox is not in a paused state.",
            ),
        }
        matches, code, conflict_message = conflicts[operation]
        if not matches:
            return
        if operation != "delete":
            # The state error may have raced with removal; use the usual 404
            # if the sandbox has disappeared since the operation failed.
            self._get_container_by_sandbox_id(sandbox_id)
        raise HTTPException(
            status_code=status.HTTP_409_CONFLICT,
            detail={"code": code, "message": conflict_message},
        ) from exc

    def delete_sandbox(self, sandbox_id: str) -> None:
        with self._sandbox_lifecycle_lock(sandbox_id):
            self._delete_sandbox_locked(sandbox_id)

    def _delete_sandbox_locked(self, sandbox_id: str) -> None:
        try:
            container = self._get_container_by_sandbox_id(sandbox_id)
        except HTTPException as exc:
            if exc.status_code == status.HTTP_404_NOT_FOUND:
                self._cleanup_egress_sidecar(sandbox_id)
            raise
        labels = container.attrs.get("Config", {}).get("Labels") or {}
        mount_keys_raw = labels.get(SANDBOX_OSSFS_MOUNTS_LABEL, "[]")
        try:
            mount_keys: list[str] = json.loads(mount_keys_raw)
        except (TypeError, json.JSONDecodeError):
            mount_keys = []
        managed_volumes_raw = labels.get(SANDBOX_MANAGED_VOLUMES_LABEL, "[]")
        try:
            managed_volumes: list[str] = json.loads(managed_volumes_raw)
        except (TypeError, json.JSONDecodeError):
            managed_volumes = []
        try:
            try:
                with self._docker_operation("kill sandbox container", sandbox_id):
                    container.kill()
            except DockerException as exc:
                # Ignore error if container is already stopped
                if "is not running" not in str(exc).lower():
                    raise
            with self._docker_operation("remove sandbox container", sandbox_id):
                container.remove(force=True)
        except DockerException as exc:
            self._raise_if_lifecycle_race(sandbox_id, exc, "delete")
            raise HTTPException(
                status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
                detail={
                    "code": SandboxErrorCodes.SANDBOX_DELETE_FAILED,
                    "message": f"Failed to delete sandbox container: {str(exc)}",
                },
            ) from exc
        self._remove_expiration_tracking(sandbox_id)
        self._cleanup_egress_sidecar(sandbox_id)
        self._cleanup_windows_oem_volume(sandbox_id, labels)
        self._release_ossfs_mounts(mount_keys)
        self._cleanup_managed_volumes(sandbox_id, managed_volumes)
        self._metadata_store.delete(sandbox_id)

    def pause_sandbox(self, sandbox_id: str) -> None:
        with self._sandbox_lifecycle_lock(sandbox_id):
            self._pause_sandbox_locked(sandbox_id)

    def _pause_sandbox_locked(self, sandbox_id: str) -> None:
        container = self._get_container_by_sandbox_id(sandbox_id)
        state = container.attrs.get("State", {})
        if not state.get("Running", False) or state.get("Paused", False):
            raise HTTPException(
                status_code=status.HTTP_409_CONFLICT,
                detail={
                    "code": SandboxErrorCodes.SANDBOX_NOT_RUNNING,
                    "message": "Sandbox is not in a running state.",
                },
            )

        labels = container.attrs.get("Config", {}).get("Labels") or {}
        egress_expected = bool(labels.get(SANDBOX_EGRESS_AUTH_TOKEN_METADATA_KEY))
        try:
            with self._docker_operation("query egress sidecar", sandbox_id):
                sidecars = self._get_egress_sidecars(sandbox_id)
        except DockerException as exc:
            raise HTTPException(
                status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
                detail={
                    "code": SandboxErrorCodes.SANDBOX_PAUSE_FAILED,
                    "message": f"Failed to query egress sidecar: {str(exc)}",
                },
            ) from exc

        try:
            with self._docker_operation("pause sandbox container", sandbox_id):
                container.pause()
        except DockerException as exc:
            self._raise_if_lifecycle_race(sandbox_id, exc, "pause")
            raise HTTPException(
                status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
                detail={
                    "code": SandboxErrorCodes.SANDBOX_PAUSE_FAILED,
                    "message": f"Failed to pause sandbox container: {str(exc)}",
                },
            ) from exc
        self._persist_transition(sandbox_id, "Paused")

        if egress_expected and not sidecars:
            raise HTTPException(
                status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
                detail={
                    "code": SandboxErrorCodes.SANDBOX_PAUSE_FAILED,
                    "message": (
                        "Sandbox container was paused, but the expected egress sidecar was not found."
                    ),
                },
            )

        paused_sidecars: list[Any] = []
        try:
            for sidecar in sidecars:
                sidecar_state = sidecar.attrs.get("State", {})
                if sidecar_state.get("Paused", False):
                    continue
                if not sidecar_state.get("Running", False):
                    raise DockerException(
                        f"Egress sidecar {sidecar.id} is not in a running state."
                    )
                with self._docker_operation("pause egress sidecar", sandbox_id):
                    sidecar.pause()
                paused_sidecars.append(sidecar)
        except DockerException as exc:
            rollback_errors: list[str] = []
            for paused_sidecar in reversed(paused_sidecars):
                try:
                    with self._docker_operation("rollback egress sidecar pause", sandbox_id):
                        paused_sidecar.unpause()
                except DockerException as rollback_exc:
                    logger.warning(
                        f"sandbox={sandbox_id} | failed to rollback egress "
                        f"sidecar pause: {rollback_exc}"
                    )
                    rollback_errors.append(str(rollback_exc))
            try:
                with self._docker_operation("rollback sandbox pause", sandbox_id):
                    container.unpause()
            except DockerException as rollback_exc:
                logger.warning(
                    f"sandbox={sandbox_id} | failed to rollback sandbox pause: {rollback_exc}"
                )
                rollback_errors.append(str(rollback_exc))
            else:
                self._persist_transition(sandbox_id, "Running")

            message = f"Failed to pause egress sidecar: {str(exc)}"
            if rollback_errors:
                message += f"; rollback failed: {'; '.join(rollback_errors)}"
            raise HTTPException(
                status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
                detail={
                    "code": SandboxErrorCodes.SANDBOX_PAUSE_FAILED,
                    "message": message,
                },
            ) from exc

    def resume_sandbox(self, sandbox_id: str) -> None:
        with self._sandbox_lifecycle_lock(sandbox_id):
            self._resume_sandbox_locked(sandbox_id)

    def _resume_sandbox_locked(self, sandbox_id: str) -> None:
        container = self._get_container_by_sandbox_id(sandbox_id)
        state = container.attrs.get("State", {})
        if not state.get("Paused", False):
            raise HTTPException(
                status_code=status.HTTP_409_CONFLICT,
                detail={
                    "code": SandboxErrorCodes.SANDBOX_NOT_PAUSED,
                    "message": "Sandbox is not in a paused state.",
                },
            )

        labels = container.attrs.get("Config", {}).get("Labels") or {}
        egress_expected = bool(labels.get(SANDBOX_EGRESS_AUTH_TOKEN_METADATA_KEY))
        try:
            with self._docker_operation("query egress sidecar", sandbox_id):
                sidecars = self._get_egress_sidecars(sandbox_id)
        except DockerException as exc:
            raise HTTPException(
                status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
                detail={
                    "code": SandboxErrorCodes.SANDBOX_RESUME_FAILED,
                    "message": f"Failed to query egress sidecar: {str(exc)}",
                },
            ) from exc

        if egress_expected and not sidecars:
            raise HTTPException(
                status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
                detail={
                    "code": SandboxErrorCodes.SANDBOX_RESUME_FAILED,
                    "message": "The expected egress sidecar was not found; sandbox remains paused.",
                },
            )

        resumed_sidecars: list[Any] = []
        resuming_main = False
        try:
            for sidecar in sidecars:
                sidecar_state = sidecar.attrs.get("State", {})
                if not sidecar_state.get("Paused", False):
                    if sidecar_state.get("Running", False):
                        continue
                    raise DockerException(
                        f"Egress sidecar {sidecar.id} is not in a paused state."
                    )
                with self._docker_operation("resume egress sidecar", sandbox_id):
                    sidecar.unpause()
                resumed_sidecars.append(sidecar)

            resuming_main = True
            with self._docker_operation("resume sandbox container", sandbox_id):
                container.unpause()
        except DockerException as exc:
            rollback_errors: list[str] = []
            for resumed_sidecar in reversed(resumed_sidecars):
                try:
                    with self._docker_operation("rollback egress sidecar resume", sandbox_id):
                        resumed_sidecar.pause()
                except DockerException as rollback_exc:
                    logger.warning(
                        f"sandbox={sandbox_id} | failed to rollback egress "
                        f"sidecar resume: {rollback_exc}"
                    )
                    rollback_errors.append(str(rollback_exc))

            failed_component = "sandbox container" if resuming_main else "egress sidecar"
            message = f"Failed to resume {failed_component}: {str(exc)}"
            if rollback_errors:
                message += f"; rollback failed: {'; '.join(rollback_errors)}"
            elif resuming_main:
                self._raise_if_lifecycle_race(sandbox_id, exc, "resume")
            raise HTTPException(
                status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
                detail={
                    "code": SandboxErrorCodes.SANDBOX_RESUME_FAILED,
                    "message": message,
                },
            ) from exc
        self._persist_transition(sandbox_id, "Running")

    def get_access_renew_extend_seconds(self, sandbox_id: str) -> Optional[int]:
        try:
            container = self._get_container_by_sandbox_id(sandbox_id)
        except HTTPException:
            return None
        labels = container.attrs.get("Config", {}).get("Labels") or {}
        raw = labels.get(ACCESS_RENEW_EXTEND_SECONDS_METADATA_KEY)
        if raw is None or not str(raw).strip():
            return None
        try:
            return int(str(raw).strip())
        except ValueError:
            return None

    def renew_expiration(
        self,
        sandbox_id: str,
        request: RenewSandboxExpirationRequest,
    ) -> RenewSandboxExpirationResponse:
        container = self._get_container_by_sandbox_id(sandbox_id)
        new_expiration = ensure_future_expiration(request.expires_at)

        labels = container.attrs.get("Config", {}).get("Labels") or {}
        if self._has_manual_cleanup(labels):
            raise HTTPException(
                status_code=status.HTTP_409_CONFLICT,
                detail={
                    "code": SandboxErrorCodes.INVALID_EXPIRATION,
                    "message": f"Sandbox {sandbox_id} does not have automatic expiration enabled.",
                },
            )
        if self._get_tracked_expiration(sandbox_id, labels) is None:
            raise HTTPException(
                status_code=status.HTTP_409_CONFLICT,
                detail={
                    "code": SandboxErrorCodes.INVALID_EXPIRATION,
                    "message": (
                        f"Sandbox {sandbox_id} is missing expiration metadata and cannot be renewed safely."
                    ),
                },
            )

        # Persist the new timeout in memory; the file-backed override also keeps renewals correct across restarts
        self._schedule_expiration(sandbox_id, new_expiration)
        try:
            self._metadata_store.set_expiration(sandbox_id, new_expiration)
        except OSError as exc:
            logger.warning(f"Failed to persist expiration override for sandbox {sandbox_id}: {exc}")
        labels[SANDBOX_EXPIRES_AT_LABEL] = new_expiration.isoformat()
        try:
            with self._docker_operation("update sandbox labels", sandbox_id):
                self._update_container_labels(container, labels)
        except (DockerException, TypeError) as exc:
            logger.warning(f"Failed to refresh labels for sandbox {sandbox_id}: {exc}")

        return RenewSandboxExpirationResponse(expires_at=new_expiration)

    def patch_sandbox_metadata(self, sandbox_id: str, patch: PatchSandboxMetadataRequest) -> Sandbox:
        """Patch sandbox metadata via JSON Merge Patch (RFC 7396). Docker cannot update labels on running containers, so metadata is persisted to file."""
        from opensandbox_server.services.validators import ensure_metadata_labels

        container = self._get_container_by_sandbox_id(sandbox_id)
        labels = dict(container.attrs.get("Config", {}).get("Labels") or {})

        for key in patch:
            if SandboxService._is_system_label(key):
                raise HTTPException(
                    status_code=400,
                    detail={
                        "code": "INVALID_METADATA_LABEL",
                        "message": f"Metadata key '{key}' is reserved (opensandbox.io/ prefix).",
                    },
                )

        patch_additions = {k: str(v) for k, v in patch.items() if v is not None}
        if patch_additions:
            ensure_metadata_labels(patch_additions)

        self._metadata_store.patch(sandbox_id, labels, patch)
        return self._container_to_sandbox(container, sandbox_id)