Secure, Fast, and Extensible Sandbox runtime for AI agents.
# Copyright 2025 The OpenSandbox Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import asyncio
import pytest
from types import SimpleNamespace
from datetime import datetime, timedelta, timezone
from typing import cast
from unittest.mock import MagicMock, patch
from fastapi import HTTPException
from pydantic import SecretStr
from opensandbox_server.services.k8s.kubernetes_service import KubernetesSandboxService
from opensandbox_server.services.constants import (
OPEN_SANDBOX_EGRESS_AUTH_HEADER,
OPEN_SANDBOX_SECURE_ACCESS_HEADER,
SANDBOX_EGRESS_AUTH_TOKEN_METADATA_KEY,
SANDBOX_SECURE_ACCESS_TOKEN_METADATA_KEY,
SANDBOX_ID_LABEL,
SANDBOX_MANAGED_VOLUMES_LABEL,
SANDBOX_MANUAL_CLEANUP_LABEL,
SandboxErrorCodes,
)
from opensandbox_server.api.schema import (
CredentialProxyConfig,
IdlePolicy,
ImageAuth,
ListSandboxesRequest,
NetworkPolicy,
PlatformSpec,
SandboxLifecycle,
)
from opensandbox_server.config import (
EGRESS_MODE_DNS,
EGRESS_MODE_DNS_NFT,
EgressConfig,
EgressUpstreamProxyConfig,
GatewayConfig,
GatewayRouteModeConfig,
IngressConfig,
SecureAccessConfig,
SecureAccessKey,
)
from opensandbox_server.api.schema import Endpoint
class TestKubernetesSandboxServiceInit:
def test_init_with_valid_config_succeeds(self, k8s_app_config):
with patch('opensandbox_server.services.k8s.kubernetes_service.K8sClient') as mock_k8s_client, \
patch('opensandbox_server.services.k8s.kubernetes_service.create_workload_provider') as mock_create_provider:
mock_provider = MagicMock()
mock_create_provider.return_value = mock_provider
service = KubernetesSandboxService(k8s_app_config)
assert service.namespace == k8s_app_config.kubernetes.namespace
assert service.execd_image == k8s_app_config.runtime.execd_image
mock_k8s_client.assert_called_once_with(k8s_app_config.kubernetes)
mock_create_provider.assert_called_once()
def test_init_without_kubernetes_config_raises_error(self, app_config_no_k8s):
# app_config_no_k8s still has kubernetes config, just without kubeconfig
# This will cause K8sClient initialization to fail and raise HTTPException
with pytest.raises(HTTPException) as exc_info:
KubernetesSandboxService(app_config_no_k8s)
assert exc_info.value.status_code == 503
assert exc_info.value.detail["code"] == SandboxErrorCodes.K8S_INITIALIZATION_ERROR
def test_init_with_wrong_runtime_type_raises_error(self, app_config_docker):
with pytest.raises(ValueError, match="requires runtime.type = 'kubernetes'"):
KubernetesSandboxService(app_config_docker)
def test_init_with_k8s_client_failure_raises_http_exception(self, k8s_app_config):
with patch('opensandbox_server.services.k8s.kubernetes_service.K8sClient') as mock_k8s_client:
mock_k8s_client.side_effect = Exception("Failed to load kubeconfig")
with pytest.raises(HTTPException) as exc_info:
KubernetesSandboxService(k8s_app_config)
assert exc_info.value.status_code == 503
assert "code" in exc_info.value.detail
assert exc_info.value.detail["code"] == SandboxErrorCodes.K8S_INITIALIZATION_ERROR
class TestKubernetesSandboxServiceCreate:
def test_credential_proxy_requires_dns_nft_mode(
self, k8s_service, create_sandbox_request
):
create_sandbox_request.network_policy = NetworkPolicy(default_action="deny", egress=[])
create_sandbox_request.credential_proxy = CredentialProxyConfig(enabled=True)
k8s_service.app_config.egress = EgressConfig(
image="opensandbox/egress:v1.1.7", mode=EGRESS_MODE_DNS
)
with pytest.raises(HTTPException) as exc_info:
k8s_service._ensure_network_policy_support(create_sandbox_request)
assert exc_info.value.status_code == 400
assert exc_info.value.detail["code"] == SandboxErrorCodes.INVALID_PARAMETER
assert "dns+nft" in exc_info.value.detail["message"]
@pytest.mark.asyncio
async def test_create_sandbox_with_valid_request_succeeds(
self, k8s_service, create_sandbox_request, mock_workload
):
k8s_service.workload_provider.create_workload.return_value = {
"name": "test-sandbox-123",
"uid": "abc-123",
}
k8s_service.workload_provider.get_workload.return_value = mock_workload
k8s_service.workload_provider.get_status.return_value = {
"state": "Running",
"reason": "",
"message": "Pod is running",
"last_transition_at": datetime.now(timezone.utc),
}
k8s_service.workload_provider.get_endpoint_info.return_value = "10.244.0.5:8080"
k8s_service.workload_provider.get_expiration.return_value = datetime.now(timezone.utc) + timedelta(hours=1)
response = await k8s_service.create_sandbox(create_sandbox_request)
assert response.id is not None
assert response.status.state == "Running"
k8s_service.workload_provider.create_workload.assert_called_once()
@pytest.mark.asyncio
async def test_create_sandbox_response_restores_extensions_from_workload(
self, k8s_service, create_sandbox_request, mock_workload
):
create_sandbox_request.extensions = {
"opensandbox.extensions.custom-label": "中文数据",
}
mock_workload["metadata"]["annotations"] = {
"opensandbox.io/extensions.custom-label": "中文数据",
}
k8s_service.workload_provider.create_workload.return_value = {
"name": "test-sandbox-123",
"uid": "abc-123",
}
k8s_service.workload_provider.get_workload.return_value = mock_workload
k8s_service.workload_provider.get_status.return_value = {
"state": "Running",
"reason": "",
"message": "Pod is running",
"last_transition_at": datetime.now(timezone.utc),
}
k8s_service.workload_provider.get_expiration.return_value = datetime.now(timezone.utc) + timedelta(hours=1)
response = await k8s_service.create_sandbox(create_sandbox_request)
assert response.extensions == {
"opensandbox.extensions.custom-label": "中文数据",
}
@pytest.mark.asyncio
async def test_create_sandbox_response_ignores_non_dict_workload_annotations(
self, k8s_service, create_sandbox_request
):
workload = SimpleNamespace(
metadata=SimpleNamespace(annotations=["not", "a", "mapping"]),
spec=SimpleNamespace(),
)
k8s_service.workload_provider.create_workload.return_value = {
"name": "test-sandbox-123",
"uid": "abc-123",
}
k8s_service.workload_provider.get_workload.return_value = workload
k8s_service.workload_provider.get_status.return_value = {
"state": "Running",
"reason": "",
"message": "Pod is running",
"last_transition_at": datetime.now(timezone.utc),
}
response = await k8s_service.create_sandbox(create_sandbox_request)
assert response.extensions is None
def test_list_sandboxes_restores_extensions_from_workloads(self, k8s_service, mock_workload):
mock_workload["metadata"]["annotations"] = {
"opensandbox.io/extensions.custom-label": "中文数据",
}
k8s_service.workload_provider.list_workloads.return_value = [mock_workload]
k8s_service.workload_provider.get_status.return_value = {
"state": "Running",
"reason": "",
"message": "Running",
"last_transition_at": datetime.now(timezone.utc),
}
k8s_service.workload_provider.get_expiration.return_value = datetime.now(timezone.utc) + timedelta(hours=1)
from opensandbox_server.api.schema import PaginationRequest
request = ListSandboxesRequest(pagination=PaginationRequest(page=1, page_size=20))
response = k8s_service.list_sandboxes(request)
assert response.items[0].extensions == {
"opensandbox.extensions.custom-label": "中文数据",
}
@pytest.mark.asyncio
async def test_create_sandbox_normalizes_allocated_status_to_running(
self, k8s_service, create_sandbox_request, mock_workload
):
k8s_service.workload_provider.create_workload.return_value = {
"name": "test-sandbox-123",
"uid": "abc-123",
}
k8s_service.workload_provider.get_workload.return_value = mock_workload
k8s_service.workload_provider.get_status.return_value = {
"state": "Allocated",
"reason": "IP_ASSIGNED",
"message": "Pod has IP assigned but not ready",
"last_transition_at": datetime.now(timezone.utc),
}
response = await k8s_service.create_sandbox(create_sandbox_request)
assert response.status.state == "Running"
assert response.status.reason == "IP_ASSIGNED"
assert response.status.message == "Pod has IP assigned and sandbox is ready for requests"
@pytest.mark.asyncio
async def test_create_sandbox_uses_configured_timeout_and_poll_interval(
self, k8s_service, create_sandbox_request, mock_workload
):
k8s_service.workload_provider.create_workload.return_value = {
"name": "test-sandbox-123",
"uid": "abc-123",
}
k8s_service.workload_provider.get_workload.return_value = mock_workload
k8s_service.workload_provider.get_status.return_value = {
"state": "Running",
"reason": "",
"message": "Pod is running",
"last_transition_at": datetime.now(timezone.utc),
}
k8s_service.app_config.kubernetes.sandbox_create_timeout_seconds = 120
k8s_service.app_config.kubernetes.pool_acquisition_timeout_seconds = 15
k8s_service.app_config.kubernetes.sandbox_create_poll_interval_seconds = 0.5
with patch.object(k8s_service, "_wait_for_sandbox_ready", wraps=k8s_service._wait_for_sandbox_ready) as mock_wait:
await k8s_service.create_sandbox(create_sandbox_request)
mock_wait.assert_called_once()
_, kwargs = mock_wait.call_args
assert kwargs["timeout_seconds"] == 120
assert kwargs["pool_acquisition_timeout_seconds"] == 15
assert kwargs["poll_interval_seconds"] == 0.5
@pytest.mark.asyncio
async def test_create_sandbox_cleans_up_workload_after_pool_capacity_timeout(
self, k8s_service, create_sandbox_request
):
create_sandbox_request.extensions = {"poolRef": "*"}
k8s_service.workload_provider.create_workload.return_value = {
"name": "test-sandbox-123",
"uid": "abc-123",
}
capacity_error = HTTPException(
status_code=429,
detail={
"code": SandboxErrorCodes.K8S_POOL_CAPACITY_EXHAUSTED,
"message": "Pool capacity remained unavailable",
},
headers={"Retry-After": "5"},
)
with patch.object(
k8s_service,
"_wait_for_sandbox_ready",
side_effect=capacity_error,
):
with pytest.raises(HTTPException) as exc_info:
await k8s_service.create_sandbox(create_sandbox_request)
assert exc_info.value is capacity_error
k8s_service.workload_provider.delete_workload.assert_called_once()
@pytest.mark.asyncio
async def test_create_sandbox_rejects_image_auth_when_provider_not_supported(
self, k8s_service, create_sandbox_request
):
k8s_service.workload_provider.supports_image_auth.return_value = False
create_sandbox_request.image.auth = ImageAuth(
username="registry-user",
password="registry-pass",
)
with pytest.raises(HTTPException) as exc_info:
await k8s_service.create_sandbox(create_sandbox_request)
assert exc_info.value.status_code == 400
assert exc_info.value.detail["code"] == SandboxErrorCodes.INVALID_PARAMETER
k8s_service.workload_provider.create_workload.assert_not_called()
@pytest.mark.asyncio
async def test_create_sandbox_allows_image_auth_when_provider_supported(
self, k8s_service, create_sandbox_request
):
k8s_service.workload_provider.supports_image_auth.return_value = True
create_sandbox_request.image.auth = ImageAuth(
username="registry-user",
password="registry-pass",
)
k8s_service.workload_provider.create_workload.return_value = {
"name": "test-id", "uid": "uid-1"
}
k8s_service.workload_provider.get_workload.return_value = MagicMock()
k8s_service.workload_provider.get_status.return_value = {
"state": "Running", "reason": "", "message": "",
"last_transition_at": datetime.now(timezone.utc),
}
await k8s_service.create_sandbox(create_sandbox_request)
k8s_service.workload_provider.create_workload.assert_called_once()
@pytest.mark.asyncio
async def test_create_sandbox_with_no_timeout_calls_provider_with_expires_at_none_and_manual_cleanup_label(
self, k8s_service, create_sandbox_request
):
"""When timeout is None (manual cleanup), provider receives expires_at=None and manual-cleanup label."""
create_sandbox_request.timeout = None
k8s_service.workload_provider.create_workload.return_value = {
"name": "test-id", "uid": "uid-1"
}
k8s_service.workload_provider.get_workload.return_value = MagicMock()
k8s_service.workload_provider.get_status.return_value = {
"state": "Running", "reason": "", "message": "",
"last_transition_at": datetime.now(timezone.utc),
}
await k8s_service.create_sandbox(create_sandbox_request)
k8s_service.workload_provider.create_workload.assert_called_once()
_, kwargs = k8s_service.workload_provider.create_workload.call_args
assert kwargs["expires_at"] is None
assert kwargs["labels"].get(SANDBOX_MANUAL_CLEANUP_LABEL) == "true"
@pytest.mark.asyncio
async def test_create_sandbox_with_network_policy_passes_unified_egress_settings(
self, k8s_service, create_sandbox_request
):
create_sandbox_request.network_policy = NetworkPolicy(default_action="deny", egress=[])
create_sandbox_request.env = {
"OPENSANDBOX_EGRESS_LOG_LEVEL": "debug",
"SANDBOX_ENV": "value",
}
k8s_service.app_config.egress = EgressConfig(
image="opensandbox/egress:v1.1.7",
disable_ipv6=False,
requests={"cpu": "25m"},
otlp_endpoint="http://otel-collector.observability:4318",
)
k8s_service.workload_provider.create_workload.return_value = {
"name": "test-id", "uid": "uid-1"
}
k8s_service.workload_provider.get_workload.return_value = MagicMock()
k8s_service.workload_provider.get_status.return_value = {
"state": "Running", "reason": "", "message": "",
"last_transition_at": datetime.now(timezone.utc),
}
with patch(
"opensandbox_server.services.k8s.kubernetes_service.generate_egress_token",
return_value="egress-token",
):
await k8s_service.create_sandbox(create_sandbox_request)
_, kwargs = k8s_service.workload_provider.create_workload.call_args
egress_settings = kwargs["egress_settings"]
assert egress_settings.network_policy is create_sandbox_request.network_policy
assert egress_settings.auth_token == "egress-token"
assert egress_settings.image == "opensandbox/egress:v1.1.7"
assert egress_settings.mode == EGRESS_MODE_DNS
assert egress_settings.disable_ipv6 is False
assert egress_settings.resource_requests == {"cpu": "25m"}
assert egress_settings.resource_limits is None
assert (
egress_settings.otlp_endpoint == "http://otel-collector.observability:4318"
)
assert egress_settings.env == {"OPENSANDBOX_EGRESS_LOG_LEVEL": "debug"}
assert kwargs["env"] == {"SANDBOX_ENV": "value"}
assert "network_policy" not in kwargs
assert kwargs["annotations"][SANDBOX_EGRESS_AUTH_TOKEN_METADATA_KEY] == "egress-token"
def _configure_upstream_proxy(self, k8s_service) -> EgressUpstreamProxyConfig:
upstream_proxy = EgressUpstreamProxyConfig(
url="http://proxy.local:3128",
authorization=SecretStr("Basic dGVzdDp0ZXN0"),
)
k8s_service.app_config.egress = EgressConfig(
image="opensandbox/egress:v1.1.7",
mode=EGRESS_MODE_DNS_NFT,
upstream_proxy=upstream_proxy,
)
return upstream_proxy
async def _create_with_egress(self, k8s_service, create_sandbox_request):
k8s_service.workload_provider.create_workload.return_value = {
"name": "test-id", "uid": "uid-1"
}
k8s_service.workload_provider.get_workload.return_value = MagicMock()
k8s_service.workload_provider.get_status.return_value = {
"state": "Running", "reason": "", "message": "",
"last_transition_at": datetime.now(timezone.utc),
}
with patch(
"opensandbox_server.services.k8s.kubernetes_service.generate_egress_token",
return_value="egress-token",
):
await k8s_service.create_sandbox(create_sandbox_request)
@pytest.mark.asyncio
async def test_create_sandbox_upstream_proxy_with_credential_proxy_succeeds(
self, k8s_service, create_sandbox_request
):
upstream_proxy = self._configure_upstream_proxy(k8s_service)
create_sandbox_request.network_policy = NetworkPolicy(default_action="deny", egress=[])
create_sandbox_request.credential_proxy = CredentialProxyConfig(enabled=True)
await self._create_with_egress(k8s_service, create_sandbox_request)
_, kwargs = k8s_service.workload_provider.create_workload.call_args
assert kwargs["egress_settings"].upstream_proxy is upstream_proxy
@pytest.mark.asyncio
@pytest.mark.parametrize("transparent", ["true", "yes"])
async def test_create_sandbox_upstream_proxy_with_transparent_env_succeeds(
self, k8s_service, create_sandbox_request, transparent
):
self._configure_upstream_proxy(k8s_service)
create_sandbox_request.network_policy = NetworkPolicy(default_action="deny", egress=[])
create_sandbox_request.env = {
"OPENSANDBOX_EGRESS_MITMPROXY_TRANSPARENT": transparent
}
await self._create_with_egress(k8s_service, create_sandbox_request)
_, kwargs = k8s_service.workload_provider.create_workload.call_args
assert kwargs["egress_settings"].upstream_proxy is not None
@pytest.mark.asyncio
@pytest.mark.parametrize("env", [{}, {"OPENSANDBOX_EGRESS_MITMPROXY_TRANSPARENT": "false"}])
async def test_create_sandbox_upstream_proxy_requires_transparent_mitm(
self, k8s_service, create_sandbox_request, env
):
self._configure_upstream_proxy(k8s_service)
create_sandbox_request.network_policy = NetworkPolicy(default_action="deny", egress=[])
create_sandbox_request.env = env
with pytest.raises(HTTPException) as exc_info:
await k8s_service.create_sandbox(create_sandbox_request)
assert exc_info.value.status_code == 400
assert exc_info.value.detail["code"] == SandboxErrorCodes.INVALID_PARAMETER
assert "credentialProxy.enabled" in exc_info.value.detail["message"]
assert "OPENSANDBOX_EGRESS_MITMPROXY_TRANSPARENT" in exc_info.value.detail["message"]
k8s_service.workload_provider.create_workload.assert_not_called()
@pytest.mark.asyncio
async def test_create_sandbox_upstream_proxy_ignores_ssl_insecure_false(
self, k8s_service, create_sandbox_request
):
upstream_proxy = self._configure_upstream_proxy(k8s_service)
create_sandbox_request.network_policy = NetworkPolicy(default_action="deny", egress=[])
create_sandbox_request.env = {
"OPENSANDBOX_EGRESS_MITMPROXY_TRANSPARENT": "true",
"OPENSANDBOX_EGRESS_MITMPROXY_SSL_INSECURE": "false",
}
await self._create_with_egress(k8s_service, create_sandbox_request)
_, kwargs = k8s_service.workload_provider.create_workload.call_args
assert kwargs["egress_settings"].upstream_proxy is upstream_proxy
@pytest.mark.asyncio
async def test_create_sandbox_upstream_proxy_without_network_policy_succeeds(
self, k8s_service, create_sandbox_request
):
self._configure_upstream_proxy(k8s_service)
await self._create_with_egress(k8s_service, create_sandbox_request)
_, kwargs = k8s_service.workload_provider.create_workload.call_args
assert kwargs["egress_settings"] is None
@pytest.mark.asyncio
async def test_create_sandbox_upstream_proxy_rejects_ssl_insecure(
self, k8s_service, create_sandbox_request
):
self._configure_upstream_proxy(k8s_service)
create_sandbox_request.network_policy = NetworkPolicy(default_action="deny", egress=[])
create_sandbox_request.env = {
"OPENSANDBOX_EGRESS_MITMPROXY_TRANSPARENT": "true",
"OPENSANDBOX_EGRESS_MITMPROXY_SSL_INSECURE": "true",
}
with pytest.raises(HTTPException) as exc_info:
await k8s_service.create_sandbox(create_sandbox_request)
assert exc_info.value.status_code == 400
assert exc_info.value.detail["code"] == SandboxErrorCodes.INVALID_PARAMETER
assert "OPENSANDBOX_EGRESS_MITMPROXY_SSL_INSECURE" in exc_info.value.detail["message"]
k8s_service.workload_provider.create_workload.assert_not_called()
@pytest.mark.asyncio
async def test_create_sandbox_with_secure_access_passes_annotations(
self, k8s_service, create_sandbox_request
):
create_sandbox_request.secure_access = True
k8s_service.app_config.ingress = IngressConfig(
mode="gateway",
gateway=GatewayConfig(
address="gateway.example.com",
route=GatewayRouteModeConfig(mode="header"),
),
)
k8s_service.ingress_config = k8s_service.app_config.ingress
k8s_service.workload_provider.create_workload.return_value = {
"name": "test-id", "uid": "uid-1"
}
k8s_service.workload_provider.get_workload.return_value = MagicMock()
k8s_service.workload_provider.get_status.return_value = {
"state": "Running", "reason": "", "message": "",
"last_transition_at": datetime.now(timezone.utc),
}
with patch(
"opensandbox_server.services.k8s.kubernetes_service.generate_secure_access_token",
return_value="secure-token",
):
await k8s_service.create_sandbox(create_sandbox_request)
_, kwargs = k8s_service.workload_provider.create_workload.call_args
assert kwargs["annotations"][SANDBOX_SECURE_ACCESS_TOKEN_METADATA_KEY] == "secure-token"
@pytest.mark.asyncio
async def test_create_sandbox_rejects_secure_access_without_gateway_ingress(
self, k8s_service, create_sandbox_request
):
create_sandbox_request.secure_access = True
k8s_service.app_config.ingress = IngressConfig(mode="direct")
k8s_service.ingress_config = k8s_service.app_config.ingress
with pytest.raises(HTTPException) as exc_info:
await k8s_service.create_sandbox(create_sandbox_request)
assert exc_info.value.status_code == 400
assert exc_info.value.detail["code"] == SandboxErrorCodes.INVALID_PARAMETER
assert "ingress.mode='gateway'" in exc_info.value.detail["message"]
k8s_service.workload_provider.create_workload.assert_not_called()
@pytest.mark.asyncio
async def test_create_sandbox_rejects_idle_policy_on_pod_backend(
self, k8s_service, create_sandbox_request
):
# OSEP-0024: auto-pause/wake is fsb-only; pods must refuse the opt-in
# instead of silently ignoring it.
create_sandbox_request.lifecycle = SandboxLifecycle(
idlePolicy=IdlePolicy(idleTimeoutSeconds=300, onIdle="pause")
)
with pytest.raises(HTTPException) as exc_info:
await k8s_service.create_sandbox(create_sandbox_request)
assert exc_info.value.status_code == 400
assert exc_info.value.detail["code"] == SandboxErrorCodes.INVALID_PARAMETER
assert "idlePolicy" in exc_info.value.detail["message"]
assert "fast-sandbox" in exc_info.value.detail["message"]
k8s_service.workload_provider.create_workload.assert_not_called()
@pytest.mark.asyncio
async def test_create_sandbox_with_network_policy_passes_egress_mode_dns_nft_from_config(
self, k8s_service, create_sandbox_request
):
create_sandbox_request.network_policy = NetworkPolicy(default_action="deny", egress=[])
k8s_service.app_config.egress = EgressConfig(
image="opensandbox/egress:v1.1.7",
mode=EGRESS_MODE_DNS_NFT,
)
k8s_service.workload_provider.create_workload.return_value = {
"name": "test-id", "uid": "uid-1"
}
k8s_service.workload_provider.get_workload.return_value = MagicMock()
k8s_service.workload_provider.get_status.return_value = {
"state": "Running", "reason": "", "message": "",
"last_transition_at": datetime.now(timezone.utc),
}
with patch(
"opensandbox_server.services.k8s.kubernetes_service.generate_egress_token",
return_value="egress-token",
):
await k8s_service.create_sandbox(create_sandbox_request)
_, kwargs = k8s_service.workload_provider.create_workload.call_args
assert kwargs["egress_settings"].mode == EGRESS_MODE_DNS_NFT
@pytest.mark.asyncio
async def test_create_sandbox_passes_platform_to_workload_provider(
self, k8s_service, create_sandbox_request
):
create_sandbox_request.platform = PlatformSpec(os="linux", arch="arm64")
k8s_service.workload_provider.create_workload.return_value = {
"name": "test-id", "uid": "uid-1"
}
k8s_service.workload_provider.get_workload.return_value = {
"spec": {
"template": {
"spec": {
"nodeSelector": {
"kubernetes.io/os": "linux",
"kubernetes.io/arch": "arm64",
}
}
}
}
}
k8s_service.workload_provider.get_status.return_value = {
"state": "Running", "reason": "", "message": "",
"last_transition_at": datetime.now(timezone.utc),
}
response = await k8s_service.create_sandbox(create_sandbox_request)
_, kwargs = k8s_service.workload_provider.create_workload.call_args
assert kwargs["platform"] == create_sandbox_request.platform
assert response.platform is not None
assert response.platform.os == "linux"
assert response.platform.arch == "arm64"
@pytest.mark.asyncio
async def test_create_sandbox_rejects_unsupported_platform(self, k8s_service, create_sandbox_request):
create_sandbox_request.platform = PlatformSpec(os="darwin", arch="arm64")
with pytest.raises(HTTPException) as exc_info:
await k8s_service.create_sandbox(create_sandbox_request)
assert exc_info.value.status_code == 400
assert exc_info.value.detail["code"] == SandboxErrorCodes.INVALID_PARAMETER
k8s_service.workload_provider.create_workload.assert_not_called()
@pytest.mark.asyncio
async def test_create_sandbox_derives_platform_from_node_affinity(
self, k8s_service, create_sandbox_request
):
create_sandbox_request.platform = None
k8s_service.workload_provider.create_workload.return_value = {
"name": "test-id", "uid": "uid-1"
}
k8s_service.workload_provider.get_workload.return_value = {
"spec": {
"template": {
"spec": {
"affinity": {
"nodeAffinity": {
"requiredDuringSchedulingIgnoredDuringExecution": {
"nodeSelectorTerms": [
{
"matchExpressions": [
{
"key": "kubernetes.io/os",
"operator": "In",
"values": ["linux"],
},
{
"key": "kubernetes.io/arch",
"operator": "In",
"values": ["arm64"],
},
]
}
]
}
}
}
}
}
}
}
k8s_service.workload_provider.get_status.return_value = {
"state": "Running", "reason": "", "message": "",
"last_transition_at": datetime.now(timezone.utc),
}
response = await k8s_service.create_sandbox(create_sandbox_request)
assert response.platform is not None
assert response.platform.os == "linux"
assert response.platform.arch == "arm64"
@pytest.mark.asyncio
async def test_create_sandbox_uses_template_platform_constraints(
self, k8s_service, create_sandbox_request
):
create_sandbox_request.platform = PlatformSpec(os="linux", arch="arm64")
k8s_service.workload_provider.create_workload.return_value = {
"name": "test-id", "uid": "uid-1"
}
k8s_service.workload_provider.get_workload.return_value = {
"spec": {
"template": {
"spec": {
"nodeSelector": {
"kubernetes.io/os": "linux",
"kubernetes.io/arch": "arm64",
}
}
}
}
}
k8s_service.workload_provider.get_status.return_value = {
"state": "Running", "reason": "", "message": "",
"last_transition_at": datetime.now(timezone.utc),
}
response = await k8s_service.create_sandbox(create_sandbox_request)
assert response.platform is not None
assert response.platform.os == "linux"
assert response.platform.arch == "arm64"
def test_get_endpoint_resolve_internal_uses_pod_ip_even_in_gateway_mode(
self, k8s_service
):
workload = {"metadata": {"annotations": {}}}
k8s_service.workload_provider.get_workload.return_value = workload
k8s_service.workload_provider.get_internal_endpoint.return_value = Endpoint(
endpoint="10.0.0.1:44772"
)
k8s_service.workload_provider.get_endpoint_info.return_value = Endpoint(
endpoint="gateway.example.com",
headers={"OpenSandbox-Ingress-To": "sbx-123-44772"},
)
endpoint = k8s_service.get_endpoint("sbx-123", 44772, resolve_internal=True)
assert endpoint.endpoint == "10.0.0.1:44772"
assert endpoint.headers is None
k8s_service.workload_provider.get_internal_endpoint.assert_called_once_with(
workload, 44772, "sbx-123"
)
k8s_service.workload_provider.get_endpoint_info.assert_not_called()
def test_get_endpoint_keeps_instance_egress_auth_header_private_for_workload_ports(
self, k8s_service
):
k8s_service.workload_provider.get_workload.return_value = {
"metadata": {
"annotations": {
SANDBOX_EGRESS_AUTH_TOKEN_METADATA_KEY: "egress-token",
}
}
}
k8s_service.workload_provider.get_endpoint_info.return_value = Endpoint(
endpoint="gateway.example.com",
headers={"OpenSandbox-Ingress-To": "sbx-123-44772"},
)
endpoint = k8s_service.get_endpoint("sbx-123", 44772)
assert endpoint.endpoint == "gateway.example.com"
assert endpoint.headers == {
"OpenSandbox-Ingress-To": "sbx-123-44772",
}
def test_get_endpoint_merges_egress_auth_header_for_egress_api_port(
self, k8s_service
):
k8s_service.workload_provider.get_workload.return_value = {
"metadata": {
"annotations": {
SANDBOX_EGRESS_AUTH_TOKEN_METADATA_KEY: "egress-token",
}
}
}
k8s_service.workload_provider.get_endpoint_info.return_value = Endpoint(
endpoint="gateway.example.com",
headers={"OpenSandbox-Ingress-To": "sbx-123-18080"},
)
endpoint = k8s_service.get_endpoint("sbx-123", 18080)
assert endpoint.endpoint == "gateway.example.com"
assert endpoint.headers == {
"OpenSandbox-Ingress-To": "sbx-123-18080",
OPEN_SANDBOX_EGRESS_AUTH_HEADER: "egress-token",
}
def test_get_execd_endpoint_merges_secure_access_header_from_instance_metadata(
self, k8s_service
):
k8s_service.workload_provider.get_workload.return_value = {
"metadata": {
"annotations": {
SANDBOX_SECURE_ACCESS_TOKEN_METADATA_KEY: "secure-token",
}
}
}
k8s_service.workload_provider.get_endpoint_info.return_value = Endpoint(
endpoint="gateway.example.com",
headers={"OpenSandbox-Ingress-To": "sbx-123-44772"},
)
endpoint = k8s_service.get_endpoint("sbx-123", 44772)
assert endpoint.endpoint == "gateway.example.com"
assert endpoint.headers == {
"OpenSandbox-Ingress-To": "sbx-123-44772",
OPEN_SANDBOX_SECURE_ACCESS_HEADER: "secure-token",
}
def test_get_user_endpoint_also_merges_secure_access_header(
self, k8s_service
):
k8s_service.workload_provider.get_workload.return_value = {
"metadata": {
"annotations": {
SANDBOX_SECURE_ACCESS_TOKEN_METADATA_KEY: "secure-token",
}
}
}
k8s_service.workload_provider.get_endpoint_info.return_value = Endpoint(
endpoint="gateway.example.com",
headers={"OpenSandbox-Ingress-To": "sbx-123-8080"},
)
endpoint = k8s_service.get_endpoint("sbx-123", 8080)
assert endpoint.headers == {
"OpenSandbox-Ingress-To": "sbx-123-8080",
OPEN_SANDBOX_SECURE_ACCESS_HEADER: "secure-token",
}
@pytest.mark.asyncio
async def test_create_sandbox_rejects_timeout_above_configured_maximum(
self, k8s_service, create_sandbox_request
):
k8s_service.app_config.server.max_sandbox_timeout_seconds = 3600
create_sandbox_request.timeout = 7200
with pytest.raises(HTTPException) as exc_info:
await k8s_service.create_sandbox(create_sandbox_request)
assert exc_info.value.status_code == 400
assert exc_info.value.detail["code"] == SandboxErrorCodes.INVALID_PARAMETER
assert "configured maximum of 3600s" in exc_info.value.detail["message"]
k8s_service.workload_provider.create_workload.assert_not_called()
@pytest.mark.asyncio
async def test_create_sandbox_pool_mode_rejects_network_policy_with_400(
self, k8s_service
):
from opensandbox_server.api.schema import CreateSandboxRequest
pool_request = CreateSandboxRequest(
extensions={"poolRef": "my-pool"},
networkPolicy=NetworkPolicy(default_action="deny", egress=[]),
)
with pytest.raises(HTTPException) as exc_info:
await k8s_service.create_sandbox(pool_request)
assert exc_info.value.status_code == 400
assert exc_info.value.detail["code"] == SandboxErrorCodes.INVALID_PARAMETER
assert "networkPolicy cannot be used together with extensions.poolRef" in (
exc_info.value.detail["message"]
)
k8s_service.k8s_client.get_custom_object.assert_not_called()
k8s_service.workload_provider.create_workload.assert_not_called()
@pytest.mark.asyncio
async def test_create_sandbox_pool_mode_rejects_credential_proxy_with_400(
self, k8s_service
):
from opensandbox_server.api.schema import CreateSandboxRequest
pool_request = CreateSandboxRequest(
extensions={"poolRef": "my-pool"},
credentialProxy=CredentialProxyConfig(enabled=True),
)
with pytest.raises(HTTPException) as exc_info:
await k8s_service.create_sandbox(pool_request)
assert exc_info.value.status_code == 400
assert exc_info.value.detail["code"] == SandboxErrorCodes.INVALID_PARAMETER
assert "credentialProxy.enabled cannot be used together with extensions.poolRef" in (
exc_info.value.detail["message"]
)
k8s_service.k8s_client.get_custom_object.assert_not_called()
k8s_service.workload_provider.create_workload.assert_not_called()
@pytest.mark.asyncio
async def test_create_sandbox_pool_mode_missing_pool_fails_fast(
self, k8s_service
):
"""Pool mode validates poolRef before creating the BatchSandbox."""
from opensandbox_server.api.schema import CreateSandboxRequest
pool_request = CreateSandboxRequest(
extensions={"poolRef": "does-not-exist-pool"},
)
k8s_service.k8s_client.get_custom_object.return_value = None
with pytest.raises(HTTPException) as exc_info:
await k8s_service.create_sandbox(pool_request)
assert exc_info.value.status_code == 404
assert exc_info.value.detail == {
"code": SandboxErrorCodes.K8S_POOL_NOT_FOUND,
"message": "Pool 'does-not-exist-pool' not found.",
}
k8s_service.k8s_client.get_custom_object.assert_called_once_with(
group="sandbox.opensandbox.io",
version="v1alpha1",
namespace=k8s_service.namespace,
plural="pools",
name="does-not-exist-pool",
)
k8s_service.workload_provider.create_workload.assert_not_called()
k8s_service.workload_provider.delete_workload.assert_not_called()
@pytest.mark.asyncio
async def test_create_sandbox_pool_mode_auto_assign_skips_pool_lookup(
self, k8s_service, mock_workload
):
"""Pool auto-assignment uses '*' and must be handled by the controller."""
from opensandbox_server.api.schema import CreateSandboxRequest
pool_request = CreateSandboxRequest(
extensions={"poolRef": "*"},
)
k8s_service.workload_provider.create_workload.return_value = {
"name": "test-sandbox-pool-auto",
"uid": "pool-auto-123",
}
k8s_service.workload_provider.get_workload.return_value = mock_workload
k8s_service.workload_provider.get_status.return_value = {
"state": "Running",
"reason": "",
"message": "Pod is running",
"last_transition_at": datetime.now(timezone.utc),
}
k8s_service.workload_provider.get_endpoint_info.return_value = "10.244.0.5:8080"
k8s_service.workload_provider.get_expiration.return_value = datetime.now(timezone.utc) + timedelta(hours=1)
response = await k8s_service.create_sandbox(pool_request)
assert response.id is not None
k8s_service.k8s_client.get_custom_object.assert_not_called()
k8s_service.workload_provider.create_workload.assert_called_once()
assert k8s_service.workload_provider.create_workload.call_args.kwargs["extensions"] == {"poolRef": "*"}
@pytest.mark.asyncio
async def test_create_sandbox_pool_mode_skips_image_and_entrypoint_validation(
self, k8s_service, mock_workload
):
"""Pool mode: poolRef only, no image/entrypoint/resourceLimits — should succeed."""
from opensandbox_server.api.schema import CreateSandboxRequest
pool_request = CreateSandboxRequest(
extensions={"poolRef": "my-pool"},
)
k8s_service.workload_provider.create_workload.return_value = {
"name": "test-sandbox-pool",
"uid": "pool-123",
}
k8s_service.workload_provider.get_workload.return_value = mock_workload
k8s_service.workload_provider.get_status.return_value = {
"state": "Running",
"reason": "",
"message": "Pod is running",
"last_transition_at": datetime.now(timezone.utc),
}
k8s_service.workload_provider.get_endpoint_info.return_value = "10.244.0.5:8080"
k8s_service.workload_provider.get_expiration.return_value = datetime.now(timezone.utc) + timedelta(hours=1)
response = await k8s_service.create_sandbox(pool_request)
assert response.id is not None
assert response.status.state == "Running"
k8s_service.workload_provider.create_workload.assert_called_once()
@pytest.mark.asyncio
async def test_create_sandbox_pool_mode_image_auth_guard_no_error(
self, k8s_service, mock_workload
):
"""Pool mode with image=None should not raise AttributeError in _ensure_image_auth_support."""
from opensandbox_server.api.schema import CreateSandboxRequest
pool_request = CreateSandboxRequest(
extensions={"poolRef": "my-pool"},
)
assert pool_request.image is None
k8s_service.workload_provider.create_workload.return_value = {
"name": "test-sandbox-pool2",
"uid": "pool-456",
}
k8s_service.workload_provider.get_workload.return_value = mock_workload
k8s_service.workload_provider.get_status.return_value = {
"state": "Running",
"reason": "",
"message": "Pod is running",
"last_transition_at": datetime.now(timezone.utc),
}
k8s_service.workload_provider.get_endpoint_info.return_value = "10.244.0.6:8080"
k8s_service.workload_provider.get_expiration.return_value = datetime.now(timezone.utc) + timedelta(hours=1)
response = await k8s_service.create_sandbox(pool_request)
assert response.id is not None
class TestWaitForSandboxReady:
def test_pool_capacity_retry_after_matches_acquisition_window(self, k8s_service):
error = k8s_service._pool_capacity_exhausted_error(30)
assert error.headers == {"Retry-After": "30"}
@pytest.mark.asyncio
async def test_wait_for_running_pod_succeeds(self, k8s_service, mock_workload):
k8s_service.workload_provider.get_workload.return_value = mock_workload
k8s_service.workload_provider.get_status.return_value = {
"state": "Running",
"reason": "",
"message": "Pod is running",
"last_transition_at": datetime.now(timezone.utc),
}
result = await k8s_service._wait_for_sandbox_ready("test-sandbox-id", timeout_seconds=10)
assert result == mock_workload
@pytest.mark.asyncio
async def test_wait_for_pending_then_running_succeeds(self, k8s_service, mock_workload):
# Mock state transition: Pending -> Allocated -> Running
status_sequence = [
{"state": "Pending", "reason": "", "message": "Pending", "last_transition_at": datetime.now(timezone.utc)},
{"state": "Allocated", "reason": "IP_ASSIGNED", "message": "IP assigned", "last_transition_at": datetime.now(timezone.utc)},
{"state": "Running", "reason": "", "message": "Running", "last_transition_at": datetime.now(timezone.utc)},
]
k8s_service.workload_provider.get_workload.return_value = mock_workload
k8s_service.workload_provider.get_status.side_effect = status_sequence
result = await k8s_service._wait_for_sandbox_ready("test-sandbox-id", timeout_seconds=10, poll_interval_seconds=0.1)
assert result == mock_workload
assert k8s_service.workload_provider.get_status.call_count == 2
@pytest.mark.asyncio
async def test_wait_for_allocated_pod_returns_immediately(self, k8s_service, mock_workload):
k8s_service.workload_provider.get_workload.return_value = mock_workload
k8s_service.workload_provider.get_status.return_value = {
"state": "Allocated",
"reason": "IP_ASSIGNED",
"message": "Pod has IP assigned",
"last_transition_at": datetime.now(timezone.utc),
}
result = await k8s_service._wait_for_sandbox_ready("test-sandbox-id", timeout_seconds=10)
assert result == mock_workload
@pytest.mark.asyncio
async def test_wait_fails_immediately_for_terminal_pod(self, k8s_service, mock_workload):
k8s_service.workload_provider.subscribe_workload.return_value = None
clock = SimpleNamespace(now=0.0)
async def advance_clock(seconds: float) -> None:
clock.now += seconds
k8s_service.workload_provider.get_workload.return_value = mock_workload
k8s_service.workload_provider.get_status.return_value = {
"state": "Failed",
"reason": "FAILED",
"message": (
"1/1 observed pods failed; primary reason=InitContainerFailed; "
"sample pod=sandbox-0"
),
"last_transition_at": datetime.now(timezone.utc),
}
with (
patch(
"opensandbox_server.services.k8s.kubernetes_service.time.monotonic",
side_effect=lambda: clock.now,
),
patch(
"opensandbox_server.services.k8s.kubernetes_service.asyncio.sleep",
new=advance_clock,
),
pytest.raises(HTTPException) as exc_info,
):
await k8s_service._wait_for_sandbox_ready(
"test-sandbox-id",
timeout_seconds=600,
poll_interval_seconds=1,
)
assert exc_info.value.status_code == 500
detail = cast(dict[str, str], exc_info.value.detail)
assert detail["code"] == SandboxErrorCodes.K8S_POD_FAILED
assert "InitContainerFailed" in detail["message"]
assert k8s_service.workload_provider.get_status.call_count == 1
@pytest.mark.asyncio
async def test_wait_stops_when_sandbox_is_already_expired(self, k8s_service, mock_workload):
k8s_service.workload_provider.get_workload.return_value = mock_workload
k8s_service.workload_provider.get_status.return_value = {
"state": "Terminated",
"reason": "SANDBOX_EXPIRED",
"message": "Sandbox expired",
"last_transition_at": datetime.now(timezone.utc),
}
with pytest.raises(HTTPException) as exc_info:
await k8s_service._wait_for_sandbox_ready(
"test-sandbox-id",
timeout_seconds=600,
poll_interval_seconds=1,
)
assert exc_info.value.status_code == 409
detail = cast(dict[str, str], exc_info.value.detail)
assert "SANDBOX_EXPIRED" in detail["message"]
assert k8s_service.workload_provider.get_status.call_count == 1
@pytest.mark.asyncio
async def test_wait_timeout_raises_exception(self, k8s_service, mock_workload):
k8s_service.workload_provider.get_workload.return_value = mock_workload
k8s_service.workload_provider.get_status.return_value = {
"state": "Pending",
"reason": "",
"message": "Still pending",
"last_transition_at": datetime.now(timezone.utc),
}
with pytest.raises(HTTPException) as exc_info:
await k8s_service._wait_for_sandbox_ready("test-sandbox-id", timeout_seconds=1, poll_interval_seconds=0.5)
assert exc_info.value.status_code == 504 # Gateway Timeout
assert "timeout" in exc_info.value.detail["message"].lower()
@pytest.mark.asyncio
async def test_wait_returns_retryable_error_when_pool_capacity_stays_exhausted(
self, k8s_service, mock_workload
):
k8s_service.workload_provider.get_workload.return_value = mock_workload
k8s_service.workload_provider.get_status.return_value = {
"state": "Pending",
"reason": "POOL_CAPACITY_EXHAUSTED",
"message": "Pool capacity is exhausted",
"last_transition_at": datetime.now(timezone.utc),
}
with pytest.raises(HTTPException) as exc_info:
await k8s_service._wait_for_sandbox_ready(
"test-sandbox-id",
timeout_seconds=1,
poll_interval_seconds=0.01,
pool_acquisition_timeout_seconds=0.02,
)
assert exc_info.value.status_code == 429
assert (
exc_info.value.detail["code"]
== SandboxErrorCodes.K8S_POOL_CAPACITY_EXHAUSTED
)
assert exc_info.value.headers == {"Retry-After": "1"}
@pytest.mark.asyncio
async def test_wait_succeeds_when_pool_capacity_is_released(
self, k8s_service, mock_workload
):
k8s_service.workload_provider.get_workload.return_value = mock_workload
k8s_service.workload_provider.get_status.side_effect = [
{
"state": "Pending",
"reason": "POOL_CAPACITY_EXHAUSTED",
"message": "Pool capacity is exhausted",
"last_transition_at": datetime.now(timezone.utc),
},
{
"state": "Running",
"reason": "RUNNING",
"message": "Sandbox is running",
"last_transition_at": datetime.now(timezone.utc),
},
]
result = await k8s_service._wait_for_sandbox_ready(
"test-sandbox-id",
timeout_seconds=1,
poll_interval_seconds=0.01,
pool_acquisition_timeout_seconds=0.2,
)
assert result == mock_workload
@pytest.mark.asyncio
async def test_overall_timeout_preserves_pool_capacity_error_classification(
self, k8s_service, mock_workload
):
k8s_service.workload_provider.get_workload.return_value = mock_workload
k8s_service.workload_provider.get_status.return_value = {
"state": "Pending",
"reason": "POOL_CAPACITY_EXHAUSTED",
"message": "Pool capacity is exhausted",
"last_transition_at": datetime.now(timezone.utc),
}
with pytest.raises(HTTPException) as exc_info:
await k8s_service._wait_for_sandbox_ready(
"test-sandbox-id",
timeout_seconds=2.0,
poll_interval_seconds=0.05,
pool_acquisition_timeout_seconds=0.2,
)
assert exc_info.value.status_code == 429
assert (
exc_info.value.detail["code"]
== SandboxErrorCodes.K8S_POOL_CAPACITY_EXHAUSTED
)
@pytest.mark.asyncio
async def test_wait_accumulates_pool_capacity_across_transient_recovery(
self, k8s_service, mock_workload
):
k8s_service.workload_provider.subscribe_workload.return_value = None
clock = SimpleNamespace(now=0.0)
async def advance_clock(seconds: float) -> None:
clock.now += seconds
capacity_status = {
"state": "Pending",
"reason": "POOL_CAPACITY_EXHAUSTED",
"message": "Pool capacity is exhausted",
"last_transition_at": datetime.now(timezone.utc),
}
generic_pending_status = {
"state": "Pending",
"reason": "BATCHSANDBOX_PENDING",
"message": "Sandbox is pending",
"last_transition_at": datetime.now(timezone.utc),
}
k8s_service.workload_provider.get_workload.return_value = mock_workload
k8s_service.workload_provider.get_status.side_effect = [
capacity_status,
generic_pending_status,
capacity_status,
generic_pending_status,
]
with (
patch(
"opensandbox_server.services.k8s.kubernetes_service.time.monotonic",
side_effect=lambda: clock.now,
),
patch(
"opensandbox_server.services.k8s.kubernetes_service.asyncio.sleep",
new=advance_clock,
),
pytest.raises(HTTPException) as exc_info,
):
await k8s_service._wait_for_sandbox_ready(
"test-sandbox-id",
timeout_seconds=4,
poll_interval_seconds=1,
pool_acquisition_timeout_seconds=2,
)
assert exc_info.value.status_code == 429
assert (
exc_info.value.detail["code"]
== SandboxErrorCodes.K8S_POOL_CAPACITY_EXHAUSTED
)
@pytest.mark.asyncio
async def test_wait_returns_400_when_scheduler_marks_platform_unschedulable(self, k8s_service, mock_workload):
k8s_service.workload_provider.get_workload.return_value = mock_workload
k8s_service.workload_provider.get_status.return_value = {
"state": "Failed",
"reason": "POD_PLATFORM_UNSCHEDULABLE",
"message": "0/1 nodes are available: 1 node(s) didn't match Pod's node affinity.",
"last_transition_at": datetime.now(timezone.utc),
}
with pytest.raises(HTTPException) as exc_info:
await k8s_service._wait_for_sandbox_ready(
"test-sandbox-id",
timeout_seconds=10,
poll_interval_seconds=0.1,
)
assert exc_info.value.status_code == 400
assert exc_info.value.detail["code"] == SandboxErrorCodes.INVALID_PARAMETER
assert "unschedulable" in exc_info.value.detail["message"].lower()
@pytest.mark.asyncio
async def test_wait_keeps_polling_for_generic_failed_scheduling(self, k8s_service, mock_workload):
status_sequence = [
{
"state": "Pending",
"reason": "FailedScheduling",
"message": "0/1 nodes are available: 1 Insufficient cpu.",
"last_transition_at": datetime.now(timezone.utc),
},
{
"state": "Running",
"reason": "",
"message": "Running",
"last_transition_at": datetime.now(timezone.utc),
},
]
k8s_service.workload_provider.get_workload.return_value = mock_workload
k8s_service.workload_provider.get_status.side_effect = status_sequence
result = await k8s_service._wait_for_sandbox_ready(
"test-sandbox-id",
timeout_seconds=10,
poll_interval_seconds=0.1,
)
assert result == mock_workload
@pytest.mark.asyncio
async def test_wait_fails_fast_when_quota_admission_rejects_pod(self, k8s_service, mock_workload):
"""ResourceQuota admission rejection surfaces as 403 QUOTA_EXCEEDED instead of blind-waiting to timeout."""
k8s_service.workload_provider.get_workload.return_value = mock_workload
k8s_service.workload_provider.get_status.return_value = {
"state": "Pending",
"reason": "ReconcilerError",
"message": (
'Error seen: pods "test-sandbox-id" is forbidden: exceeded quota: '
"sandbox-quota, requested: requests.cpu=1, used: requests.cpu=0, "
"limited: requests.cpu=10m"
),
"last_transition_at": datetime.now(timezone.utc),
}
with pytest.raises(HTTPException) as exc_info:
await k8s_service._wait_for_sandbox_ready(
"test-sandbox-id",
timeout_seconds=10,
poll_interval_seconds=0.1,
)
assert exc_info.value.status_code == 403
assert exc_info.value.detail["code"] == SandboxErrorCodes.K8S_QUOTA_EXCEEDED
assert "exceeded quota" in exc_info.value.detail["message"]
@pytest.mark.asyncio
async def test_wait_keeps_polling_for_non_quota_reconciler_error(self, k8s_service, mock_workload):
"""ReconcilerError without quota text (transient controller error) must keep polling."""
status_sequence = [
{
"state": "Pending",
"reason": "ReconcilerError",
"message": "Error seen: unable to create pod: transient API server hiccup",
"last_transition_at": datetime.now(timezone.utc),
},
{
"state": "Running",
"reason": "",
"message": "Running",
"last_transition_at": datetime.now(timezone.utc),
},
]
k8s_service.workload_provider.get_workload.return_value = mock_workload
k8s_service.workload_provider.get_status.side_effect = status_sequence
result = await k8s_service._wait_for_sandbox_ready(
"test-sandbox-id",
timeout_seconds=10,
poll_interval_seconds=0.1,
)
assert result == mock_workload
@pytest.mark.asyncio
async def test_create_sandbox_quota_403_fails_fast_with_quota_error(
self, k8s_service, create_sandbox_request
):
"""Sync count-quota 403 on CR create maps to 403 QUOTA_EXCEEDED (not 500), rollback preserved."""
from kubernetes.client import ApiException
api_exc = ApiException(status=403, reason="Forbidden")
api_exc.body = (
'{"kind":"Status","status":"Failure","message":'
'"sandboxes.agents.x-k8s.io \\"test-sandbox\\" is forbidden: '
'exceeded quota: sandbox-quota, requested: count/sandboxes.agents.x-k8s.io=1, '
'used: 10, limited: 10"}'
)
k8s_service.workload_provider.create_workload.side_effect = api_exc
with pytest.raises(HTTPException) as exc_info:
await k8s_service.create_sandbox(create_sandbox_request)
assert exc_info.value.status_code == 403
assert exc_info.value.detail["code"] == SandboxErrorCodes.K8S_QUOTA_EXCEEDED
assert "exceeded quota" in exc_info.value.detail["message"]
k8s_service.workload_provider.delete_workload.assert_called_once()
class TestKubernetesSandboxServiceRenew:
def test_renew_expiration_rejects_manual_cleanup_sandbox(self, k8s_service):
k8s_service.workload_provider.get_workload.return_value = MagicMock()
k8s_service.workload_provider.get_expiration.return_value = None
request = MagicMock(expires_at=datetime.now(timezone.utc) + timedelta(hours=1))
with pytest.raises(HTTPException) as exc_info:
k8s_service.renew_expiration("test-sandbox-id", request)
assert exc_info.value.status_code == 409
assert exc_info.value.detail["code"] == SandboxErrorCodes.INVALID_EXPIRATION
assert (
exc_info.value.detail["message"]
== "Sandbox test-sandbox-id does not have automatic expiration enabled."
)
class TestGetSandbox:
def test_get_existing_sandbox_succeeds(self, k8s_service, mock_workload):
mock_workload["spec"] = {
"template": {
"spec": {
"nodeSelector": {
"kubernetes.io/os": "linux",
"kubernetes.io/arch": "amd64",
}
}
}
}
k8s_service.workload_provider.get_workload.return_value = mock_workload
k8s_service.workload_provider.get_status.return_value = {
"state": "Running",
"reason": "",
"message": "Running",
"last_transition_at": datetime.now(timezone.utc),
}
k8s_service.workload_provider.get_endpoint_info.return_value = "10.0.0.1:8080"
k8s_service.workload_provider.get_expiration.return_value = datetime.now(timezone.utc) + timedelta(hours=1)
sandbox = k8s_service.get_sandbox("test-sandbox-123")
assert sandbox.id == "test-sandbox-123"
assert sandbox.status.state == "Running"
assert sandbox.platform is not None
assert sandbox.platform.os == "linux"
assert sandbox.platform.arch == "amd64"
def test_get_nonexistent_sandbox_raises_404(self, k8s_service):
k8s_service.workload_provider.get_workload.return_value = None
with pytest.raises(HTTPException) as exc_info:
k8s_service.get_sandbox("nonexistent-id")
assert exc_info.value.status_code == 404
assert "not found" in exc_info.value.detail["message"].lower()
def test_get_sandbox_extracts_platform_from_affinity(self, k8s_service, mock_workload):
mock_workload["spec"] = {
"template": {
"spec": {
"affinity": {
"nodeAffinity": {
"requiredDuringSchedulingIgnoredDuringExecution": {
"nodeSelectorTerms": [
{
"matchExpressions": [
{
"key": "kubernetes.io/os",
"operator": "In",
"values": ["linux"],
},
{
"key": "kubernetes.io/arch",
"operator": "In",
"values": ["amd64"],
},
]
}
]
}
}
}
}
}
}
k8s_service.workload_provider.get_workload.return_value = mock_workload
k8s_service.workload_provider.get_status.return_value = {
"state": "Running",
"reason": "",
"message": "Running",
"last_transition_at": datetime.now(timezone.utc),
}
k8s_service.workload_provider.get_expiration.return_value = datetime.now(timezone.utc) + timedelta(hours=1)
sandbox = k8s_service.get_sandbox("test-sandbox-123")
assert sandbox.platform is not None
assert sandbox.platform.os == "linux"
assert sandbox.platform.arch == "amd64"
def test_get_sandbox_uses_template_platform_constraints(self, k8s_service, mock_workload):
mock_workload["spec"] = {
"template": {
"spec": {
"nodeSelector": {
"kubernetes.io/os": "linux",
"kubernetes.io/arch": "arm64",
}
}
}
}
k8s_service.workload_provider.get_workload.return_value = mock_workload
k8s_service.workload_provider.get_status.return_value = {
"state": "Running",
"reason": "",
"message": "Running",
"last_transition_at": datetime.now(timezone.utc),
}
k8s_service.workload_provider.get_expiration.return_value = datetime.now(timezone.utc) + timedelta(hours=1)
sandbox = k8s_service.get_sandbox("test-sandbox-123")
assert sandbox.platform is not None
assert sandbox.platform.os == "linux"
assert sandbox.platform.arch == "arm64"
def test_get_sandbox_merges_selector_and_affinity_platform_constraints(self, k8s_service, mock_workload):
mock_workload["spec"] = {
"template": {
"spec": {
"nodeSelector": {
"kubernetes.io/os": "linux",
},
"affinity": {
"nodeAffinity": {
"requiredDuringSchedulingIgnoredDuringExecution": {
"nodeSelectorTerms": [
{
"matchExpressions": [
{
"key": "kubernetes.io/arch",
"operator": "In",
"values": ["arm64"],
}
]
}
]
}
}
},
}
}
}
k8s_service.workload_provider.get_workload.return_value = mock_workload
k8s_service.workload_provider.get_status.return_value = {
"state": "Running",
"reason": "",
"message": "Running",
"last_transition_at": datetime.now(timezone.utc),
}
k8s_service.workload_provider.get_expiration.return_value = datetime.now(timezone.utc) + timedelta(hours=1)
sandbox = k8s_service.get_sandbox("test-sandbox-123")
assert sandbox.platform is not None
assert sandbox.platform.os == "linux"
assert sandbox.platform.arch == "arm64"
def test_get_sandbox_returns_null_platform_for_default_scheduling(self, k8s_service, mock_workload):
mock_workload["spec"] = {
"template": {
"spec": {
# no nodeSelector/nodeAffinity constraints
}
}
}
k8s_service.workload_provider.get_workload.return_value = mock_workload
k8s_service.workload_provider.get_status.return_value = {
"state": "Running",
"reason": "",
"message": "Running",
"last_transition_at": datetime.now(timezone.utc),
}
k8s_service.workload_provider.get_expiration.return_value = datetime.now(timezone.utc) + timedelta(hours=1)
sandbox = k8s_service.get_sandbox("test-sandbox-123")
assert sandbox.platform is None
class TestDeleteSandbox:
def test_delete_existing_sandbox_succeeds(self, k8s_service, mock_workload):
k8s_service.workload_provider.get_workload.return_value = mock_workload
k8s_service.workload_provider.delete_workload.return_value = None
k8s_service.delete_sandbox("test-sandbox-id")
k8s_service.workload_provider.delete_workload.assert_called_once_with(
sandbox_id="test-sandbox-id",
namespace=k8s_service.namespace
)
def test_delete_nonexistent_sandbox_raises_404(self, k8s_service):
k8s_service.workload_provider.delete_workload.side_effect = Exception("Sandbox not found")
with pytest.raises(HTTPException) as exc_info:
k8s_service.delete_sandbox("nonexistent-id")
assert exc_info.value.status_code == 404
def test_delete_sandbox_removes_managed_pvcs(self, k8s_service, mock_workload):
k8s_service.workload_provider.get_workload.return_value = mock_workload
k8s_service.workload_provider.delete_workload.return_value = None
pvc = MagicMock()
pvc.metadata = MagicMock(name="metadata")
pvc.metadata.name = "auto-data"
k8s_service.k8s_client.list_pvcs.return_value = [pvc]
k8s_service.delete_sandbox("test-sandbox-id")
selector = (
f"{SANDBOX_MANAGED_VOLUMES_LABEL}=server,"
f"{SANDBOX_ID_LABEL}=test-sandbox-id"
)
k8s_service.k8s_client.list_pvcs.assert_called_once_with(
k8s_service.namespace,
label_selector=selector,
)
k8s_service.k8s_client.delete_pvc.assert_called_once_with(
k8s_service.namespace,
"auto-data",
)
def test_delete_sandbox_skips_pvc_cleanup_on_transient_error(self, k8s_service):
# A non-404 delete failure must NOT trigger PVC deletion: the workload
# is still using them.
k8s_service.workload_provider.delete_workload.side_effect = Exception(
"kubernetes api server unavailable"
)
with pytest.raises(HTTPException):
k8s_service.delete_sandbox("flaky-id")
k8s_service.k8s_client.list_pvcs.assert_not_called()
k8s_service.k8s_client.delete_pvc.assert_not_called()
def test_delete_sandbox_sweeps_orphaned_pvcs_on_404(self, k8s_service):
# If the workload is already gone, we still sweep orphaned managed PVCs
# so a retry after a partial cleanup heals state.
k8s_service.workload_provider.delete_workload.side_effect = Exception(
"Sandbox not found"
)
pvc = MagicMock()
pvc.metadata = MagicMock(name="metadata")
pvc.metadata.name = "orphan-pvc"
k8s_service.k8s_client.list_pvcs.return_value = [pvc]
with pytest.raises(HTTPException) as exc_info:
k8s_service.delete_sandbox("ghost-id")
assert exc_info.value.status_code == 404
k8s_service.k8s_client.delete_pvc.assert_called_once_with(
k8s_service.namespace,
"orphan-pvc",
)
def test_delete_sandbox_pvc_cleanup_is_best_effort(self, k8s_service, mock_workload):
# list_pvcs failure (e.g. 403) must not break sandbox deletion.
k8s_service.workload_provider.get_workload.return_value = mock_workload
k8s_service.workload_provider.delete_workload.return_value = None
k8s_service.k8s_client.list_pvcs.side_effect = Exception("forbidden")
# Should not raise — workload deletion already succeeded.
k8s_service.delete_sandbox("test-sandbox-id")
k8s_service.k8s_client.delete_pvc.assert_not_called()
class TestEnsurePvcVolumes:
"""_ensure_pvc_volumes labels auto-created PVCs for later cleanup."""
def _make_volume(
self,
claim_name: str,
*,
create_if_not_exists: bool = True,
delete_on_sandbox_termination: bool = True,
):
vol = MagicMock()
vol.pvc = MagicMock()
vol.pvc.claim_name = claim_name
vol.pvc.create_if_not_exists = create_if_not_exists
vol.pvc.delete_on_sandbox_termination = delete_on_sandbox_termination
vol.pvc.storage = None
vol.pvc.access_modes = None
vol.pvc.storage_class = None
return vol
def test_opt_in_pvc_carries_managed_and_id_labels(self, k8s_service):
# deleteOnSandboxTermination=true: PVC is labeled so cleanup will sweep it.
k8s_service.k8s_client.get_pvc.return_value = None # doesn't exist
k8s_service.k8s_client.create_pvc.return_value = None
k8s_service._ensure_pvc_volumes(
[self._make_volume("data-claim", delete_on_sandbox_termination=True)],
sandbox_id="sandbox-xyz",
)
k8s_service.k8s_client.create_pvc.assert_called_once()
_ns, body = k8s_service.k8s_client.create_pvc.call_args.args
labels = body.metadata.labels or {}
assert labels.get(SANDBOX_MANAGED_VOLUMES_LABEL) == "server"
assert labels.get(SANDBOX_ID_LABEL) == "sandbox-xyz"
def test_opt_out_pvc_is_not_labeled_for_cleanup(self, k8s_service):
# deleteOnSandboxTermination=false (default): PVC is created but not
# labeled, so _cleanup_managed_pvcs will leave it alone.
k8s_service.k8s_client.get_pvc.return_value = None
k8s_service.k8s_client.create_pvc.return_value = None
k8s_service._ensure_pvc_volumes(
[self._make_volume("durable-claim", delete_on_sandbox_termination=False)],
sandbox_id="sandbox-xyz",
)
k8s_service.k8s_client.create_pvc.assert_called_once()
_ns, body = k8s_service.k8s_client.create_pvc.call_args.args
labels = body.metadata.labels or {}
assert SANDBOX_MANAGED_VOLUMES_LABEL not in labels
assert SANDBOX_ID_LABEL not in labels
def test_existing_pvc_is_not_labeled_or_modified(self, k8s_service):
# Pre-existing PVC: get returns non-None, create must not be called.
existing = MagicMock()
existing.metadata.labels = {} # unlabeled = user-managed, allow reuse
k8s_service.k8s_client.get_pvc.return_value = existing
k8s_service._ensure_pvc_volumes(
[self._make_volume("existing-claim")],
sandbox_id="sandbox-xyz",
)
k8s_service.k8s_client.create_pvc.assert_not_called()
def test_existing_pvc_managed_by_other_sandbox_is_rejected_409(self, k8s_service):
# If a PVC is already labeled as managed by another sandbox, allowing
# this sandbox to mount it would let the owner's cleanup (label sweep
# or ownerReference GC on CR delete/TTL) delete the volume out from
# under us. Reject 409 instead.
from opensandbox_server.services.constants import (
SANDBOX_ID_LABEL,
SANDBOX_MANAGED_VOLUMES_LABEL,
)
existing = MagicMock()
existing.metadata.labels = {
SANDBOX_MANAGED_VOLUMES_LABEL: "server",
SANDBOX_ID_LABEL: "other-sandbox",
}
k8s_service.k8s_client.get_pvc.return_value = existing
with pytest.raises(HTTPException) as exc_info:
k8s_service._ensure_pvc_volumes(
[self._make_volume("shared-claim", delete_on_sandbox_termination=True)],
sandbox_id="sandbox-xyz",
)
assert exc_info.value.status_code == 409
k8s_service.k8s_client.create_pvc.assert_not_called()
def test_existing_pvc_owned_by_other_is_rejected_even_without_create_if_not_exists(
self, k8s_service
):
# ``createIfNotExists=false`` mounts a user-named PVC. If that PVC
# happens to be labeled as managed by another sandbox, the new
# sandbox would still be exposed to the owner's cleanup. The pre-pass
# must catch this for ALL PVC mounts, not just create-if-not-exists.
from opensandbox_server.services.constants import (
SANDBOX_ID_LABEL,
SANDBOX_MANAGED_VOLUMES_LABEL,
)
existing = MagicMock()
existing.metadata.labels = {
SANDBOX_MANAGED_VOLUMES_LABEL: "server",
SANDBOX_ID_LABEL: "other-sandbox",
}
k8s_service.k8s_client.get_pvc.return_value = existing
with pytest.raises(HTTPException) as exc_info:
k8s_service._ensure_pvc_volumes(
[self._make_volume("borrowed-claim", create_if_not_exists=False)],
sandbox_id="sandbox-xyz",
)
assert exc_info.value.status_code == 409
k8s_service.k8s_client.create_pvc.assert_not_called()
def test_pre_pass_get_pvc_403_fails_closed(self, k8s_service):
# Without 'get' on PVCs the ownership pre-pass cannot verify whether
# the claim is already labeled by another sandbox. Silently skipping
# would let a request with createIfNotExists=false mount someone
# else's managed PVC, exposing it to the owner's cleanup. Refuse
# with 500 — RBAC misconfig is a persistent server-side error, not
# something retrying will fix.
from kubernetes.client import ApiException
k8s_service.k8s_client.get_pvc.side_effect = ApiException(
status=403, reason="Forbidden"
)
with pytest.raises(HTTPException) as exc_info:
k8s_service._ensure_pvc_volumes(
[self._make_volume("unknown-owner", create_if_not_exists=False)],
sandbox_id="sandbox-xyz",
)
assert exc_info.value.status_code == 500
k8s_service.k8s_client.create_pvc.assert_not_called()
def test_pvc_create_server_error_returns_internal_error(self, k8s_service):
"""Unexpected PVC create errors must stay structured HTTP failures."""
from kubernetes.client import ApiException
k8s_service.k8s_client.get_pvc.return_value = None
k8s_service.k8s_client.create_pvc.side_effect = ApiException(
status=500, reason="Internal Server Error"
)
with pytest.raises(HTTPException) as exc_info:
k8s_service._ensure_pvc_volumes(
[self._make_volume("data-claim")], sandbox_id="sandbox-xyz"
)
assert exc_info.value.status_code == 500
assert exc_info.value.detail == {
"code": SandboxErrorCodes.INTERNAL_ERROR,
"message": "Failed to auto-create PVC 'data-claim': Internal Server Error",
}
def test_create_race_409_fails_closed_when_winner_pvc_already_gone(
self, k8s_service
):
# Winner created the PVC, our create raced and 409'd, then the
# winner's own failure cleanup deleted the PVC before our re-fetch
# could observe it. Proceeding would let the workload reference a
# missing claim and fail at pod scheduling; raise 503 so the caller
# retries and their fresh pre-pass sees the absent PVC.
from kubernetes.client import ApiException
# Pre-pass sees absent; create_pvc loses race with 409; re-fetch returns None.
k8s_service.k8s_client.get_pvc.side_effect = [None, None]
k8s_service.k8s_client.create_pvc.side_effect = ApiException(status=409, reason="AlreadyExists")
with pytest.raises(HTTPException) as exc_info:
k8s_service._ensure_pvc_volumes(
[self._make_volume("contested-claim", delete_on_sandbox_termination=True)],
sandbox_id="sandbox-b",
)
assert exc_info.value.status_code == 503
def test_create_race_409_fails_closed_when_re_fetch_fails(self, k8s_service):
# If the post-409 ownership re-check itself fails (transient/403),
# we cannot confirm the winner didn't label the PVC for another
# sandbox. Silently proceeding could mount hostile storage; raise
# 503 instead so the caller can retry.
from kubernetes.client import ApiException
# Pre-pass sees absent; create_pvc loses race with 409; re-fetch fails.
k8s_service.k8s_client.get_pvc.side_effect = [None, Exception("network blip")]
k8s_service.k8s_client.create_pvc.side_effect = ApiException(status=409, reason="AlreadyExists")
with pytest.raises(HTTPException) as exc_info:
k8s_service._ensure_pvc_volumes(
[self._make_volume("contested-claim", delete_on_sandbox_termination=True)],
sandbox_id="sandbox-b",
)
assert exc_info.value.status_code == 503
def test_create_race_409_re_checks_ownership_and_rejects_other_winner(
self, k8s_service
):
# Two concurrent creates: both see the PVC absent, A wins the create
# and labels it for sandbox A, B's create raises 409. B must re-fetch
# and apply the same managed-by-other guard, otherwise B silently
# mounts A's labeled PVC and A's cleanup will reap it under B.
from kubernetes.client import ApiException
from opensandbox_server.services.constants import (
SANDBOX_ID_LABEL,
SANDBOX_MANAGED_VOLUMES_LABEL,
)
racer = MagicMock()
racer.metadata.labels = {
SANDBOX_MANAGED_VOLUMES_LABEL: "server",
SANDBOX_ID_LABEL: "sandbox-a",
}
# First get_pvc (pre-pass) sees nothing; second (post-409) sees A's labeled PVC.
k8s_service.k8s_client.get_pvc.side_effect = [None, racer]
k8s_service.k8s_client.create_pvc.side_effect = ApiException(status=409, reason="AlreadyExists")
with pytest.raises(HTTPException) as exc_info:
k8s_service._ensure_pvc_volumes(
[self._make_volume("contested-claim", delete_on_sandbox_termination=True)],
sandbox_id="sandbox-b",
)
assert exc_info.value.status_code == 409
def test_existing_pvc_labeled_by_same_sandbox_is_allowed(self, k8s_service):
# Idempotent retry: same sandbox_id label on the existing PVC means
# the previous create_sandbox attempt got partway and the user is
# retrying. Don't 409 ourselves.
from opensandbox_server.services.constants import (
SANDBOX_ID_LABEL,
SANDBOX_MANAGED_VOLUMES_LABEL,
)
existing = MagicMock()
existing.metadata.labels = {
SANDBOX_MANAGED_VOLUMES_LABEL: "server",
SANDBOX_ID_LABEL: "sandbox-xyz",
}
k8s_service.k8s_client.get_pvc.return_value = existing
k8s_service._ensure_pvc_volumes(
[self._make_volume("retried-claim", delete_on_sandbox_termination=True)],
sandbox_id="sandbox-xyz",
)
k8s_service.k8s_client.create_pvc.assert_not_called()
def test_conflicting_delete_flag_is_rejected_400(self, k8s_service):
# Same claim mounted twice with different delete flags must fail
# 400 before any side effects: otherwise the first occurrence wins
# and either leaks (opt-in then opt-out) or unexpectedly deletes
# (opt-out then opt-in).
with pytest.raises(HTTPException) as exc_info:
k8s_service._ensure_pvc_volumes(
[
self._make_volume("shared", delete_on_sandbox_termination=True),
self._make_volume("shared", delete_on_sandbox_termination=False),
],
sandbox_id="sandbox-xyz",
)
assert exc_info.value.status_code == 400
k8s_service.k8s_client.create_pvc.assert_not_called()
def test_conflicting_create_flag_is_rejected_400(self, k8s_service):
with pytest.raises(HTTPException) as exc_info:
k8s_service._ensure_pvc_volumes(
[
self._make_volume("shared", create_if_not_exists=True),
self._make_volume("shared", create_if_not_exists=False),
],
sandbox_id="sandbox-xyz",
)
assert exc_info.value.status_code == 400
k8s_service.k8s_client.create_pvc.assert_not_called()
def test_duplicate_claim_with_matching_flags_is_allowed(self, k8s_service):
# Mounting the same claim at multiple paths is legitimate as long as
# the provisioning flags agree.
k8s_service.k8s_client.get_pvc.return_value = None
k8s_service.k8s_client.create_pvc.return_value = None
k8s_service._ensure_pvc_volumes(
[
self._make_volume("shared", delete_on_sandbox_termination=True),
self._make_volume("shared", delete_on_sandbox_termination=True),
],
sandbox_id="sandbox-xyz",
)
# Only one PVC is created despite two mounts (dedup by claim).
k8s_service.k8s_client.create_pvc.assert_called_once()
class TestCreateSandboxPvcFailureCleanup:
"""Auto-created PVCs are swept when sandbox creation fails before returning."""
def _request_with_pvc(self):
from opensandbox_server.api.schema import (
CreateSandboxRequest,
ImageSpec,
PVC,
ResourceLimits,
Volume,
)
return CreateSandboxRequest(
image=ImageSpec(uri="python:3.11"),
entrypoint=["/bin/bash", "-c", "sleep 3600"],
timeout=3600,
resourceLimits=ResourceLimits(root={"cpu": "1", "memory": "1Gi"}),
volumes=[
Volume(
name="data",
mountPath="/data",
pvc=PVC(
claimName="auto-data",
createIfNotExists=True,
deleteOnSandboxTermination=True,
),
),
],
)
@pytest.mark.asyncio
async def test_shared_pvc_mixed_read_only_mounts_are_allowed(
self, k8s_service, mock_workload
):
from opensandbox_server.api.schema import (
CreateSandboxRequest,
ImageSpec,
PVC,
ResourceLimits,
Volume,
)
request = CreateSandboxRequest(
image=ImageSpec(uri="python:3.11"),
entrypoint=["/bin/bash", "-c", "sleep 3600"],
timeout=3600,
resourceLimits=ResourceLimits(root={"cpu": "1", "memory": "1Gi"}),
volumes=[
Volume(
name="shared-skills",
mountPath="/data/skills",
subPath="common/skills",
readOnly=True,
pvc=PVC(
claimName="auto-data",
createIfNotExists=True,
deleteOnSandboxTermination=True,
),
),
Volume(
name="shared-user",
mountPath="/data/user",
subPath="user1",
readOnly=False,
pvc=PVC(
claimName="auto-data",
createIfNotExists=True,
deleteOnSandboxTermination=True,
),
),
],
)
k8s_service.k8s_client.get_pvc.return_value = None
k8s_service.k8s_client.create_pvc.return_value = None
k8s_service.workload_provider.create_workload.return_value = {
"name": "test-id",
"uid": "abc",
}
k8s_service.workload_provider.get_status.return_value = {
"state": "Running",
"reason": "",
"message": "Pod is running",
"last_transition_at": datetime.now(timezone.utc),
}
with patch.object(k8s_service, "_wait_for_sandbox_ready", return_value=mock_workload):
response = await k8s_service.create_sandbox(request)
assert response.id is not None
k8s_service.k8s_client.create_pvc.assert_called_once()
k8s_service.workload_provider.create_workload.assert_called_once()
@pytest.mark.asyncio
async def test_pool_ref_with_volumes_rejected_before_pvc_creation(
self, k8s_service
):
# poolRef + volumes is invalid. The provider also rejects this but
# raises after _ensure_pvc_volumes has already auto-created and
# labeled PVCs — combined with the pessimistic workload_left_alive
# flag, the finally would skip cleanup and leak storage. The outer
# service must validate up-front, before any side effects.
from opensandbox_server.api.schema import (
CreateSandboxRequest,
ImageSpec,
PVC,
ResourceLimits,
Volume,
)
request = CreateSandboxRequest(
image=ImageSpec(uri="python:3.11"),
entrypoint=["/bin/bash", "-c", "sleep 3600"],
timeout=3600,
resourceLimits=ResourceLimits(root={"cpu": "1", "memory": "1Gi"}),
extensions={"poolRef": "my-pool"},
volumes=[
Volume(
name="data",
mountPath="/data",
pvc=PVC(
claimName="auto-data",
createIfNotExists=True,
deleteOnSandboxTermination=True,
),
),
],
)
with pytest.raises(HTTPException) as exc_info:
await k8s_service.create_sandbox(request)
assert exc_info.value.status_code == 400
# No PVC was created, no workload was attempted, no cleanup needed.
k8s_service.k8s_client.create_pvc.assert_not_called()
k8s_service.workload_provider.create_workload.assert_not_called()
@pytest.mark.asyncio
async def test_pvc_cleanup_runs_when_wait_for_ready_rollback_reports_not_found(
self, k8s_service, mock_workload
):
# The CR can disappear between create_workload returning and the
# readiness rollback (e.g. controller deletes it on a short sandbox
# ``timeout`` before ``sandbox_create_timeout_seconds``). The rollback
# then gets "not found" — same safe state as a successful delete,
# so PVCs must still be swept.
k8s_service.k8s_client.get_pvc.return_value = None
k8s_service.k8s_client.create_pvc.return_value = None
k8s_service.workload_provider.create_workload.return_value = {
"name": "test-id",
"uid": "abc",
}
k8s_service.workload_provider.delete_workload.side_effect = Exception(
"BatchSandbox 'test-id' not found"
)
async def _raise_ready(*args, **kwargs):
raise HTTPException(
status_code=500,
detail={"code": "X", "message": "pod never ready"},
)
with patch.object(k8s_service, "_wait_for_sandbox_ready", _raise_ready):
with patch.object(k8s_service, "_cleanup_managed_pvcs") as mock_cleanup:
with pytest.raises(HTTPException):
await k8s_service.create_sandbox(self._request_with_pvc())
mock_cleanup.assert_called_once()
@pytest.mark.asyncio
async def test_pvc_cleanup_runs_when_wait_for_ready_fails(
self, k8s_service, mock_workload
):
# PVC creation succeeds, workload is created, but the pod never
# becomes ready and _wait_for_sandbox_ready raises. The create API
# returns no id, so the caller cannot trigger delete_sandbox — the
# server must sweep the labeled PVC itself.
k8s_service.k8s_client.get_pvc.return_value = None
k8s_service.k8s_client.create_pvc.return_value = None
k8s_service.workload_provider.create_workload.return_value = {
"name": "test-id",
"uid": "abc",
}
async def _raise_ready(*args, **kwargs):
raise HTTPException(
status_code=500,
detail={"code": "X", "message": "pod never ready"},
)
with patch.object(k8s_service, "_wait_for_sandbox_ready", _raise_ready):
with patch.object(k8s_service, "_cleanup_managed_pvcs") as mock_cleanup:
with pytest.raises(HTTPException):
await k8s_service.create_sandbox(self._request_with_pvc())
mock_cleanup.assert_called_once()
@pytest.mark.asyncio
async def test_pvc_cleanup_skipped_when_rollback_delete_workload_fails(
self, k8s_service, mock_workload
):
# _wait_for_sandbox_ready raises after create_workload succeeded, then
# the rollback delete_workload also raises (e.g. transient API error).
# The CR is still alive in the cluster — sweeping its PVCs would leave
# a live workload referencing missing storage. ownerReference GC will
# reclaim them once the CR is eventually deleted.
k8s_service.k8s_client.get_pvc.return_value = None
k8s_service.k8s_client.create_pvc.return_value = None
k8s_service.workload_provider.create_workload.return_value = {
"name": "test-id",
"uid": "abc",
"apiVersion": "sandbox.opensandbox.io/v1alpha1",
"kind": "BatchSandbox",
}
k8s_service.workload_provider.delete_workload.side_effect = Exception(
"transient api error"
)
async def _raise_ready(*args, **kwargs):
raise HTTPException(
status_code=500,
detail={"code": "X", "message": "pod never ready"},
)
with patch.object(k8s_service, "_wait_for_sandbox_ready", _raise_ready):
with patch.object(k8s_service, "_cleanup_managed_pvcs") as mock_cleanup:
with pytest.raises(HTTPException):
await k8s_service.create_sandbox(self._request_with_pvc())
mock_cleanup.assert_not_called()
@pytest.mark.asyncio
async def test_pvc_cleanup_runs_when_create_workload_raises_and_rollback_succeeds(
self, k8s_service
):
# ``create_workload`` may have created the CR before raising; on any
# non-ValueError we attempt rollback delete_workload. When that
# rollback succeeds (or the CR never existed), the CR is gone and
# the labeled PVCs can be safely swept.
k8s_service.k8s_client.get_pvc.return_value = None
k8s_service.k8s_client.create_pvc.return_value = None
k8s_service.workload_provider.create_workload.side_effect = Exception(
"workload api 500"
)
k8s_service.workload_provider.delete_workload.return_value = None
with patch.object(k8s_service, "_cleanup_managed_pvcs") as mock_cleanup:
with pytest.raises(HTTPException):
await k8s_service.create_sandbox(self._request_with_pvc())
# Rollback ran, then PVCs were swept.
k8s_service.workload_provider.delete_workload.assert_called_once()
mock_cleanup.assert_called_once()
@pytest.mark.asyncio
async def test_pvc_cleanup_runs_when_rollback_reports_not_found(
self, k8s_service
):
# ``create_workload`` raised after the provider's internal rollback
# already deleted the CR (e.g. BatchSandboxProvider deletes the CR
# itself after imagePullSecret fails, then re-raises). Our outer
# rollback then gets "not found" — that's the same safe state as a
# successful delete, so PVCs must still be swept.
k8s_service.k8s_client.get_pvc.return_value = None
k8s_service.k8s_client.create_pvc.return_value = None
k8s_service.workload_provider.create_workload.side_effect = Exception(
"workload api 500"
)
k8s_service.workload_provider.delete_workload.side_effect = Exception(
"BatchSandbox 'sandbox-x' not found"
)
with patch.object(k8s_service, "_cleanup_managed_pvcs") as mock_cleanup:
with pytest.raises(HTTPException):
await k8s_service.create_sandbox(self._request_with_pvc())
mock_cleanup.assert_called_once()
@pytest.mark.asyncio
async def test_pvc_cleanup_skipped_when_create_workload_raises_and_rollback_fails(
self, k8s_service
):
# ``create_workload`` raised and the rollback delete_workload also
# raised — CR may still be alive. Skip the PVC sweep so we don't
# yank storage from a live workload; the user's delete_sandbox
# retry path will reclaim PVCs once the CR is confirmed gone.
k8s_service.k8s_client.get_pvc.return_value = None
k8s_service.k8s_client.create_pvc.return_value = None
k8s_service.workload_provider.create_workload.side_effect = Exception(
"workload api 500"
)
k8s_service.workload_provider.delete_workload.side_effect = Exception(
"rollback also failed"
)
with patch.object(k8s_service, "_cleanup_managed_pvcs") as mock_cleanup:
with pytest.raises(HTTPException):
await k8s_service.create_sandbox(self._request_with_pvc())
mock_cleanup.assert_not_called()
@pytest.mark.asyncio
async def test_pvc_cleanup_runs_when_create_workload_raises_value_error(
self, k8s_service
):
# Provider convention: ValueError == preflight failure with no CR
# touched (e.g. AgentSandboxProvider rejecting windows platform).
# PVCs we just labeled are orphans we must sweep, without needing
# rollback delete_workload.
k8s_service.k8s_client.get_pvc.return_value = None
k8s_service.k8s_client.create_pvc.return_value = None
k8s_service.workload_provider.create_workload.side_effect = ValueError(
"platform.os=windows not supported by this provider"
)
with patch.object(k8s_service, "_cleanup_managed_pvcs") as mock_cleanup:
with pytest.raises(HTTPException) as exc_info:
await k8s_service.create_sandbox(self._request_with_pvc())
assert exc_info.value.status_code == 400
# No rollback needed (no CR), but PVC cleanup still runs.
k8s_service.workload_provider.delete_workload.assert_not_called()
mock_cleanup.assert_called_once()
@pytest.mark.asyncio
async def test_pvc_cleanup_does_not_run_on_success(
self, k8s_service, mock_workload
):
# On the success path the caller owns the sandbox and any auto-created
# PVCs; the server must not double-sweep them.
k8s_service.k8s_client.get_pvc.return_value = None
k8s_service.k8s_client.create_pvc.return_value = None
k8s_service.workload_provider.create_workload.return_value = {
"name": "test-id",
"uid": "abc",
}
k8s_service.workload_provider.get_workload.return_value = mock_workload
k8s_service.workload_provider.get_status.return_value = {
"state": "Running",
"reason": "",
"message": "Running",
"last_transition_at": datetime.now(timezone.utc),
}
k8s_service.workload_provider.get_endpoint_info.return_value = "10.0.0.1:8080"
k8s_service.workload_provider.get_expiration.return_value = (
datetime.now(timezone.utc) + timedelta(hours=1)
)
with patch.object(k8s_service, "_cleanup_managed_pvcs") as mock_cleanup:
await k8s_service.create_sandbox(self._request_with_pvc())
mock_cleanup.assert_not_called()
class TestAttachPvcOwnerReferences:
"""OwnerReferences let K8s GC sweep PVCs on controller-driven CR deletion (TTL)."""
def test_patches_each_managed_pvc_with_owner_ref(self, k8s_service):
k8s_service._attach_pvc_owner_references(
["pvc-a", "pvc-b"],
{
"name": "sandbox-1",
"uid": "owner-uid",
"apiVersion": "sandbox.opensandbox.io/v1alpha1",
"kind": "BatchSandbox",
},
)
assert k8s_service.k8s_client.patch_pvc.call_count == 2
for call in k8s_service.k8s_client.patch_pvc.call_args_list:
ns, name, body = call.args
assert ns == k8s_service.namespace
owner_ref = body["metadata"]["ownerReferences"][0]
assert owner_ref["uid"] == "owner-uid"
assert owner_ref["name"] == "sandbox-1"
assert owner_ref["kind"] == "BatchSandbox"
assert owner_ref["apiVersion"] == "sandbox.opensandbox.io/v1alpha1"
# blockOwnerDeletion=False so PVC delete failures don't stall CR delete.
assert owner_ref["blockOwnerDeletion"] is False
def test_no_patch_when_no_managed_pvcs(self, k8s_service):
k8s_service._attach_pvc_owner_references([], {"name": "s", "uid": "u", "apiVersion": "g/v", "kind": "K"})
k8s_service.k8s_client.patch_pvc.assert_not_called()
def test_patch_failure_is_best_effort(self, k8s_service):
# Patch failure must not propagate — the sandbox is already created.
k8s_service.k8s_client.patch_pvc.side_effect = Exception("forbidden")
k8s_service._attach_pvc_owner_references(
["pvc-a"],
{"name": "s", "uid": "u", "apiVersion": "g/v", "kind": "K"},
)
def test_skips_when_owner_info_incomplete(self, k8s_service):
# If the workload provider didn't return full owner info, we must not
# produce a malformed ownerReference. Label-based cleanup remains.
k8s_service._attach_pvc_owner_references(
["pvc-a"],
{"name": "s", "uid": "u"}, # missing apiVersion/kind
)
k8s_service.k8s_client.patch_pvc.assert_not_called()
class TestListSandboxes:
def test_deleting_batchsandbox_status_in_list_and_get(self, k8s_service, mock_workload):
from opensandbox_server.api.schema import PaginationRequest, SandboxFilter
from opensandbox_server.services.k8s.batchsandbox_provider import BatchSandboxProvider
deletion_timestamp = datetime(2026, 10, 8, 10, tzinfo=timezone.utc)
mock_workload["metadata"]["deletionTimestamp"] = deletion_timestamp.isoformat()
mock_workload["status"] = {"phase": "Pending", "ready": 0}
provider = k8s_service.workload_provider
provider.list_workloads.return_value = [mock_workload]
provider.get_workload.return_value = mock_workload
provider.get_status.side_effect = BatchSandboxProvider(MagicMock()).get_status
provider.get_expiration.return_value = datetime.now(timezone.utc) + timedelta(hours=1)
request = ListSandboxesRequest(
filter=SandboxFilter(state=["Stopping"]), pagination=PaginationRequest(page=1, page_size=20)
)
response = k8s_service.list_sandboxes(request)
assert response.pagination.total_items == 1
assert len(response.items) == 1
sandbox = k8s_service.get_sandbox("test-sandbox-123")
for result in (response.items[0], sandbox):
assert result.status.state == "Stopping"
assert result.status.reason == "DELETING"
assert result.status.last_transition_at == deletion_timestamp
request.filter.state = ["Pending"]
assert not k8s_service.list_sandboxes(request).items
def test_list_all_sandboxes_succeeds(self, k8s_service, mock_workload):
k8s_service.workload_provider.list_workloads.return_value = [mock_workload]
k8s_service.workload_provider.get_status.return_value = {
"state": "Running",
"reason": "",
"message": "Running",
"last_transition_at": datetime.now(timezone.utc),
}
k8s_service.workload_provider.get_endpoint_info.return_value = "10.0.0.1:8080"
k8s_service.workload_provider.get_expiration.return_value = datetime.now(timezone.utc) + timedelta(hours=1)
from opensandbox_server.api.schema import PaginationRequest
request = ListSandboxesRequest(pagination=PaginationRequest(page=1, page_size=20))
response = k8s_service.list_sandboxes(request)
assert len(response.items) == 1
assert response.items[0].id == "test-sandbox-123"
assert response.pagination.total_items == 1
def test_list_sandboxes_with_pagination(self, k8s_service, mock_workload):
workloads = []
for i in range(10):
workload = {
"metadata": {
"name": f"sandbox-{i}",
"uid": f"uid-{i}",
"labels": {
"opensandbox.io/id": f"sandbox-{i}",
},
"annotations": mock_workload["metadata"]["annotations"].copy(),
"creationTimestamp": datetime.now(timezone.utc).isoformat(),
},
"spec": {},
"status": {},
}
workloads.append(workload)
k8s_service.workload_provider.list_workloads.return_value = workloads
k8s_service.workload_provider.get_status.return_value = {
"state": "Running",
"reason": "",
"message": "Running",
"last_transition_at": datetime.now(timezone.utc),
}
k8s_service.workload_provider.get_endpoint_info.return_value = "10.0.0.1:8080"
k8s_service.workload_provider.get_expiration.return_value = datetime.now(timezone.utc) + timedelta(hours=1)
from opensandbox_server.api.schema import PaginationRequest
request = ListSandboxesRequest(pagination=PaginationRequest(page=1, page_size=5))
response = k8s_service.list_sandboxes(request)
assert len(response.items) == 5
assert response.pagination.page == 1
assert response.pagination.page_size == 5
assert response.pagination.total_items == 10
assert response.pagination.total_pages == 2
def test_list_sandboxes_sorted_by_creation_time(self, k8s_service, mock_workload):
base_time = datetime.now(timezone.utc)
workloads = []
# We'll create them in random order to verify sorting works
creation_times = [
base_time - timedelta(hours=5), # Oldest
base_time - timedelta(hours=2),
base_time - timedelta(hours=1),
base_time - timedelta(minutes=30),
base_time, # Newest
]
for i, created_at in enumerate(creation_times):
workload = {
"metadata": {
"name": f"sandbox-{i}",
"uid": f"uid-{i}",
"labels": {
"opensandbox.io/id": f"sandbox-{i}",
},
"annotations": mock_workload["metadata"]["annotations"].copy(),
"creationTimestamp": created_at.isoformat(),
},
"spec": {},
"status": {},
}
workloads.append(workload)
k8s_service.workload_provider.list_workloads.return_value = workloads
k8s_service.workload_provider.get_status.return_value = {
"state": "Running",
"reason": "",
"message": "Running",
"last_transition_at": datetime.now(timezone.utc),
}
k8s_service.workload_provider.get_endpoint_info.return_value = "10.0.0.1:8080"
k8s_service.workload_provider.get_expiration.return_value = datetime.now(timezone.utc) + timedelta(hours=1)
from opensandbox_server.api.schema import PaginationRequest
request = ListSandboxesRequest(pagination=PaginationRequest(page=1, page_size=10))
response = k8s_service.list_sandboxes(request)
assert len(response.items) == 5
assert response.items[0].id == "sandbox-4" # Newest
assert response.items[1].id == "sandbox-3"
assert response.items[2].id == "sandbox-2"
assert response.items[3].id == "sandbox-1"
assert response.items[4].id == "sandbox-0" # Oldest
for i in range(len(response.items) - 1):
assert response.items[i].created_at >= response.items[i + 1].created_at
def test_list_sandboxes_returns_null_platform_for_default_scheduling(self, k8s_service):
workloads = [
{
"metadata": {
"name": "sandbox-1",
"uid": "uid-1",
"labels": {"opensandbox.io/id": "sandbox-1"},
"annotations": {},
"creationTimestamp": datetime.now(timezone.utc).isoformat(),
},
"spec": {
"template": {
"spec": {
# Default scheduling: no nodeSelector/nodeAffinity constraints.
}
}
},
"status": {},
}
]
k8s_service.workload_provider.list_workloads.return_value = workloads
k8s_service.workload_provider.get_status.return_value = {
"state": "Running",
"reason": "",
"message": "Running",
"last_transition_at": datetime.now(timezone.utc),
}
k8s_service.workload_provider.get_expiration.return_value = datetime.now(timezone.utc) + timedelta(hours=1)
from opensandbox_server.api.schema import PaginationRequest
request = ListSandboxesRequest(pagination=PaginationRequest(page=1, page_size=10))
response = k8s_service.list_sandboxes(request)
assert len(response.items) == 1
assert response.items[0].platform is None
class TestRenewExpiration:
@pytest.mark.parametrize(
"delta_seconds",
[-600, 0, 600, 7200],
ids=["shorten", "unchanged", "extend", "above-create-limit"],
)
def test_renew_sets_future_timestamp_without_create_limit(
self, k8s_service, mock_workload, delta_seconds
):
from opensandbox_server.api.schema import RenewSandboxExpirationRequest
k8s_service.app_config.server.max_sandbox_timeout_seconds = 3600
current_expiration = datetime.now(timezone.utc) + timedelta(minutes=30)
new_expiration = current_expiration + timedelta(seconds=delta_seconds)
k8s_service.workload_provider.get_workload.return_value = mock_workload
k8s_service.workload_provider.get_expiration.return_value = current_expiration
response = k8s_service.renew_expiration(
"test-sandbox-id", RenewSandboxExpirationRequest(expiresAt=new_expiration)
)
assert response.expires_at == new_expiration
k8s_service.workload_provider.update_expiration.assert_called_once_with(
sandbox_id="test-sandbox-id",
namespace=k8s_service.namespace,
expires_at=new_expiration,
)
def test_renew_expiration_succeeds(self, k8s_service, mock_workload):
new_expiration = datetime.now(timezone.utc) + timedelta(hours=2)
k8s_service.workload_provider.get_workload.return_value = mock_workload
k8s_service.workload_provider.update_expiration.return_value = None
k8s_service.workload_provider.get_expiration.return_value = new_expiration
from opensandbox_server.api.schema import RenewSandboxExpirationRequest
request = RenewSandboxExpirationRequest(expires_at=new_expiration)
response = k8s_service.renew_expiration("test-sandbox-id", request)
assert response.expires_at == new_expiration
k8s_service.workload_provider.update_expiration.assert_called_once_with(
sandbox_id="test-sandbox-id",
namespace=k8s_service.namespace,
expires_at=new_expiration
)
def test_renew_with_past_time_raises_error(self, k8s_service, mock_workload):
past_time = datetime.now(timezone.utc) - timedelta(hours=1)
k8s_service.workload_provider.get_workload.return_value = mock_workload
from opensandbox_server.api.schema import RenewSandboxExpirationRequest
request = RenewSandboxExpirationRequest(expires_at=past_time)
with pytest.raises(HTTPException) as exc_info:
k8s_service.renew_expiration("test-sandbox-id", request)
assert exc_info.value.status_code == 400
def test_renew_returns_409_when_sandbox_has_no_expiration(self, k8s_service):
"""Renew is rejected with 409 when sandbox has no TTL (manual cleanup)."""
k8s_service.workload_provider.get_workload.return_value = MagicMock()
k8s_service.workload_provider.get_expiration.return_value = None
from opensandbox_server.api.schema import RenewSandboxExpirationRequest
request = RenewSandboxExpirationRequest(
expires_at=datetime.now(timezone.utc) + timedelta(hours=1)
)
with pytest.raises(HTTPException) as exc_info:
k8s_service.renew_expiration("no-ttl-sandbox", request)
assert exc_info.value.status_code == 409
assert "does not have automatic expiration" in exc_info.value.detail["message"]
k8s_service.workload_provider.update_expiration.assert_not_called()
class TestSignedEndpoint:
"""Test signed route token generation in get_endpoint."""
BASE64_SECRET = "bXktdGVzdC1zZWNyZXQ=" # "my-test-secret"
def _setup_gateway_with_secure_access(self, k8s_service, route_mode="wildcard"):
"""Helper to configure ingress gateway with secure_access on the service."""
address = "*.sandbox.example.com" if route_mode == "wildcard" else "gateway.sandbox.example.com"
k8s_service.ingress_config = IngressConfig(
mode="gateway",
gateway=GatewayConfig(
address=address,
route=GatewayRouteModeConfig(mode=route_mode),
),
secure_access=SecureAccessConfig(
active_key="a",
keys=[
SecureAccessKey(key_id="a", key=self.BASE64_SECRET),
],
),
)
def test_signed_endpoint_embeds_token_in_url(self, k8s_service):
self._setup_gateway_with_secure_access(k8s_service)
k8s_service.workload_provider.get_workload.return_value = {
"metadata": {"annotations": {}},
}
endpoint = k8s_service.get_endpoint("sbx-001", 8080, expires=2000000000)
assert endpoint.endpoint.startswith("sbx-001-8080-")
assert endpoint.endpoint.endswith(".sandbox.example.com")
# The signature should be embedded in the endpoint URL (right-split for hyphenated sandbox_id)
parts = endpoint.endpoint.split(".")[0].rsplit("-", 3)
assert len(parts) == 4 # sandbox_id-port-b36-signature
def test_signed_endpoint_omits_secure_access_header(self, k8s_service):
"""Signed endpoint must NOT include the static SecureAccessToken header."""
self._setup_gateway_with_secure_access(k8s_service)
k8s_service.workload_provider.get_workload.return_value = {
"metadata": {
"annotations": {
SANDBOX_SECURE_ACCESS_TOKEN_METADATA_KEY: "static-token",
}
},
}
endpoint = k8s_service.get_endpoint("sbx-001", 8080, expires=2000000000)
if endpoint.headers:
assert OPEN_SANDBOX_SECURE_ACCESS_HEADER not in endpoint.headers, (
"Signed endpoint should not carry the static SecureAccessToken header"
)
def test_unsigned_endpoint_attaches_secure_access_header(self, k8s_service):
"""Unsigned endpoint must include the static SecureAccessToken header."""
self._setup_gateway_with_secure_access(k8s_service)
k8s_service.workload_provider.get_workload.return_value = {
"metadata": {
"annotations": {
SANDBOX_SECURE_ACCESS_TOKEN_METADATA_KEY: "static-token",
}
},
}
k8s_service.workload_provider.get_endpoint_info.return_value = Endpoint(
endpoint="sbx-001-8080.sandbox.example.com",
)
endpoint = k8s_service.get_endpoint("sbx-001", 8080)
assert endpoint.headers is not None
assert endpoint.headers[OPEN_SANDBOX_SECURE_ACCESS_HEADER] == "static-token"
def test_signed_endpoint_header_mode(self, k8s_service):
self._setup_gateway_with_secure_access(k8s_service, route_mode="header")
k8s_service.workload_provider.get_workload.return_value = {
"metadata": {"annotations": {}},
}
endpoint = k8s_service.get_endpoint("sbx-001", 8080, expires=2000000000)
assert endpoint.endpoint == "gateway.sandbox.example.com"
assert endpoint.headers is not None
ingress_val = endpoint.headers.get("OpenSandbox-Ingress-To", "")
# Header value should contain the signed route: {sid}-{port}-{b36}-{sig}
parts = ingress_val.rsplit("-", 3)
assert len(parts) == 4
def test_signed_endpoint_uri_mode(self, k8s_service):
self._setup_gateway_with_secure_access(k8s_service, route_mode="uri")
k8s_service.workload_provider.get_workload.return_value = {
"metadata": {"annotations": {}},
}
endpoint = k8s_service.get_endpoint("sbx-001", 8080, expires=2000000000)
# URI mode: endpoint is {addr}/{sid}/{port}/{b36}/{sig}
assert "x2qxvk" in endpoint.endpoint
assert "0ff8cd39a" in endpoint.endpoint
def test_expires_negative_rejected(self, k8s_service):
k8s_service.workload_provider.get_workload.return_value = {
"metadata": {"annotations": {}},
}
with pytest.raises(HTTPException) as exc:
k8s_service.get_endpoint("sbx-001", 8080, expires=-1)
assert exc.value.status_code == 400
def test_expires_in_past_rejected(self, k8s_service):
k8s_service.workload_provider.get_workload.return_value = {
"metadata": {"annotations": {}},
}
with pytest.raises(HTTPException) as exc:
k8s_service.get_endpoint("sbx-001", 8080, expires=1000000)
assert exc.value.status_code == 400
assert "must be greater than current time" in exc.value.detail["message"]
def test_expires_without_gateway_rejected(self, k8s_service):
"""No ingress config at all."""
k8s_service.ingress_config = None
k8s_service.workload_provider.get_workload.return_value = {
"metadata": {"annotations": {}},
}
with pytest.raises(HTTPException) as exc:
k8s_service.get_endpoint("sbx-001", 8080, expires=2000000000)
assert exc.value.status_code == 400
assert "gateway" in exc.value.detail["message"].lower()
def test_expires_without_secure_access_keys_rejected(self, k8s_service):
"""Gateway configured but no secure_access keys block."""
k8s_service.ingress_config = IngressConfig(
mode="gateway",
gateway=GatewayConfig(
address="*.example.com",
route=GatewayRouteModeConfig(mode="wildcard"),
),
)
k8s_service.workload_provider.get_workload.return_value = {
"metadata": {"annotations": {}},
}
with pytest.raises(HTTPException) as exc:
k8s_service.get_endpoint("sbx-001", 8080, expires=2000000000)
assert exc.value.status_code == 400
assert "secure_access" in exc.value.detail["message"].lower()
def test_unsigned_endpoint_no_expires(self, k8s_service):
self._setup_gateway_with_secure_access(k8s_service)
k8s_service.workload_provider.get_workload.return_value = {
"metadata": {"annotations": {}},
}
k8s_service.workload_provider.get_endpoint_info.return_value = Endpoint(
endpoint="sbx-001-8080.sandbox.example.com",
)
endpoint = k8s_service.get_endpoint("sbx-001", 8080)
assert endpoint.endpoint == "sbx-001-8080.sandbox.example.com"
def test_signed_endpoint_different_expires_produces_different_endpoints(self, k8s_service):
self._setup_gateway_with_secure_access(k8s_service)
k8s_service.workload_provider.get_workload.return_value = {
"metadata": {"annotations": {}},
}
ep1 = k8s_service.get_endpoint("sbx-001", 8080, expires=2000000000)
ep2 = k8s_service.get_endpoint("sbx-001", 8080, expires=2000000500)
assert ep1.endpoint != ep2.endpoint
class TestPatchSandboxMetadata:
"""Verify patch_sandbox_metadata builds the JSON merge-patch body correctly
and uses the API server's PATCH response (not a cache-prone re-fetch)."""
@staticmethod
def _workload(labels: dict) -> dict:
return {
"metadata": {
"name": "sandbox-sbx-001",
"labels": dict(labels),
"creationTimestamp": datetime(2026, 1, 1, tzinfo=timezone.utc),
},
"spec": {},
"status": {"conditions": []},
}
@staticmethod
def _stub_provider_status(k8s_service) -> None:
k8s_service.workload_provider.get_status.return_value = {
"state": "Running",
"reason": None,
"message": None,
"last_transition_at": None,
}
k8s_service.workload_provider.get_expiration.return_value = None
def test_patch_body_sends_null_for_deleted_keys(self, k8s_service):
initial = {"opensandbox.io/id": "sbx-001", "team": "infra", "env": "dev"}
patched = {"opensandbox.io/id": "sbx-001", "env": "stage"}
k8s_service.workload_provider.get_workload.return_value = self._workload(initial)
k8s_service.workload_provider.patch_labels.return_value = self._workload(patched)
self._stub_provider_status(k8s_service)
k8s_service.patch_sandbox_metadata("sbx-001", {"env": "stage", "team": None})
k8s_service.workload_provider.patch_labels.assert_called_once()
body_labels = k8s_service.workload_provider.patch_labels.call_args.kwargs["labels"]
assert body_labels["env"] == "stage"
assert body_labels["team"] is None
assert body_labels["opensandbox.io/id"] == "sbx-001"
def test_returns_sandbox_from_patch_response(self, k8s_service):
"""The PATCH response is authoritative; re-reading via get_workload
could hit a stale informer cache."""
initial = {"opensandbox.io/id": "sbx-001", "env": "dev"}
patched = {"opensandbox.io/id": "sbx-001", "env": "stage"}
k8s_service.workload_provider.get_workload.return_value = self._workload(initial)
k8s_service.workload_provider.patch_labels.return_value = self._workload(patched)
self._stub_provider_status(k8s_service)
sandbox = k8s_service.patch_sandbox_metadata("sbx-001", {"env": "stage"})
assert sandbox.metadata == {"env": "stage"}
# Pre-patch read only; no second get_workload after patch_labels.
assert k8s_service.workload_provider.get_workload.call_count == 1
class TestSharedNamespaceTenantIsolation:
"""Same-namespace tenants must only see sandboxes stamped with their name."""
def _as_tenant(self, name: str):
from opensandbox_server.tenants.context import get_current_tenant, set_current_tenant
from opensandbox_server.tenants.models import TenantEntry
previous = get_current_tenant()
set_current_tenant(TenantEntry(name=name, namespace="opensandbox-system", api_keys=("k",)))
return previous
def _clear_tenant(self, previous) -> None:
from opensandbox_server.tenants.context import set_current_tenant
set_current_tenant(previous)
def _labeled_workload(self, mock_workload, sandbox_id: str, tenant: str | None):
from copy import deepcopy
workload = deepcopy(mock_workload)
workload["metadata"]["name"] = sandbox_id
workload["metadata"]["labels"] = {"opensandbox.io/id": sandbox_id}
if tenant is not None:
workload["metadata"]["labels"]["opensandbox.io/tenant"] = tenant
return workload
def _stub_list_status(self, k8s_service) -> None:
k8s_service.workload_provider.get_status.return_value = {
"state": "Running",
"reason": "",
"message": "Running",
"last_transition_at": datetime.now(timezone.utc),
}
k8s_service.workload_provider.get_endpoint_info.return_value = "10.0.0.1:8080"
k8s_service.workload_provider.get_expiration.return_value = datetime.now(timezone.utc) + timedelta(hours=1)
@pytest.mark.asyncio
async def test_create_stamps_tenant_label(
self, k8s_service, create_sandbox_request, mock_workload
):
previous = self._as_tenant("gdds")
try:
k8s_service.workload_provider.create_workload.return_value = {
"name": "test-sandbox-123",
"uid": "abc-123",
}
k8s_service.workload_provider.get_workload.return_value = mock_workload
self._stub_list_status(k8s_service)
await k8s_service.create_sandbox(create_sandbox_request)
labels = k8s_service.workload_provider.create_workload.call_args.kwargs["labels"]
assert labels["opensandbox.io/tenant"] == "gdds"
finally:
self._clear_tenant(previous)
def test_list_hides_other_tenant_keeps_own_and_legacy(
self, k8s_service, mock_workload
):
own = self._labeled_workload(mock_workload, "own-id", "gdds")
other = self._labeled_workload(mock_workload, "other-id", "geip")
legacy = self._labeled_workload(mock_workload, "legacy-id", None)
k8s_service.workload_provider.list_workloads.return_value = [own, other, legacy]
self._stub_list_status(k8s_service)
previous = self._as_tenant("gdds")
try:
from opensandbox_server.api.schema import PaginationRequest
response = k8s_service.list_sandboxes(
ListSandboxesRequest(pagination=PaginationRequest(page=1, page_size=20))
)
assert {item.id for item in response.items} == {"own-id", "legacy-id"}
finally:
self._clear_tenant(previous)
def test_get_other_tenant_returns_404(self, k8s_service, mock_workload):
other = self._labeled_workload(mock_workload, "other-id", "geip")
k8s_service.workload_provider.get_workload.return_value = other
self._stub_list_status(k8s_service)
previous = self._as_tenant("gdds")
try:
with pytest.raises(HTTPException) as exc_info:
k8s_service.get_sandbox("other-id")
assert exc_info.value.status_code == 404
finally:
self._clear_tenant(previous)
def test_get_own_tenant_succeeds(self, k8s_service, mock_workload):
own = self._labeled_workload(mock_workload, "own-id", "gdds")
k8s_service.workload_provider.get_workload.return_value = own
self._stub_list_status(k8s_service)
previous = self._as_tenant("gdds")
try:
sandbox = k8s_service.get_sandbox("own-id")
assert sandbox.id == "own-id"
finally:
self._clear_tenant(previous)
def test_delete_other_tenant_returns_404(self, k8s_service, mock_workload):
other = self._labeled_workload(mock_workload, "other-id", "geip")
k8s_service.workload_provider.get_workload.return_value = other
previous = self._as_tenant("gdds")
try:
with patch.object(k8s_service, "_cleanup_managed_pvcs") as mock_cleanup:
with pytest.raises(HTTPException) as exc_info:
k8s_service.delete_sandbox("other-id")
assert exc_info.value.status_code == 404
k8s_service.workload_provider.delete_workload.assert_not_called()
mock_cleanup.assert_not_called()
finally:
self._clear_tenant(previous)
def test_pause_other_tenant_returns_404(self, k8s_service, mock_workload):
other = self._labeled_workload(mock_workload, "other-id", "geip")
k8s_service.workload_provider.get_workload.return_value = other
previous = self._as_tenant("gdds")
try:
with pytest.raises(HTTPException) as exc_info:
k8s_service.pause_sandbox("other-id")
assert exc_info.value.status_code == 404
k8s_service.workload_provider.pause_sandbox.assert_not_called()
finally:
self._clear_tenant(previous)
def test_resume_other_tenant_returns_404(self, k8s_service, mock_workload):
other = self._labeled_workload(mock_workload, "other-id", "geip")
k8s_service.workload_provider.get_workload.return_value = other
previous = self._as_tenant("gdds")
try:
with pytest.raises(HTTPException) as exc_info:
k8s_service.resume_sandbox("other-id")
assert exc_info.value.status_code == 404
k8s_service.workload_provider.resume_sandbox.assert_not_called()
finally:
self._clear_tenant(previous)
@pytest.mark.asyncio
@pytest.mark.parametrize("notification", [
"MODIFIED", "ADDED", "SYNC", "during_read", "DELETED", "replaced",
"missing_version", "deleting", "burst",
])
async def test_create_wait_uses_event_object(k8s_service, notification):
provider = k8s_service.workload_provider
pending_seen = asyncio.Event()
unsubscribe = MagicMock()
callback = None
pending = {
"metadata": {"name": "test-sandbox-id", "uid": "uid-1", "resourceVersion": "one"},
"status": {"state": "Pending", "reason": "", "message": "pending"},
}
ready = {
"metadata": {**pending["metadata"], "resourceVersion": "two"},
"status": {"state": "Running", "reason": "", "message": "ready"},
}
event = {**ready, "metadata": dict(ready["metadata"])}
if notification == "replaced":
event["metadata"]["uid"] = "different"
elif notification == "missing_version":
event["metadata"].pop("resourceVersion")
elif notification == "deleting":
event["metadata"]["deletionTimestamp"] = "2026-09-23T00:00:00Z"
def subscribe(sandbox_id, namespace, notify):
nonlocal callback
callback = notify
return unsubscribe
provider.subscribe_workload.side_effect = subscribe
reads = 0
def read(**kwargs):
nonlocal reads
reads += 1
assert callback is not None
if reads == 1:
if notification == "during_read":
callback("MODIFIED", event)
return pending
return ready
provider.get_workload.side_effect = read
def status(workload):
pending_seen.set()
return workload["status"]
provider.get_status.side_effect = status
task = asyncio.create_task(k8s_service._wait_for_sandbox_ready(
"test-sandbox-id", timeout_seconds=60, poll_interval_seconds=1,
))
try:
await asyncio.wait_for(pending_seen.wait(), timeout=1)
assert callback is not None
if notification != "during_read":
def dispatch():
if notification == "burst":
callback("MODIFIED", pending)
callback(notification if notification in ("ADDED", "SYNC", "DELETED") else "MODIFIED", event)
await asyncio.to_thread(dispatch)
result = await asyncio.wait_for(task, timeout=0.5)
direct_event = notification in ("MODIFIED", "ADDED", "SYNC", "burst")
assert result is (event if direct_event else ready)
assert provider.get_workload.call_count == (1 if direct_event else 2)
unsubscribe.assert_called_once_with()
callback("MODIFIED", event) # Late delivery after cleanup is harmless.
await asyncio.sleep(0)
assert provider.get_workload.call_count == (1 if direct_event else 2)
finally:
if not task.done():
task.cancel()
with pytest.raises(asyncio.CancelledError):
await task
@pytest.mark.asyncio
@pytest.mark.parametrize("subscription", ["supported", "unsupported", "failed"])
async def test_create_wait_falls_back_without_notifications(k8s_service, mock_workload, subscription):
provider = k8s_service.workload_provider
unsubscribe = MagicMock()
provider.subscribe_workload.return_value = unsubscribe if subscription == "supported" else None
if subscription == "failed":
provider.subscribe_workload.side_effect = RuntimeError("watch unavailable")
provider.get_workload.return_value = mock_workload
provider.get_status.side_effect = [
{"state": "Pending", "reason": "", "message": "pending"},
{"state": "Running", "reason": "", "message": "ready"},
]
result = await asyncio.wait_for(k8s_service._wait_for_sandbox_ready(
"test-sandbox-id", timeout_seconds=10, poll_interval_seconds=0.01,
), timeout=1)
assert result == mock_workload
if subscription == "supported":
unsubscribe.assert_called_once_with()
@pytest.mark.asyncio
@pytest.mark.parametrize("outcome", ["ready", "failed", "timeout", "pool_timeout", "cancelled"])
async def test_create_wait_cleans_up_and_honors_deadlines(k8s_service, mock_workload, outcome):
provider = k8s_service.workload_provider
unsubscribe = MagicMock()
provider.subscribe_workload.return_value = unsubscribe
provider.get_workload.return_value = mock_workload
observed = asyncio.Event()
def status(workload):
observed.set()
return {
"state": {"ready": "Running", "failed": "Failed"}.get(outcome, "Pending"),
"reason": "POOL_CAPACITY_EXHAUSTED" if outcome == "pool_timeout" else "",
"message": "test",
}
provider.get_status.side_effect = status
task = asyncio.create_task(k8s_service._wait_for_sandbox_ready(
"test-sandbox-id", timeout_seconds=1 if outcome == "timeout" else 60,
poll_interval_seconds=30, pool_acquisition_timeout_seconds=0.02,
))
try:
await asyncio.wait_for(observed.wait(), timeout=1)
if outcome == "cancelled":
task.cancel()
with pytest.raises(asyncio.CancelledError):
await task
elif outcome == "ready":
assert await asyncio.wait_for(task, timeout=1) == mock_workload
else:
with pytest.raises(HTTPException) as exc_info:
await asyncio.wait_for(task, timeout=2)
assert exc_info.value.status_code == {"failed": 500, "timeout": 504, "pool_timeout": 429}[outcome]
unsubscribe.assert_called_once_with()
finally:
if not task.done():
task.cancel()
with pytest.raises(asyncio.CancelledError):
await task