Skip to content

Secure, Fast, and Extensible Sandbox runtime for AI agents.

73.7 KB
Raw Blame
# Copyright 2025 The OpenSandbox Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from datetime import datetime, timezone
from types import SimpleNamespace
from unittest.mock import MagicMock

import pytest
from fastapi import HTTPException
from kubernetes.client import ApiException

from opensandbox_server.api.schema import ImageAuth, ImageSpec, NetworkPolicy, NetworkRule, PlatformSpec
from opensandbox_server.config import (
    AppConfig,
    AgentSandboxRuntimeConfig,
    EGRESS_MODE_DNS,
    EGRESS_MODE_DNS_NFT,
    EgressUpstreamProxyConfig,
    ExecdInitResources,
    KubernetesRuntimeConfig,
    RuntimeConfig,
)
from opensandbox_server.services.constants import (
    EGRESS_UPSTREAM_EXTRA_CA_PATH,
    EGRESS_UPSTREAM_EXTRA_CA_SECRET_KEY,
    EGRESS_UPSTREAM_EXTRA_CA_VOLUME_NAME,
    OPEN_SANDBOX_EGRESS_AUTH_HEADER,
    OPENSANDBOX_EGRESS_MITMPROXY_TRANSPARENT,
    OPENSANDBOX_EGRESS_MITMPROXY_UPSTREAM_EXTRA_CA,
    OPENSANDBOX_RUNTIME_MOUNT_PATH,
    OPENSANDBOX_RUNTIME_VOLUME_NAME,
    SANDBOX_EGRESS_AUTH_TOKEN_METADATA_KEY,
)
from opensandbox_server.services.k8s.agent_sandbox_provider import AgentSandboxProvider
from opensandbox_server.services.k8s.image_pull_secret_helper import (
    IMAGE_AUTH_SECRET_PREFIX,
)
from opensandbox_server.services.k8s.workload_provider import EgressWorkloadSettings
from opensandbox_server.services.constants import OPENSANDBOX_EGRESS_TOKEN


def _app_config(
    shutdown_policy: str = "Delete",
    execd_init_resources: ExecdInitResources | None = None,
) -> AppConfig:
    """Build an AppConfig for AgentSandboxProvider tests."""
    return AppConfig(
        runtime=RuntimeConfig(type="kubernetes", execd_image="execd:test"),
        kubernetes=KubernetesRuntimeConfig(
            namespace="test-ns",
            workload_provider="agent-sandbox",
            execd_init_resources=execd_init_resources,
        ),
        agent_sandbox=AgentSandboxRuntimeConfig(shutdown_policy=shutdown_policy),
    )


def _egress_settings(
    network_policy: NetworkPolicy,
    *,
    image: str = "opensandbox/egress:v1.1.7",
    mode: str = EGRESS_MODE_DNS,
    auth_token: str | None = None,
    credential_proxy_enabled: bool = False,
    disable_ipv6: bool = True,
    upstream_proxy: EgressUpstreamProxyConfig | None = None,
) -> EgressWorkloadSettings:
    return EgressWorkloadSettings(
        network_policy=network_policy,
        image=image,
        mode=mode,
        auth_token=auth_token,
        credential_proxy_enabled=credential_proxy_enabled,
        env={},
        disable_ipv6=disable_ipv6,
        resource_requests=None,
        resource_limits=None,
        upstream_proxy=upstream_proxy,
    )


class TestAgentSandboxProvider:
    def test_init_sets_crd_constants_correctly(self, mock_k8s_client):
        provider = AgentSandboxProvider(mock_k8s_client)

        assert provider.group == "agents.x-k8s.io"
        assert provider.version == "v1beta1"
        assert provider.plural == "sandboxes"

    def test_create_workload_builds_correct_manifest_init_mode(self, mock_k8s_client):
        provider = AgentSandboxProvider(
            mock_k8s_client,
            _app_config(shutdown_policy="Delete"),
        )
        mock_k8s_client.create_custom_object.return_value = {
            "metadata": {"name": "test-id", "uid": "test-uid"}
        }

        expires_at = datetime(2025, 12, 31, 10, 0, 0, tzinfo=timezone.utc)

        result = provider.create_workload(
            sandbox_id="test-id",
            namespace="test-ns",
            image_spec=ImageSpec(uri="python:3.11"),
            entrypoint=["/bin/bash"],
            env={"FOO": "bar"},
            resource_limits={"cpu": "1", "memory": "1Gi"},
            labels={"opensandbox.io/id": "test-id"},
            expires_at=expires_at,
            execd_image="execd:latest",
        )

        assert result == {"name": "test-id", "uid": "test-uid", "apiVersion": "agents.x-k8s.io/v1beta1", "kind": "Sandbox"}

        call_kwargs = mock_k8s_client.create_custom_object.call_args.kwargs
        assert call_kwargs["group"] == "agents.x-k8s.io"
        assert call_kwargs["version"] == "v1beta1"
        assert call_kwargs["plural"] == "sandboxes"
        body = call_kwargs["body"]
        assert body["apiVersion"] == "agents.x-k8s.io/v1beta1"
        assert body["kind"] == "Sandbox"
        assert body["metadata"]["name"] == "test-id"
        assert body["metadata"]["namespace"] == "test-ns"
        assert body["spec"]["operatingMode"] == "Running"
        assert body["spec"]["service"] is True
        assert "replicas" not in body["spec"]
        assert body["spec"]["shutdownTime"] == "2025-12-31T10:00:00+00:00"
        assert body["spec"]["shutdownPolicy"] == "Delete"
        assert body["spec"]["podTemplate"]["spec"]["automountServiceAccountToken"] is False
        assert "serviceAccountName" not in body["spec"]["podTemplate"]["spec"]
        assert "initContainers" in body["spec"]["podTemplate"]["spec"]
        assert "containers" in body["spec"]["podTemplate"]["spec"]
        assert "volumes" in body["spec"]["podTemplate"]["spec"]

    @pytest.mark.parametrize("expires_at", [None, datetime(2026, 12, 31, tzinfo=timezone.utc)])
    def test_create_workload_overrides_template_lifecycle(self, mock_k8s_client, expires_at):
        provider = AgentSandboxProvider(mock_k8s_client, _app_config(shutdown_policy="Retain"))
        provider.template_manager._template = {
            "spec": {
                "operatingMode": "Suspended",
                "service": False,
                "shutdownPolicy": "Delete",
                "shutdownTime": "2025-01-01T00:00:00Z",
                "podTemplate": {"spec": {"nodeSelector": {"env": "test"}}},
            }
        }
        mock_k8s_client.create_custom_object.return_value = {
            "metadata": {"name": "test-id", "uid": "test-uid"}
        }

        provider.create_workload(
            sandbox_id="test-id",
            namespace="test-ns",
            image_spec=ImageSpec(uri="python:3.11", auth=None),
            entrypoint=["/bin/bash"],
            env={},
            resource_limits={"cpu": "1", "memory": "1Gi"},
            labels={},
            expires_at=expires_at,
            execd_image="execd:latest",
        )

        spec = mock_k8s_client.create_custom_object.call_args.kwargs["body"]["spec"]
        assert spec["operatingMode"] == "Running"
        assert spec["service"] is True
        assert "replicas" not in spec
        assert spec["shutdownPolicy"] == "Retain"
        assert spec["podTemplate"]["spec"]["nodeSelector"] == {"env": "test"}
        if expires_at is None:
            assert "shutdownTime" not in spec
        else:
            assert spec["shutdownTime"] == expires_at.isoformat()

    def test_create_workload_injects_platform_node_selector(self, mock_k8s_client):
        provider = AgentSandboxProvider(mock_k8s_client, _app_config())
        mock_k8s_client.create_custom_object.return_value = {
            "metadata": {"name": "test-id", "uid": "test-uid"}
        }

        provider.create_workload(
            sandbox_id="test-id",
            namespace="test-ns",
            image_spec=ImageSpec(uri="python:3.11"),
            entrypoint=["/bin/bash"],
            env={},
            resource_limits={"cpu": "1", "memory": "1Gi"},
            labels={"opensandbox.io/id": "test-id"},
            expires_at=None,
            execd_image="execd:latest",
            platform=PlatformSpec(os="linux", arch="arm64"),
        )

        body = mock_k8s_client.create_custom_object.call_args.kwargs["body"]
        selector = body["spec"]["podTemplate"]["spec"]["nodeSelector"]
        assert selector["kubernetes.io/os"] == "linux"
        assert selector["kubernetes.io/arch"] == "arm64"

    def test_create_workload_uses_separate_resource_requests(self, mock_k8s_client):
        provider = AgentSandboxProvider(mock_k8s_client, _app_config())
        mock_k8s_client.create_custom_object.return_value = {
            "metadata": {"name": "test-id", "uid": "test-uid"}
        }

        provider.create_workload(
            sandbox_id="test-id",
            namespace="test-ns",
            image_spec=ImageSpec(uri="python:3.11"),
            entrypoint=["/bin/bash"],
            env={},
            resource_limits={"cpu": "2", "memory": "2Gi"},
            resource_requests={"cpu": "500m", "memory": "512Mi"},
            labels={"opensandbox.io/id": "test-id"},
            expires_at=None,
            execd_image="execd:latest",
        )

        body = mock_k8s_client.create_custom_object.call_args.kwargs["body"]
        resources = body["spec"]["podTemplate"]["spec"]["containers"][0]["resources"]

        assert resources["limits"] == {"cpu": "2", "memory": "2Gi"}
        assert resources["requests"] == {"cpu": "500m", "memory": "512Mi"}

    def test_create_workload_translates_gpu_to_nvidia_extended_resource(self, mock_k8s_client):
        provider = AgentSandboxProvider(mock_k8s_client, _app_config())
        mock_k8s_client.create_custom_object.return_value = {
            "metadata": {"name": "test-id", "uid": "test-uid"}
        }

        provider.create_workload(
            sandbox_id="test-id",
            namespace="test-ns",
            image_spec=ImageSpec(uri="python:3.11"),
            entrypoint=["/bin/bash"],
            env={},
            resource_limits={"cpu": "1", "memory": "1Gi", "gpu": "2"},
            labels={"opensandbox.io/id": "test-id"},
            expires_at=None,
            execd_image="execd:latest",
        )

        body = mock_k8s_client.create_custom_object.call_args.kwargs["body"]
        resources = body["spec"]["podTemplate"]["spec"]["containers"][0]["resources"]

        assert resources["limits"]["nvidia.com/gpu"] == "2"
        assert resources["requests"]["nvidia.com/gpu"] == "2"
        assert "gpu" not in resources["limits"]
        assert "gpu" not in resources["requests"]

    def test_create_workload_without_gpu_omits_nvidia_extended_resource(self, mock_k8s_client):
        provider = AgentSandboxProvider(mock_k8s_client, _app_config())
        mock_k8s_client.create_custom_object.return_value = {
            "metadata": {"name": "test-id", "uid": "test-uid"}
        }

        provider.create_workload(
            sandbox_id="test-id",
            namespace="test-ns",
            image_spec=ImageSpec(uri="python:3.11"),
            entrypoint=["/bin/bash"],
            env={},
            resource_limits={"cpu": "1", "memory": "1Gi"},
            labels={"opensandbox.io/id": "test-id"},
            expires_at=None,
            execd_image="execd:latest",
        )

        body = mock_k8s_client.create_custom_object.call_args.kwargs["body"]
        resources = body["spec"]["podTemplate"]["spec"]["containers"][0]["resources"]

        assert "nvidia.com/gpu" not in resources["limits"]
        assert "nvidia.com/gpu" not in resources["requests"]

    def test_create_workload_rejects_gpu_all_sentinel(self, mock_k8s_client):
        provider = AgentSandboxProvider(mock_k8s_client, _app_config())

        with pytest.raises(HTTPException) as excinfo:
            provider.create_workload(
                sandbox_id="test-id",
                namespace="test-ns",
                image_spec=ImageSpec(uri="python:3.11"),
                entrypoint=["/bin/bash"],
                env={},
                resource_limits={"cpu": "1", "gpu": "all"},
                labels={"opensandbox.io/id": "test-id"},
                expires_at=None,
                execd_image="execd:latest",
            )
        assert excinfo.value.status_code == 400

    def test_create_workload_rejects_windows_platform(self, mock_k8s_client):
        provider = AgentSandboxProvider(mock_k8s_client, _app_config())

        with pytest.raises(ValueError, match="agent-sandbox does not support platform.os=windows"):
            provider.create_workload(
                sandbox_id="test-id",
                namespace="test-ns",
                image_spec=ImageSpec(uri="dockurr/windows:latest"),
                entrypoint=["cmd", "/c", "echo hello"],
                env={},
                resource_limits={"cpu": "4", "memory": "8G", "disk": "64G"},
                labels={"opensandbox.io/id": "test-id"},
                expires_at=None,
                execd_image="execd:latest",
                platform=PlatformSpec(os="windows", arch="amd64"),
            )

    def test_create_workload_rejects_platform_conflict_with_template_selector(
        self, mock_k8s_client, tmp_path
    ):
        template_file = tmp_path / "agent_template.yaml"
        template_file.write_text(
            """
spec:
  podTemplate:
    spec:
      nodeSelector:
        kubernetes.io/os: linux
        kubernetes.io/arch: amd64
"""
        )
        app_config = _app_config()
        app_config.agent_sandbox.template_file = str(template_file)
        provider = AgentSandboxProvider(mock_k8s_client, app_config)

        with pytest.raises(ValueError, match="platform conflict with template nodeSelector"):
            provider.create_workload(
                sandbox_id="test-id",
                namespace="test-ns",
                image_spec=ImageSpec(uri="python:3.11"),
                entrypoint=["/bin/bash"],
                env={},
                resource_limits={"cpu": "1", "memory": "1Gi"},
                labels={"opensandbox.io/id": "test-id"},
                expires_at=None,
                execd_image="execd:latest",
                platform=PlatformSpec(os="linux", arch="arm64"),
            )

    def test_create_workload_rejects_platform_conflict_with_template_node_affinity(
        self, mock_k8s_client, tmp_path
    ):
        template_file = tmp_path / "agent_template.yaml"
        template_file.write_text(
            """
spec:
  podTemplate:
    spec:
      affinity:
        nodeAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
            nodeSelectorTerms:
              - matchExpressions:
                  - key: kubernetes.io/arch
                    operator: In
                    values: ["amd64"]
"""
        )
        app_config = _app_config()
        app_config.agent_sandbox.template_file = str(template_file)
        provider = AgentSandboxProvider(mock_k8s_client, app_config)

        with pytest.raises(ValueError, match="platform conflict with template nodeAffinity"):
            provider.create_workload(
                sandbox_id="test-id",
                namespace="test-ns",
                image_spec=ImageSpec(uri="python:3.11"),
                entrypoint=["/bin/bash"],
                env={},
                resource_limits={"cpu": "1", "memory": "1Gi"},
                labels={"opensandbox.io/id": "test-id"},
                expires_at=None,
                execd_image="execd:latest",
                platform=PlatformSpec(os="linux", arch="arm64"),
            )

    def test_create_workload_sanitizes_resource_name(self, mock_k8s_client):
        provider = AgentSandboxProvider(mock_k8s_client)
        mock_k8s_client.create_custom_object.return_value = {
            "metadata": {"name": "sandbox-1234", "uid": "test-uid"}
        }

        expires_at = datetime(2025, 12, 31, 10, 0, 0, tzinfo=timezone.utc)

        result = provider.create_workload(
            sandbox_id="1234",
            namespace="test-ns",
            image_spec=ImageSpec(uri="python:3.11"),
            entrypoint=["/bin/bash"],
            env={"FOO": "bar"},
            resource_limits={"cpu": "1", "memory": "1Gi"},
            labels={"opensandbox.io/id": "1234"},
            expires_at=expires_at,
            execd_image="execd:latest",
        )

        assert result == {"name": "sandbox-1234", "uid": "test-uid", "apiVersion": "agents.x-k8s.io/v1beta1", "kind": "Sandbox"}
        body = mock_k8s_client.create_custom_object.call_args.kwargs["body"]
        assert body["metadata"]["name"] == "sandbox-1234"

    def test_resource_name_uses_hash_when_id_has_no_alnum(self, mock_k8s_client):
        provider = AgentSandboxProvider(mock_k8s_client)

        first = provider._resource_name("!!!")
        second = provider._resource_name("???")

        assert first.startswith("sandbox-")
        assert second.startswith("sandbox-")
        assert first != second

    def test_get_workload_returns_none_on_404(self, mock_k8s_client):
        provider = AgentSandboxProvider(mock_k8s_client)
        mock_k8s_client.get_custom_object.return_value = None

        result = provider.get_workload("test-id", "test-ns")

        assert result is None

    def test_get_workload_prefers_sanitized_name(self, mock_k8s_client):
        provider = AgentSandboxProvider(mock_k8s_client)
        mock_k8s_client.get_custom_object.side_effect = [
            None,
            {"metadata": {"name": "1234"}},
        ]

        result = provider.get_workload("1234", "test-ns")

        assert result["metadata"]["name"] == "1234"
        assert mock_k8s_client.get_custom_object.call_args_list[0].kwargs["name"] == "sandbox-1234"
        assert mock_k8s_client.get_custom_object.call_args_list[1].kwargs["name"] == "1234"

    def test_get_workload_falls_back_to_legacy_name(self, mock_k8s_client):
        provider = AgentSandboxProvider(mock_k8s_client)
        mock_k8s_client.get_custom_object.side_effect = [
            None,
            {"metadata": {"name": "sandbox-test-id"}},
        ]

        result = provider.get_workload("test-id", "test-ns")

        assert result["metadata"]["name"] == "sandbox-test-id"
        assert mock_k8s_client.get_custom_object.call_args_list[0].kwargs["name"] == "test-id"
        assert (
            mock_k8s_client.get_custom_object.call_args_list[1].kwargs["name"] == "sandbox-test-id"
        )

    def test_get_workload_reraises_non_404_exceptions(self, mock_k8s_client):
        provider = AgentSandboxProvider(mock_k8s_client)
        mock_k8s_client.get_custom_object.side_effect = ApiException(status=500)

        with pytest.raises(ApiException) as exc_info:
            provider.get_workload("test-id", "test-ns")

        assert exc_info.value.status == 500

    def test_get_workload_prefers_informer_cache(self, mock_k8s_client):
        cached = {"metadata": {"name": "test-id"}}
        mock_k8s_client.get_custom_object.return_value = cached

        provider = AgentSandboxProvider(mock_k8s_client)

        result = provider.get_workload("test-id", "test-ns")

        assert result == cached
        mock_k8s_client.get_custom_object.assert_called()

    def test_create_workload_updates_informer_cache(self, mock_k8s_client):
        created_body = {"metadata": {"name": "test-id", "uid": "test-uid"}}
        mock_k8s_client.create_custom_object.return_value = created_body

        provider = AgentSandboxProvider(mock_k8s_client)

        expires_at = datetime(2025, 12, 31, 10, 0, 0, tzinfo=timezone.utc)

        result = provider.create_workload(
            sandbox_id="test-id",
            namespace="test-ns",
            image_spec=ImageSpec(uri="python:3.11"),
            entrypoint=["/bin/bash"],
            env={"FOO": "bar"},
            resource_limits={"cpu": "1", "memory": "1Gi"},
            labels={"opensandbox.io/id": "test-id"},
            expires_at=expires_at,
            execd_image="execd:latest",
        )

        assert result == {"name": "test-id", "uid": "test-uid", "apiVersion": "agents.x-k8s.io/v1beta1", "kind": "Sandbox"}

    def test_update_expiration_patches_spec(self, mock_k8s_client):
        provider = AgentSandboxProvider(mock_k8s_client)
        mock_k8s_client.get_custom_object.return_value = {"metadata": {"name": "sandbox-test-id"}}

        expires_at = datetime(2025, 12, 31, 0, 0, 0, tzinfo=timezone.utc)
        provider.update_expiration("test-id", "test-ns", expires_at)

        call_kwargs = mock_k8s_client.patch_custom_object.call_args.kwargs
        assert call_kwargs["body"] == {"spec": {"shutdownTime": "2025-12-31T00:00:00+00:00"}}

    def test_get_expiration_parses_z_suffix(self):
        provider = AgentSandboxProvider(MagicMock())
        workload = {"spec": {"shutdownTime": "2025-12-31T10:00:00Z"}}

        result = provider.get_expiration(workload)

        assert result == datetime(2025, 12, 31, 10, 0, 0, tzinfo=timezone.utc)

    def test_get_status_ready_condition_true(self):
        provider = AgentSandboxProvider(MagicMock())
        workload = {
            "status": {
                "conditions": [
                    {
                        "type": "Ready",
                        "status": "True",
                        "reason": "SandboxReady",
                        "message": "Ready",
                        "lastTransitionTime": "2025-12-31T10:00:00Z",
                    }
                ]
            },
            "metadata": {"creationTimestamp": "2025-12-31T09:00:00Z"},
        }

        result = provider.get_status(workload)

        assert result["state"] == "Running"
        assert result["reason"] == "SandboxReady"
        assert result["message"] == "Ready"

    def test_get_status_suspended_condition_true_maps_to_paused(self):
        provider = AgentSandboxProvider(MagicMock())
        workload = {
            "spec": {"operatingMode": "Suspended"},
            "status": {
                "conditions": [
                    {
                        "type": "Suspended",
                        "status": "True",
                        "reason": "PodTerminated",
                        "message": "Sandbox pod has been terminated",
                        "lastTransitionTime": "2025-12-31T10:05:00Z",
                    },
                    {
                        "type": "Ready",
                        "status": "False",
                        "reason": "NotReady",
                        "message": "Sandbox is suspended",
                        "lastTransitionTime": "2025-12-31T10:00:00Z",
                    },
                ]
            },
            "metadata": {"creationTimestamp": "2025-12-31T09:00:00Z"},
        }

        result = provider.get_status(workload)

        assert result["state"] == "Paused"
        assert result["reason"] == "PodTerminated"
        assert result["message"] == "Sandbox pod has been terminated"
        assert result["last_transition_at"] == "2025-12-31T10:05:00Z"

    def test_get_status_suspended_true_with_running_mode_maps_to_resuming(self):
        provider = AgentSandboxProvider(MagicMock())
        workload = {
            "spec": {"operatingMode": "Running"},
            "status": {
                "conditions": [
                    {
                        "type": "Suspended",
                        "status": "True",
                        "reason": "PodTerminated",
                        "message": "Sandbox pod has been terminated",
                        "lastTransitionTime": "2025-12-31T10:05:00Z",
                    }
                ]
            },
            "metadata": {"creationTimestamp": "2025-12-31T09:00:00Z"},
        }

        result = provider.get_status(workload)

        assert result["state"] == "Resuming"
        assert result["message"] == "Sandbox is resuming"

    def test_get_status_expired_outranks_suspended_condition(self):
        provider = AgentSandboxProvider(MagicMock())
        workload = {
            "spec": {"operatingMode": "Suspended"},
            "status": {
                "conditions": [
                    {
                        "type": "Suspended",
                        "status": "True",
                        "reason": "PodTerminated",
                        "message": "Sandbox pod has been terminated",
                        "lastTransitionTime": "2025-12-31T10:05:00Z",
                    },
                    {
                        "type": "Ready",
                        "status": "False",
                        "reason": "SandboxExpired",
                        "message": "Sandbox has expired",
                        "lastTransitionTime": "2025-12-31T11:00:00Z",
                    },
                ]
            },
            "metadata": {"creationTimestamp": "2025-12-31T09:00:00Z"},
        }

        result = provider.get_status(workload)

        assert result["state"] == "Terminated"
        assert result["reason"] == "SandboxExpired"

    def test_get_status_suspended_condition_takes_precedence_over_ready_true(self):
        provider = AgentSandboxProvider(MagicMock())
        workload = {
            "spec": {"operatingMode": "Suspended"},
            "status": {
                "conditions": [
                    {
                        "type": "Ready",
                        "status": "True",
                        "reason": "SandboxReady",
                        "message": "Ready",
                        "lastTransitionTime": "2025-12-31T10:00:00Z",
                    },
                    {
                        "type": "Suspended",
                        "status": "True",
                        "reason": "PodTerminated",
                        "message": "Sandbox pod has been terminated",
                        "lastTransitionTime": "2025-12-31T10:05:00Z",
                    },
                ]
            },
            "metadata": {"creationTimestamp": "2025-12-31T09:00:00Z"},
        }

        result = provider.get_status(workload)

        assert result["state"] == "Paused"

    def test_get_status_operating_mode_suspended_condition_not_true_maps_to_pausing(self):
        provider = AgentSandboxProvider(MagicMock())
        workload = {
            "spec": {"operatingMode": "Suspended"},
            "status": {
                "conditions": [
                    {
                        "type": "Suspended",
                        "status": "False",
                        "reason": "PodTerminating",
                        "message": "Sandbox pod is terminating",
                        "lastTransitionTime": "2025-12-31T10:05:00Z",
                    }
                ]
            },
            "metadata": {"creationTimestamp": "2025-12-31T09:00:00Z"},
        }

        result = provider.get_status(workload)

        assert result["state"] == "Pausing"
        assert result["reason"] == "PodTerminating"
        assert result["message"] == "Sandbox pod is terminating"
        assert result["last_transition_at"] == "2025-12-31T10:05:00Z"

    def test_get_status_operating_mode_suspended_without_condition_maps_to_pausing(self):
        provider = AgentSandboxProvider(MagicMock())
        workload = {
            "spec": {"operatingMode": "Suspended"},
            "status": {"conditions": []},
            "metadata": {"creationTimestamp": "2025-12-31T09:00:00Z"},
        }

        result = provider.get_status(workload)

        assert result["state"] == "Pausing"
        assert result["reason"] is None
        assert result["message"] == "Pausing sandbox"

    def test_get_status_operating_mode_running_with_suspended_false_uses_ready_mapping(self):
        provider = AgentSandboxProvider(MagicMock())
        workload = {
            "spec": {"operatingMode": "Running"},
            "status": {
                "conditions": [
                    {
                        "type": "Suspended",
                        "status": "False",
                        "reason": "NotSuspended",
                        "message": "",
                        "lastTransitionTime": "2025-12-31T09:30:00Z",
                    },
                    {
                        "type": "Ready",
                        "status": "True",
                        "reason": "SandboxReady",
                        "message": "Ready",
                        "lastTransitionTime": "2025-12-31T10:00:00Z",
                    },
                ]
            },
            "metadata": {"creationTimestamp": "2025-12-31T09:00:00Z"},
        }

        result = provider.get_status(workload)

        assert result["state"] == "Running"
        assert result["reason"] == "SandboxReady"

    def test_get_status_expired_condition(self):
        provider = AgentSandboxProvider(MagicMock())
        workload = {
            "status": {
                "conditions": [
                    {
                        "type": "Ready",
                        "status": "False",
                        "reason": "SandboxExpired",
                        "message": "Expired",
                        "lastTransitionTime": "2025-12-31T10:00:00Z",
                    }
                ]
            },
            "metadata": {"creationTimestamp": "2025-12-31T09:00:00Z"},
        }

        result = provider.get_status(workload)

        assert result["state"] == "Terminated"
        assert result["reason"] == "SandboxExpired"

    def test_get_status_pod_succeeded_maps_to_terminated(self):
        provider = AgentSandboxProvider(MagicMock())
        workload = {
            "status": {
                "conditions": [
                    {
                        "type": "Ready",
                        "status": "False",
                        "reason": "PodSucceeded",
                        "message": "Pod completed successfully",
                        "lastTransitionTime": "2025-12-31T10:00:00Z",
                    }
                ]
            },
            "metadata": {"creationTimestamp": "2025-12-31T09:00:00Z"},
        }

        result = provider.get_status(workload)

        assert result["state"] == "Terminated"
        assert result["reason"] == "PodSucceeded"

    def test_get_status_pod_failed_maps_to_failed(self):
        provider = AgentSandboxProvider(MagicMock())
        workload = {
            "status": {
                "conditions": [
                    {
                        "type": "Ready",
                        "status": "False",
                        "reason": "PodFailed",
                        "message": "Pod failed",
                        "lastTransitionTime": "2025-12-31T10:00:00Z",
                    }
                ]
            },
            "metadata": {"creationTimestamp": "2025-12-31T09:00:00Z"},
        }

        result = provider.get_status(workload)

        assert result["state"] == "Failed"
        assert result["reason"] == "PodFailed"

    def test_get_status_falls_back_to_pod_state(self, mock_k8s_client):
        provider = AgentSandboxProvider(mock_k8s_client)
        mock_k8s_client.list_pods.return_value = [
            SimpleNamespace(status=SimpleNamespace(phase="Running", pod_ip="10.0.0.2"))
        ]
        workload = {
            "status": {"conditions": [], "selector": "app=sandbox"},
            "metadata": {"creationTimestamp": "2025-12-31T09:00:00Z", "namespace": "test-ns"},
        }

        result = provider.get_status(workload)

        assert result["state"] == "Running"
        assert result["reason"] == "POD_READY"

    def test_get_status_falls_back_to_allocated_when_ip_assigned_not_running(self, mock_k8s_client):
        provider = AgentSandboxProvider(mock_k8s_client)
        mock_k8s_client.list_pods.return_value = [
            SimpleNamespace(status=SimpleNamespace(phase="Pending", pod_ip="10.0.0.2"))
        ]
        workload = {
            "status": {"conditions": [], "selector": "app=sandbox"},
            "metadata": {"creationTimestamp": "2025-12-31T09:00:00Z", "namespace": "test-ns"},
        }

        result = provider.get_status(workload)

        assert result["state"] == "Allocated"
        assert result["reason"] == "IP_ASSIGNED"

    def test_get_status_returns_failed_when_ready_condition_unschedulable(self):
        provider = AgentSandboxProvider(MagicMock())
        workload = {
            "spec": {
                "podTemplate": {
                    "spec": {
                        "nodeSelector": {
                            "kubernetes.io/os": "linux",
                            "kubernetes.io/arch": "arm64",
                        }
                    }
                }
            },
            "status": {
                "conditions": [
                    {
                        "type": "Ready",
                        "status": "False",
                        "reason": "FailedScheduling",
                        "message": "0/1 nodes are available: 1 node(s) didn't match Pod's node affinity.",
                        "lastTransitionTime": "2025-12-31T10:00:00Z",
                    }
                ]
            },
            "metadata": {"creationTimestamp": "2025-12-31T09:00:00Z"},
        }

        result = provider.get_status(workload)

        assert result["state"] == "Failed"
        assert result["reason"] == "POD_PLATFORM_UNSCHEDULABLE"
        assert "nodes are available" in result["message"]

    def test_get_status_keeps_pending_for_generic_scheduler_backpressure(self, mock_k8s_client):
        provider = AgentSandboxProvider(mock_k8s_client)
        mock_k8s_client.list_pods.return_value = [
            SimpleNamespace(
                status=SimpleNamespace(
                    phase="Pending",
                    pod_ip=None,
                    conditions=[
                        SimpleNamespace(
                            type="PodScheduled",
                            status="False",
                            reason="Unschedulable",
                            message="0/1 nodes are available: 1 Insufficient cpu.",
                        )
                    ],
                )
            )
        ]
        workload = {
            "status": {"conditions": [], "selector": "app=sandbox"},
            "metadata": {"creationTimestamp": "2025-12-31T09:00:00Z", "namespace": "test-ns"},
        }

        result = provider.get_status(workload)

        assert result["state"] == "Pending"
        assert result["reason"] in {"POD_SCHEDULED", "POD_PENDING"}

    def test_get_status_keeps_pending_when_non_platform_affinity_mismatch(self, mock_k8s_client):
        provider = AgentSandboxProvider(mock_k8s_client)
        mock_k8s_client.list_pods.return_value = [
            SimpleNamespace(
                status=SimpleNamespace(
                    phase="Pending",
                    pod_ip=None,
                    conditions=[
                        SimpleNamespace(
                            type="PodScheduled",
                            status="False",
                            reason="Unschedulable",
                            message="0/1 nodes are available: 1 node(s) didn't match Pod's node affinity.",
                        )
                    ],
                )
            )
        ]
        workload = {
            "spec": {
                "podTemplate": {
                    "spec": {
                        "nodeSelector": {
                            "kubernetes.io/os": "linux",
                            "kubernetes.io/arch": "arm64",
                            "workload-class": "gpu",
                        }
                    }
                }
            },
            "status": {"conditions": [], "selector": "app=sandbox"},
            "metadata": {"creationTimestamp": "2025-12-31T09:00:00Z", "namespace": "test-ns"},
        }

        result = provider.get_status(workload)

        assert result["state"] == "Pending"
        assert result["reason"] in {"POD_SCHEDULED", "POD_PENDING"}

    def test_get_status_keeps_pending_for_mixed_capacity_and_affinity_message(
        self, mock_k8s_client
    ):
        provider = AgentSandboxProvider(mock_k8s_client)
        mock_k8s_client.list_pods.return_value = [
            SimpleNamespace(
                status=SimpleNamespace(
                    phase="Pending",
                    pod_ip=None,
                    conditions=[
                        SimpleNamespace(
                            type="PodScheduled",
                            status="False",
                            reason="Unschedulable",
                            message=(
                                "0/2 nodes are available: 1 Insufficient cpu, "
                                "1 node(s) didn't match Pod's node affinity/selector."
                            ),
                        )
                    ],
                )
            )
        ]
        workload = {
            "spec": {
                "podTemplate": {
                    "spec": {
                        "nodeSelector": {
                            "kubernetes.io/os": "linux",
                            "kubernetes.io/arch": "arm64",
                        }
                    }
                }
            },
            "status": {"conditions": [], "selector": "app=sandbox"},
            "metadata": {"creationTimestamp": "2025-12-31T09:00:00Z", "namespace": "test-ns"},
        }

        result = provider.get_status(workload)

        assert result["state"] == "Pending"
        assert result["reason"] in {"POD_SCHEDULED", "POD_PENDING"}

    def test_get_internal_endpoint_uses_first_valid_status_pod_ip(self, mock_k8s_client):
        provider = AgentSandboxProvider(mock_k8s_client)
        workload = {"status": {"podIPs": ["10.0.0.9", "fd00::9"]}}

        endpoint = provider.get_internal_endpoint(workload, 8080, "sandbox-123")

        assert endpoint.endpoint == "10.0.0.9:8080"
        assert endpoint.headers is None
        mock_k8s_client.list_pods.assert_not_called()

    @pytest.mark.parametrize(
        "workload",
        [
            {},
            {"status": {}},
            {"status": None},
            {"status": []},
            {"status": {"podIPs": []}},
            {"status": {"podIPs": "10.0.0.9"}},
            {"status": {"podIPs": [None]}},
            {"status": {"podIPs": [""]}},
        ],
    )
    def test_get_internal_endpoint_returns_none_without_valid_primary_pod_ip(
        self, mock_k8s_client, workload
    ):
        provider = AgentSandboxProvider(mock_k8s_client)

        endpoint = provider.get_internal_endpoint(workload, 8080, "sandbox-123")

        assert endpoint is None
        mock_k8s_client.list_pods.assert_not_called()

    def test_get_internal_endpoint_skips_invalid_pod_ip_entries(self, mock_k8s_client):
        provider = AgentSandboxProvider(mock_k8s_client)
        workload = {"status": {"podIPs": [None, "", "10.0.0.10"]}}

        endpoint = provider.get_internal_endpoint(workload, 8080, "sandbox-123")

        assert endpoint.endpoint == "10.0.0.10:8080"
        assert endpoint.headers is None
        mock_k8s_client.list_pods.assert_not_called()

    def test_get_internal_endpoint_brackets_ipv6_pod_ip(self, mock_k8s_client):
        provider = AgentSandboxProvider(mock_k8s_client)
        workload = {"status": {"podIPs": ["fd00::9"]}}

        endpoint = provider.get_internal_endpoint(workload, 8080, "sandbox-123")

        assert endpoint.endpoint == "[fd00::9]:8080"
        assert endpoint.headers is None
        mock_k8s_client.list_pods.assert_not_called()

    def test_get_endpoint_info_prefers_running_pod(self, mock_k8s_client):
        provider = AgentSandboxProvider(mock_k8s_client)
        mock_k8s_client.list_pods.return_value = [
            SimpleNamespace(status=SimpleNamespace(phase="Running", pod_ip="10.0.0.9"))
        ]
        workload = {
            "status": {"selector": "app=sandbox"},
            "metadata": {"namespace": "test-ns"},
        }

        endpoint = provider.get_endpoint_info(workload, 8080, "sandbox-123")

        assert endpoint.endpoint == "10.0.0.9:8080"
        assert endpoint.headers is None

    def test_get_endpoint_info_falls_back_to_service_fqdn(self, mock_k8s_client):
        provider = AgentSandboxProvider(mock_k8s_client)
        mock_k8s_client.list_pods.side_effect = Exception("boom")
        workload = {
            "status": {"selector": "app=sandbox", "serviceFQDN": "svc.example.com"},
            "metadata": {"namespace": "test-ns"},
        }

        endpoint = provider.get_endpoint_info(workload, 9000, "sandbox-123")

        assert endpoint.endpoint == "svc.example.com:9000"
        assert endpoint.headers is None

    # ===== Pause / Resume Tests =====

    def _running_sandbox(self) -> dict:
        return {
            "metadata": {"name": "test-id", "namespace": "test-ns"},
            "spec": {"operatingMode": "Running"},
            "status": {
                "conditions": [
                    {
                        "type": "Ready",
                        "status": "True",
                        "reason": "SandboxReady",
                        "message": "Ready",
                        "lastTransitionTime": "2025-12-31T10:00:00Z",
                    }
                ]
            },
        }

    def _paused_sandbox(self) -> dict:
        return {
            "metadata": {"name": "test-id", "namespace": "test-ns"},
            "spec": {"operatingMode": "Suspended"},
            "status": {
                "conditions": [
                    {
                        "type": "Suspended",
                        "status": "True",
                        "reason": "PodTerminated",
                        "message": "Sandbox pod has been terminated",
                        "lastTransitionTime": "2025-12-31T10:05:00Z",
                    }
                ]
            },
        }

    def _pausing_sandbox(self) -> dict:
        return {
            "metadata": {"name": "test-id", "namespace": "test-ns"},
            "spec": {"operatingMode": "Suspended"},
            "status": {
                "conditions": [
                    {
                        "type": "Suspended",
                        "status": "False",
                        "reason": "PodTerminating",
                        "message": "Sandbox pod is terminating",
                        "lastTransitionTime": "2025-12-31T10:05:00Z",
                    }
                ]
            },
        }

    def _pending_sandbox(self) -> dict:
        return {
            "metadata": {"name": "test-id", "namespace": "test-ns"},
            "spec": {"operatingMode": "Running"},
            "status": {"conditions": []},
        }

    def test_pause_sandbox_running_allows(self, mock_k8s_client):
        """Pause allowed when Ready=True; patches spec.operatingMode=Suspended."""
        provider = AgentSandboxProvider(mock_k8s_client)
        mock_k8s_client.get_custom_object.return_value = self._running_sandbox()
        mock_k8s_client.patch_custom_object.return_value = {}

        provider.pause_sandbox("test-id", "test-ns")

        mock_k8s_client.patch_custom_object.assert_called_once()
        call_kwargs = mock_k8s_client.patch_custom_object.call_args.kwargs
        assert call_kwargs["group"] == "agents.x-k8s.io"
        assert call_kwargs["version"] == "v1beta1"
        assert call_kwargs["plural"] == "sandboxes"
        assert call_kwargs["name"] == "test-id"
        assert call_kwargs["body"] == {"spec": {"operatingMode": "Suspended"}}

    @pytest.mark.parametrize(
        "workload_builder,expected_match",
        [
            ("_paused_sandbox", "already paused"),
            ("_pausing_sandbox", "operation in progress"),
            ("_pending_sandbox", "Cannot pause sandbox in state Pending, expected Running"),
        ],
        ids=["already-paused", "pausing", "pending"],
    )
    def test_pause_sandbox_rejects_non_running_state(self, mock_k8s_client, workload_builder, expected_match):
        """Pause is only allowed from the Running state; no patch may be issued."""
        provider = AgentSandboxProvider(mock_k8s_client)
        mock_k8s_client.get_custom_object.return_value = getattr(self, workload_builder)()

        with pytest.raises(ValueError, match=expected_match):
            provider.pause_sandbox("test-id", "test-ns")

        mock_k8s_client.patch_custom_object.assert_not_called()

    def test_pause_sandbox_terminated_rejects(self, mock_k8s_client):
        """Pause rejected with the public state name when Terminated."""
        provider = AgentSandboxProvider(mock_k8s_client)
        mock_k8s_client.get_custom_object.return_value = {
            "metadata": {"name": "test-id", "namespace": "test-ns"},
            "spec": {"operatingMode": "Running"},
            "status": {
                "conditions": [
                    {
                        "type": "Ready",
                        "status": "False",
                        "reason": "SandboxExpired",
                        "message": "Expired",
                        "lastTransitionTime": "2025-12-31T10:00:00Z",
                    }
                ]
            },
        }

        with pytest.raises(ValueError, match="state Terminated"):
            provider.pause_sandbox("test-id", "test-ns")

    def test_pause_sandbox_not_found(self, mock_k8s_client):
        """Pause raises ValueError with 'not found' so the service maps it to 404."""
        provider = AgentSandboxProvider(mock_k8s_client)
        mock_k8s_client.get_custom_object.return_value = None

        with pytest.raises(ValueError, match="not found"):
            provider.pause_sandbox("test-id", "test-ns")

    def test_pause_patch_after_delete_maps_to_not_found(self, mock_k8s_client):
        """Patch 404 (CR deleted since the read) maps to the public not-found error."""
        provider = AgentSandboxProvider(mock_k8s_client)
        mock_k8s_client.get_custom_object.return_value = self._running_sandbox()
        mock_k8s_client.patch_custom_object.side_effect = ApiException(status=404)

        with pytest.raises(ValueError, match="not found"):
            provider.pause_sandbox("test-id", "test-ns")

    def test_resume_sandbox_paused_allows(self, mock_k8s_client):
        """Resume allowed when the sandbox is paused; patches spec.operatingMode=Running."""
        provider = AgentSandboxProvider(mock_k8s_client)
        mock_k8s_client.get_custom_object.return_value = self._paused_sandbox()
        mock_k8s_client.patch_custom_object.return_value = {}

        provider.resume_sandbox("test-id", "test-ns")

        mock_k8s_client.patch_custom_object.assert_called_once()
        call_kwargs = mock_k8s_client.patch_custom_object.call_args.kwargs
        assert call_kwargs["group"] == "agents.x-k8s.io"
        assert call_kwargs["version"] == "v1beta1"
        assert call_kwargs["plural"] == "sandboxes"
        assert call_kwargs["name"] == "test-id"
        assert call_kwargs["body"] == {"spec": {"operatingMode": "Running"}}

    @pytest.mark.parametrize(
        "workload_builder,expected_match",
        [
            ("_running_sandbox", "Cannot resume sandbox in state Running, expected Paused"),
            ("_pausing_sandbox", "operation in progress"),
            ("_pending_sandbox", "state Pending"),
        ],
        ids=["running", "pausing", "pending"],
    )
    def test_resume_sandbox_rejects_non_paused_state(self, mock_k8s_client, workload_builder, expected_match):
        """Resume is only allowed from the Paused state; no patch may be issued."""
        provider = AgentSandboxProvider(mock_k8s_client)
        mock_k8s_client.get_custom_object.return_value = getattr(self, workload_builder)()

        with pytest.raises(ValueError, match=expected_match):
            provider.resume_sandbox("test-id", "test-ns")

        mock_k8s_client.patch_custom_object.assert_not_called()

    def test_resume_sandbox_not_found(self, mock_k8s_client):
        """Resume raises ValueError with 'not found' so the service maps it to 404."""
        provider = AgentSandboxProvider(mock_k8s_client)
        mock_k8s_client.get_custom_object.return_value = None

        with pytest.raises(ValueError, match="not found"):
            provider.resume_sandbox("test-id", "test-ns")

    def test_resume_patch_after_delete_maps_to_not_found(self, mock_k8s_client):
        """Patch 404 (CR deleted since the read) maps to the public not-found error."""
        provider = AgentSandboxProvider(mock_k8s_client)
        mock_k8s_client.get_custom_object.return_value = self._paused_sandbox()
        mock_k8s_client.patch_custom_object.side_effect = ApiException(status=404)

        with pytest.raises(ValueError, match="not found"):
            provider.resume_sandbox("test-id", "test-ns")


    # ===== Image Auth / Pull Secrets Tests =====

    def test_supports_image_auth_returns_true(self, mock_k8s_client):
        provider = AgentSandboxProvider(mock_k8s_client, _app_config())
        assert provider.supports_image_auth() is True

    def test_create_workload_with_image_auth_injects_image_pull_secrets(
        self, mock_k8s_client
    ):
        provider = AgentSandboxProvider(mock_k8s_client, _app_config())
        mock_k8s_client.create_custom_object.return_value = {
            "metadata": {"name": "test-id", "uid": "uid-123"}
        }

        provider.create_workload(
            sandbox_id="test-id",
            namespace="test-ns",
            image_spec=ImageSpec(
                uri="registry.example.com/img:tag",
                auth=ImageAuth(username="user", password="pass"),
            ),
            entrypoint=["/bin/bash"],
            env={},
            resource_limits={},
            labels={},
            expires_at=datetime(2025, 12, 31, tzinfo=timezone.utc),
            execd_image="execd:latest",
        )

        body = mock_k8s_client.create_custom_object.call_args.kwargs["body"]
        pull_secrets = body["spec"]["podTemplate"]["spec"].get("imagePullSecrets")
        assert pull_secrets == [{"name": f"{IMAGE_AUTH_SECRET_PREFIX}-test-id"}]

    def test_create_workload_with_image_auth_preserves_template_pull_secrets(
        self, mock_k8s_client, tmp_path
    ):
        # The template merge replaces lists wholesale, so the per-request
        # secret must be appended to the merged spec, not assigned to the
        # pre-merge pod spec — otherwise template-provided imagePullSecrets
        # (e.g. for pulling a private execd image) are dropped.
        template_file = tmp_path / "agent_template.yaml"
        template_file.write_text(
            """
spec:
  podTemplate:
    spec:
      imagePullSecrets:
        - name: template-regcred
"""
        )
        app_config = _app_config()
        app_config.agent_sandbox.template_file = str(template_file)
        provider = AgentSandboxProvider(mock_k8s_client, app_config)
        mock_k8s_client.create_custom_object.return_value = {
            "metadata": {"name": "test-id", "uid": "uid-123"}
        }

        provider.create_workload(
            sandbox_id="test-id",
            namespace="test-ns",
            image_spec=ImageSpec(
                uri="registry.example.com/img:tag",
                auth=ImageAuth(username="user", password="pass"),
            ),
            entrypoint=["/bin/bash"],
            env={},
            resource_limits={},
            labels={},
            expires_at=datetime(2025, 12, 31, tzinfo=timezone.utc),
            execd_image="execd:latest",
        )

        body = mock_k8s_client.create_custom_object.call_args.kwargs["body"]
        pull_secrets = body["spec"]["podTemplate"]["spec"]["imagePullSecrets"]
        assert pull_secrets == [
            {"name": "template-regcred"},
            {"name": f"{IMAGE_AUTH_SECRET_PREFIX}-test-id"},
        ]

    def test_create_workload_with_image_auth_creates_secret(self, mock_k8s_client):
        # CR name gets a "sandbox-" prefix for digit-leading ids; the Secret's
        # ownerReference must carry the CR name (not sandbox_id) or K8s GC
        # deletes the Secret while the sandbox is still running.
        provider = AgentSandboxProvider(mock_k8s_client, _app_config())
        mock_k8s_client.create_custom_object.return_value = {
            "metadata": {"name": "sandbox-test-id", "uid": "uid-abc"}
        }

        provider.create_workload(
            sandbox_id="test-id",
            namespace="test-ns",
            image_spec=ImageSpec(
                uri="registry.example.com/img:tag",
                auth=ImageAuth(username="user", password="pass"),
            ),
            entrypoint=["/bin/bash"],
            env={},
            resource_limits={},
            labels={},
            expires_at=datetime(2025, 12, 31, tzinfo=timezone.utc),
            execd_image="execd:latest",
        )

        mock_k8s_client.create_secret.assert_called_once()
        call_kwargs = mock_k8s_client.create_secret.call_args.kwargs
        assert call_kwargs["namespace"] == "test-ns"
        secret = call_kwargs["body"]
        assert secret.type == "kubernetes.io/dockerconfigjson"
        assert secret.metadata.name == f"{IMAGE_AUTH_SECRET_PREFIX}-test-id"
        ref = secret.metadata.owner_references[0]
        assert ref.uid == "uid-abc"
        assert ref.kind == "Sandbox"
        assert ref.name == "sandbox-test-id"

    def test_create_workload_without_image_auth_skips_secret(self, mock_k8s_client):
        provider = AgentSandboxProvider(mock_k8s_client, _app_config())
        mock_k8s_client.create_custom_object.return_value = {
            "metadata": {"name": "test-id", "uid": "uid-123"}
        }

        provider.create_workload(
            sandbox_id="test-id",
            namespace="test-ns",
            image_spec=ImageSpec(uri="python:3.11"),
            entrypoint=["/bin/bash"],
            env={},
            resource_limits={},
            labels={},
            expires_at=datetime(2025, 12, 31, tzinfo=timezone.utc),
            execd_image="execd:latest",
        )

        mock_k8s_client.create_secret.assert_not_called()
        body = mock_k8s_client.create_custom_object.call_args.kwargs["body"]
        assert "imagePullSecrets" not in body["spec"]["podTemplate"]["spec"]

    def test_create_workload_with_image_auth_secret_failure_rolls_back_sandbox(
        self, mock_k8s_client
    ):
        provider = AgentSandboxProvider(mock_k8s_client, _app_config())
        mock_k8s_client.create_custom_object.return_value = {
            "metadata": {"name": "test-id", "uid": "uid-123"}
        }
        mock_k8s_client.create_secret.side_effect = ApiException(status=403)

        with pytest.raises(ApiException):
            provider.create_workload(
                sandbox_id="test-id",
                namespace="test-ns",
                image_spec=ImageSpec(
                    uri="registry.example.com/img:tag",
                    auth=ImageAuth(username="user", password="pass"),
                ),
                entrypoint=["/bin/bash"],
                env={},
                resource_limits={},
                labels={},
                expires_at=datetime(2025, 12, 31, tzinfo=timezone.utc),
                execd_image="execd:latest",
            )

        mock_k8s_client.delete_custom_object.assert_called_once_with(
            group=provider.group,
            version=provider.version,
            namespace="test-ns",
            plural=provider.plural,
            name="test-id",
            grace_period_seconds=0,
        )

class TestAgentSandboxProviderExecdInit:
    """AgentSandboxProvider execd init container resource tests"""

    def test_init_container_has_no_resources_when_not_configured(self, mock_k8s_client):
        provider = AgentSandboxProvider(mock_k8s_client)
        mock_k8s_client.create_custom_object.return_value = {
            "metadata": {"name": "test-id", "uid": "test-uid"}
        }

        provider.create_workload(
            sandbox_id="test-id",
            namespace="test-ns",
            image_spec=ImageSpec(uri="python:3.11"),
            entrypoint=["/bin/bash"],
            env={},
            resource_limits={},
            labels={},
            expires_at=datetime(2025, 12, 31, 10, 0, 0, tzinfo=timezone.utc),
            execd_image="execd:latest",
        )

        body = mock_k8s_client.create_custom_object.call_args.kwargs["body"]
        init_containers = body["spec"]["podTemplate"]["spec"]["initContainers"]
        assert len(init_containers) == 1
        assert "resources" not in init_containers[0]
        init_script = init_containers[0]["args"][0]
        assert (
            "cp /usr/local/libexec/opensandbox-session-gate "
            "/opt/opensandbox/opensandbox-session-gate"
        ) in init_script

    def test_init_container_has_resources_when_configured(self, mock_k8s_client):
        provider = AgentSandboxProvider(
            mock_k8s_client,
            _app_config(
                execd_init_resources=ExecdInitResources(
                    limits={"cpu": "100m", "memory": "128Mi"},
                    requests={"cpu": "50m", "memory": "64Mi"},
                )
            ),
        )
        mock_k8s_client.create_custom_object.return_value = {
            "metadata": {"name": "test-id", "uid": "test-uid"}
        }

        provider.create_workload(
            sandbox_id="test-id",
            namespace="test-ns",
            image_spec=ImageSpec(uri="python:3.11"),
            entrypoint=["/bin/bash"],
            env={},
            resource_limits={},
            labels={},
            expires_at=datetime(2025, 12, 31, 10, 0, 0, tzinfo=timezone.utc),
            execd_image="execd:latest",
        )

        body = mock_k8s_client.create_custom_object.call_args.kwargs["body"]
        init_containers = body["spec"]["podTemplate"]["spec"]["initContainers"]
        assert init_containers[0]["resources"]["limits"] == {"cpu": "100m", "memory": "128Mi"}
        assert init_containers[0]["resources"]["requests"] == {"cpu": "50m", "memory": "64Mi"}


class TestAgentSandboxProviderEgress:
    """AgentSandboxProvider egress sidecar tests"""

    def test_create_workload_without_network_policy_no_sidecar(self, mock_k8s_client):
        provider = AgentSandboxProvider(mock_k8s_client)
        mock_k8s_client.create_custom_object.return_value = {
            "metadata": {"name": "test-id", "uid": "test-uid"}
        }

        expires_at = datetime(2025, 12, 31, 10, 0, 0, tzinfo=timezone.utc)

        provider.create_workload(
            sandbox_id="test-id",
            namespace="test-ns",
            image_spec=ImageSpec(uri="python:3.11"),
            entrypoint=["/bin/bash"],
            env={},
            resource_limits={},
            labels={},
            expires_at=expires_at,
            execd_image="execd:latest",
        )

        body = mock_k8s_client.create_custom_object.call_args.kwargs["body"]
        pod_spec = body["spec"]["podTemplate"]["spec"]
        containers = pod_spec["containers"]

        assert len(containers) == 1
        assert containers[0]["name"] == "sandbox"
        # Should not have securityContext with sysctls
        assert "securityContext" not in pod_spec or "sysctls" not in pod_spec.get(
            "securityContext", {}
        )

    def test_create_workload_with_network_policy_adds_sidecar(self, mock_k8s_client):
        provider = AgentSandboxProvider(mock_k8s_client)
        mock_k8s_client.create_custom_object.return_value = {
            "metadata": {"name": "test-id", "uid": "test-uid"}
        }

        expires_at = datetime(2025, 12, 31, 10, 0, 0, tzinfo=timezone.utc)
        network_policy = NetworkPolicy(
            default_action="deny",
            egress=[NetworkRule(action="allow", target="pypi.org")],
        )

        provider.create_workload(
            sandbox_id="test-id",
            namespace="test-ns",
            image_spec=ImageSpec(uri="python:3.11"),
            entrypoint=["/bin/bash"],
            env={},
            resource_limits={},
            labels={},
            expires_at=expires_at,
            execd_image="execd:latest",
            egress_settings=_egress_settings(
                network_policy,
                credential_proxy_enabled=True,
            ),
        )

        body = mock_k8s_client.create_custom_object.call_args.kwargs["body"]
        pod_spec = body["spec"]["podTemplate"]["spec"]
        containers = pod_spec["containers"]

        assert len(containers) == 2

        sidecar = next((c for c in containers if c["name"] == "egress"), None)
        assert sidecar is not None
        assert sidecar["image"] == "opensandbox/egress:v1.1.7"

        env_vars = {e["name"]: e["value"] for e in sidecar.get("env", [])}
        assert "OPENSANDBOX_EGRESS_RULES" in env_vars
        assert env_vars["OPENSANDBOX_EGRESS_MODE"] == EGRESS_MODE_DNS
        assert env_vars[OPENSANDBOX_EGRESS_MITMPROXY_TRANSPARENT] == "true"

        caps = sidecar.get("securityContext", {}).get("capabilities", {})
        assert "NET_ADMIN" in caps.get("add", [])
        assert sidecar.get("securityContext", {}).get("privileged") is not True
        assert "command" not in sidecar
        assert sidecar["readinessProbe"]["httpGet"]["path"] == "/healthz"
        assert sidecar["readinessProbe"]["httpGet"]["port"] == 18080

        inits = pod_spec.get("initContainers", [])
        assert len(inits) == 1
        execd_init = inits[0]
        assert execd_init["name"] == "execd-installer"
        assert execd_init["image"] == "execd:latest"
        assert execd_init.get("securityContext", {}).get("privileged") is True
        assert "/proc/sys/net/ipv6/conf/all/disable_ipv6" in execd_init["args"][0]

        main = next(c for c in containers if c["name"] == "sandbox")
        main_env = {e["name"]: e["value"] for e in main["env"]}
        assert main_env[OPENSANDBOX_EGRESS_MITMPROXY_TRANSPARENT] == "true"
        assert "SSL_CERT_FILE" not in main_env
        assert "REQUESTS_CA_BUNDLE" not in main_env
        assert "CURL_CA_BUNDLE" not in main_env
        assert "GIT_SSL_CAINFO" not in main_env
        assert "NODE_EXTRA_CA_CERTS" not in main_env
        assert "opensandbox-mitm-ca" not in {v["name"] for v in pod_spec["volumes"]}
        assert {
            "name": OPENSANDBOX_RUNTIME_VOLUME_NAME,
            "mountPath": OPENSANDBOX_RUNTIME_MOUNT_PATH,
        } in main["volumeMounts"]
        assert {
            "name": OPENSANDBOX_RUNTIME_VOLUME_NAME,
            "mountPath": OPENSANDBOX_RUNTIME_MOUNT_PATH,
        } in sidecar["volumeMounts"]

    def test_create_workload_with_network_policy_persists_annotation_and_sidecar_token(
        self, mock_k8s_client
    ):
        provider = AgentSandboxProvider(mock_k8s_client)
        mock_k8s_client.create_custom_object.return_value = {
            "metadata": {"name": "test-id", "uid": "test-uid"}
        }

        provider.create_workload(
            sandbox_id="test-id",
            namespace="test-ns",
            image_spec=ImageSpec(uri="python:3.11"),
            entrypoint=["/bin/bash"],
            env={},
            resource_limits={},
            labels={},
            expires_at=None,
            execd_image="execd:latest",
            annotations={SANDBOX_EGRESS_AUTH_TOKEN_METADATA_KEY: "egress-token"},
            egress_settings=_egress_settings(
                NetworkPolicy(default_action="deny", egress=[]),
                auth_token="egress-token",
            ),
        )

        body = mock_k8s_client.create_custom_object.call_args.kwargs["body"]
        assert (
            body["metadata"]["annotations"][SANDBOX_EGRESS_AUTH_TOKEN_METADATA_KEY]
            == "egress-token"
        )

        containers = body["spec"]["podTemplate"]["spec"]["containers"]
        sidecar = next((c for c in containers if c["name"] == "egress"), None)
        assert sidecar is not None
        env_vars = {e["name"]: e["value"] for e in sidecar.get("env", [])}
        assert env_vars[OPENSANDBOX_EGRESS_TOKEN] == "egress-token"
        assert env_vars["OPENSANDBOX_EGRESS_MODE"] == EGRESS_MODE_DNS
        assert sidecar["readinessProbe"]["httpGet"]["httpHeaders"] == [
            {"name": OPEN_SANDBOX_EGRESS_AUTH_HEADER, "value": "egress-token"}
        ]

    def test_create_workload_with_egress_mode_dns_nft(self, mock_k8s_client):
        provider = AgentSandboxProvider(mock_k8s_client)
        mock_k8s_client.create_custom_object.return_value = {
            "metadata": {"name": "test-id", "uid": "test-uid"}
        }

        provider.create_workload(
            sandbox_id="test-id",
            namespace="test-ns",
            image_spec=ImageSpec(uri="python:3.11"),
            entrypoint=["/bin/bash"],
            env={},
            resource_limits={},
            labels={},
            expires_at=None,
            execd_image="execd:latest",
            egress_settings=_egress_settings(
                NetworkPolicy(default_action="deny", egress=[]),
                mode=EGRESS_MODE_DNS_NFT,
            ),
        )

        body = mock_k8s_client.create_custom_object.call_args.kwargs["body"]
        containers = body["spec"]["podTemplate"]["spec"]["containers"]
        sidecar = next((c for c in containers if c["name"] == "egress"), None)
        assert sidecar is not None
        env_vars = {e["name"]: e["value"] for e in sidecar.get("env", [])}
        assert env_vars["OPENSANDBOX_EGRESS_MODE"] == EGRESS_MODE_DNS_NFT

    def test_create_workload_with_network_policy_does_not_add_pod_ipv6_sysctls(
        self, mock_k8s_client
    ):
        provider = AgentSandboxProvider(mock_k8s_client)
        mock_k8s_client.create_custom_object.return_value = {
            "metadata": {"name": "test-id", "uid": "test-uid"}
        }

        expires_at = datetime(2025, 12, 31, 10, 0, 0, tzinfo=timezone.utc)
        network_policy = NetworkPolicy(
            default_action="deny",
            egress=[NetworkRule(action="allow", target="example.com")],
        )

        provider.create_workload(
            sandbox_id="test-id",
            namespace="test-ns",
            image_spec=ImageSpec(uri="python:3.11"),
            entrypoint=["/bin/bash"],
            env={},
            resource_limits={},
            labels={},
            expires_at=expires_at,
            execd_image="execd:latest",
            egress_settings=_egress_settings(network_policy),
        )

        body = mock_k8s_client.create_custom_object.call_args.kwargs["body"]
        pod_spec = body["spec"]["podTemplate"]["spec"]

        assert "securityContext" not in pod_spec or "sysctls" not in pod_spec.get(
            "securityContext", {}
        )

        sidecar = next(c for c in pod_spec["containers"] if c["name"] == "egress")
        assert "command" not in sidecar
        execd_init = pod_spec["initContainers"][0]
        assert execd_init["name"] == "execd-installer"
        assert "/proc/sys/net/ipv6/conf/all/disable_ipv6" in execd_init["args"][0]

    def test_create_workload_with_egress_skips_ipv6_disable_when_not_configured(
        self, mock_k8s_client
    ):
        """With ``egress.disable_ipv6`` false, execd init stays unprivileged without sysctl writes."""
        provider = AgentSandboxProvider(mock_k8s_client)
        mock_k8s_client.create_custom_object.return_value = {
            "metadata": {"name": "test-id", "uid": "test-uid"}
        }

        network_policy = NetworkPolicy(
            default_action="deny",
            egress=[NetworkRule(action="allow", target="example.com")],
        )

        provider.create_workload(
            sandbox_id="test-id",
            namespace="test-ns",
            image_spec=ImageSpec(uri="python:3.11"),
            entrypoint=["/bin/bash"],
            env={},
            resource_limits={},
            labels={},
            expires_at=None,
            execd_image="execd:latest",
            egress_settings=_egress_settings(network_policy, disable_ipv6=False),
        )

        body = mock_k8s_client.create_custom_object.call_args.kwargs["body"]
        pod_spec = body["spec"]["podTemplate"]["spec"]
        execd_init = pod_spec["initContainers"][0]
        assert execd_init["name"] == "execd-installer"
        assert "securityContext" not in execd_init
        assert "/proc/sys/net/ipv6/conf/all/disable_ipv6" not in execd_init["args"][0]

    def test_create_workload_with_network_policy_drops_net_admin_from_main_container(
        self, mock_k8s_client
    ):
        provider = AgentSandboxProvider(mock_k8s_client)
        mock_k8s_client.create_custom_object.return_value = {
            "metadata": {"name": "test-id", "uid": "test-uid"}
        }

        expires_at = datetime(2025, 12, 31, 10, 0, 0, tzinfo=timezone.utc)
        network_policy = NetworkPolicy(
            default_action="deny",
            egress=[NetworkRule(action="allow", target="example.com")],
        )

        provider.create_workload(
            sandbox_id="test-id",
            namespace="test-ns",
            image_spec=ImageSpec(uri="python:3.11"),
            entrypoint=["/bin/bash"],
            env={},
            resource_limits={},
            labels={},
            expires_at=expires_at,
            execd_image="execd:latest",
            egress_settings=_egress_settings(network_policy),
        )

        body = mock_k8s_client.create_custom_object.call_args.kwargs["body"]
        pod_spec = body["spec"]["podTemplate"]["spec"]
        containers = pod_spec["containers"]

        main_container = next((c for c in containers if c["name"] == "sandbox"), None)
        assert main_container is not None

        assert "securityContext" in main_container
        assert "capabilities" in main_container["securityContext"]
        assert "drop" in main_container["securityContext"]["capabilities"]
        assert "NET_ADMIN" in main_container["securityContext"]["capabilities"]["drop"]

    def test_egress_sidecar_contains_network_policy_in_env(self, mock_k8s_client):
        provider = AgentSandboxProvider(mock_k8s_client)
        mock_k8s_client.create_custom_object.return_value = {
            "metadata": {"name": "test-id", "uid": "test-uid"}
        }

        expires_at = datetime(2025, 12, 31, 10, 0, 0, tzinfo=timezone.utc)
        network_policy = NetworkPolicy(
            default_action="deny",
            egress=[
                NetworkRule(action="allow", target="pypi.org"),
                NetworkRule(action="deny", target="*.malicious.com"),
            ],
        )

        provider.create_workload(
            sandbox_id="test-id",
            namespace="test-ns",
            image_spec=ImageSpec(uri="python:3.11"),
            entrypoint=["/bin/bash"],
            env={},
            resource_limits={},
            labels={},
            expires_at=expires_at,
            execd_image="execd:latest",
            egress_settings=_egress_settings(network_policy),
        )

        body = mock_k8s_client.create_custom_object.call_args.kwargs["body"]
        pod_spec = body["spec"]["podTemplate"]["spec"]
        containers = pod_spec["containers"]

        sidecar = next((c for c in containers if c["name"] == "egress"), None)
        assert sidecar is not None

        env_vars = {e["name"]: e["value"] for e in sidecar.get("env", [])}
        assert "OPENSANDBOX_EGRESS_RULES" in env_vars

        import json

        policy_json = json.loads(env_vars["OPENSANDBOX_EGRESS_RULES"])
        assert policy_json["defaultAction"] == "deny"
        assert len(policy_json["egress"]) == 2
        assert policy_json["egress"][0]["action"] == "allow"
        assert policy_json["egress"][0]["target"] == "pypi.org"

    def test_create_workload_mounts_upstream_proxy_ca_secret_only_on_egress(
        self, mock_k8s_client
    ):
        provider = AgentSandboxProvider(mock_k8s_client)
        mock_k8s_client.create_custom_object.return_value = {
            "metadata": {"name": "test-id", "uid": "test-uid"}
        }
        network_policy = NetworkPolicy(
            default_action="deny",
            egress=[NetworkRule(action="allow", target="pypi.org")],
        )

        provider.create_workload(
            sandbox_id="test-id",
            namespace="test-ns",
            image_spec=ImageSpec(uri="python:3.11"),
            entrypoint=["/bin/bash"],
            env={},
            resource_limits={},
            labels={},
            expires_at=datetime(2025, 12, 31, 10, 0, 0, tzinfo=timezone.utc),
            execd_image="execd:latest",
            egress_settings=_egress_settings(
                network_policy,
                credential_proxy_enabled=True,
                upstream_proxy=EgressUpstreamProxyConfig(
                    url="https://proxy.local:8443",
                    ca_secret_name="corp-proxy-ca",
                ),
            ),
        )

        body = mock_k8s_client.create_custom_object.call_args.kwargs["body"]
        pod_spec = body["spec"]["podTemplate"]["spec"]
        containers = pod_spec["containers"]

        ca_volume = {
            "name": EGRESS_UPSTREAM_EXTRA_CA_VOLUME_NAME,
            "secret": {
                "secretName": "corp-proxy-ca",
                "items": [
                    {
                        "key": EGRESS_UPSTREAM_EXTRA_CA_SECRET_KEY,
                        "path": EGRESS_UPSTREAM_EXTRA_CA_SECRET_KEY,
                    }
                ],
            },
        }
        assert ca_volume in pod_spec["volumes"]

        sidecar = next(c for c in containers if c["name"] == "egress")
        assert {
            "name": EGRESS_UPSTREAM_EXTRA_CA_VOLUME_NAME,
            "mountPath": EGRESS_UPSTREAM_EXTRA_CA_PATH,
            "subPath": EGRESS_UPSTREAM_EXTRA_CA_SECRET_KEY,
            "readOnly": True,
        } in sidecar["volumeMounts"]
        env_vars = {e["name"]: e["value"] for e in sidecar["env"]}
        assert (
            env_vars[OPENSANDBOX_EGRESS_MITMPROXY_UPSTREAM_EXTRA_CA]
            == EGRESS_UPSTREAM_EXTRA_CA_PATH
        )

        main = next(c for c in containers if c["name"] == "sandbox")
        assert EGRESS_UPSTREAM_EXTRA_CA_VOLUME_NAME not in {
            m["name"] for m in main.get("volumeMounts", [])
        }
        assert (
            OPENSANDBOX_EGRESS_MITMPROXY_UPSTREAM_EXTRA_CA
            not in {e["name"] for e in main.get("env", [])}
        )

    def test_main_container_no_security_context_without_network_policy(self, mock_k8s_client):
        provider = AgentSandboxProvider(mock_k8s_client)
        mock_k8s_client.create_custom_object.return_value = {
            "metadata": {"name": "test-id", "uid": "test-uid"}
        }

        expires_at = datetime(2025, 12, 31, 10, 0, 0, tzinfo=timezone.utc)

        provider.create_workload(
            sandbox_id="test-id",
            namespace="test-ns",
            image_spec=ImageSpec(uri="python:3.11"),
            entrypoint=["/bin/bash"],
            env={},
            resource_limits={},
            labels={},
            expires_at=expires_at,
            execd_image="execd:latest",
        )

        body = mock_k8s_client.create_custom_object.call_args.kwargs["body"]
        pod_spec = body["spec"]["podTemplate"]["spec"]
        containers = pod_spec["containers"]

        main_container = containers[0]
        assert "securityContext" not in main_container