Skip to content

Secure, Fast, and Extensible Sandbox runtime for AI agents.

65.0 KB
Raw Blame
# pyright: reportAttributeAccessIssue=false
# protobuf-generated modules expose dynamic attributes.

# Copyright 2026 The OpenSandbox Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""HTTP-to-gRPC integration tests for the fsb runtime."""

from concurrent import futures
from copy import deepcopy
from datetime import datetime, timedelta, timezone
import json
import logging
from types import SimpleNamespace
from unittest.mock import Mock, patch

import grpc
import pytest
from fastapi import FastAPI, HTTPException
from fastapi.testclient import TestClient
from kubernetes.client import ApiException, CustomObjectsApi, V1APIResource, V1APIResourceList

from opensandbox_server.api import lifecycle, network_policy
from opensandbox_server.api.schema import RenewSandboxExpirationRequest
from opensandbox_server.config import (
    AppConfig,
    EGRESS_MODE_DNS_NFT,
    EgressConfig,
    EgressUpstreamProxyConfig,
    IngressConfig,
    KubernetesRuntimeConfig,
    RuntimeConfig,
    ServerConfig,
)
from opensandbox_server.middleware.request_id import RequestIdMiddleware
from opensandbox_server.services.constants import SandboxErrorCodes
from opensandbox_server.services.fast_sandbox.fastpath_client import (
    FastPathClient,
    FastPathResourceExhausted,
    FastPathUnavailable,
)
from opensandbox_server.services.composite_service import CompositeSandboxService
from opensandbox_server.services.factory import create_sandbox_service
from opensandbox_server.services.fast_sandbox.service import FastSandboxService
from opensandbox_server.services.k8s.client import K8sClient
from opensandbox_server.services.k8s.informer import WorkloadInformer
from opensandbox_server.services.k8s.kubernetes_service import KubernetesSandboxService
from opensandbox_server.services.fast_sandbox.cr_mapping import METADATA_PREFIX
from opensandbox_server.services.fast_sandbox.generated import fastpath_pb2 as pb2
from opensandbox_server.services.fast_sandbox.generated import fastpath_pb2_grpc as pb2_grpc
from opensandbox_server.tenants.context import get_current_tenant, set_current_tenant
from opensandbox_server.tenants.models import TenantEntry


class _FakeFastPathService(pb2_grpc.FastPathServiceServicer):
    def __init__(self):
        self.sandboxes: dict[tuple[str, str], tuple[str, int]] = {}
        self.crs: dict[tuple[str, str], dict] = {}
        self.last_create: pb2.CreateSandboxRequest | None = None
        self.last_get: pb2.GetSandboxRequest | None = None
        self.last_update: pb2.UpdateSandboxRequest | None = None
        self.last_delete: pb2.DeleteRequest | None = None
        self.abort_update_with: grpc.StatusCode | None = None
        self.abort_create_with: grpc.StatusCode | None = None
        self.reject_create_with: grpc.StatusCode | None = None
        self.abort_pause_with: grpc.StatusCode | None = None
        self.abort_resume_with: grpc.StatusCode | None = None
        self.pause_requests: list[pb2.PauseSandboxRequest] = []
        self.resume_requests: list[pb2.ResumeSandboxRequest] = []
        self.create_pending = False
        self.create_time_remaining: float | None = None
        self.diagnostic_runtime_state = pb2.RUNTIME_STATE_READY
        self.pool_has_resources = True
        self.get_error_by_namespace: dict[str, grpc.StatusCode] = {}
        # Per-sandbox recorded policy state, simulating the egress-held
        # /policy/{uid} surface: seeded at create from the egress binding
        # input (default deny), mutated by the relay, DELETE lands a
        # default-deny tombstone at the successor revision.
        self.policy_records: dict[str, dict] = {}
        self.last_relay: pb2.InvokeHandlerActionRequest | None = None

    @staticmethod
    def _info(name: str, uid: str, namespace: str = "ns-1") -> pb2.SandboxInfo:
        return pb2.SandboxInfo(
            identity=pb2.SandboxIdentity(uid=uid, name=name, namespace=namespace),
            applied_generation=1,
            runtime=pb2.RuntimeInfo(state=pb2.RUNTIME_STATE_READY),
            data_plane=pb2.DataPlaneInfo(state=pb2.DATA_PLANE_STATE_READY),
            ready=True,
        )

    def CreateSandbox(self, request, context):
        self.last_create = request
        if self.reject_create_with is not None:
            context.abort(self.reject_create_with, "scripted rejection before persistence")
        self.create_time_remaining = context.time_remaining()
        uid = f"uid-{request.request_id}"
        self.sandboxes[(request.namespace, request.request_id)] = (uid, 1)
        self.crs[(request.namespace, request.request_id)] = {
            "metadata": {
                "name": request.request_id,
                "namespace": request.namespace,
                "uid": uid,
                "generation": 1,
                "resourceVersion": "1",
                "creationTimestamp": datetime.now(timezone.utc).isoformat(),
                "labels": {METADATA_PREFIX + key: value for key, value in request.metadata.items()},
            },
            "spec": {
                "image": request.image,
                "command": list(request.command),
                "poolRef": request.pool_ref,
                "actionBindings": [
                    {"handler": b.handler, "input": b.input} for b in request.action_bindings
                ],
                "expireTime": datetime.fromtimestamp(
                    request.expires_at_unix_seconds, timezone.utc
                ).isoformat(),
            },
            "status": {
                "observedGeneration": 1,
                "runtime": {"state": "Creating" if self.create_pending else "Ready"},
                "dataPlane": {"state": "Pending" if self.create_pending else "Ready"},
                "conditions": [
                    {
                        "type": "Ready",
                        "status": "False" if self.create_pending else "True",
                        "observedGeneration": 1,
                    }
                ],
            },
        }
        if self.abort_create_with is not None:
            context.abort(self.abort_create_with, "scripted post-persistence failure")
        seed = {"defaultAction": "deny", "egress": []}
        for binding in request.action_bindings:
            if binding.handler == "egress":
                try:
                    parsed = json.loads(binding.input)
                    if isinstance(parsed, dict):
                        seed = parsed
                except json.JSONDecodeError:
                    pass
        self.policy_records[request.request_id] = {"revision": 1, "policy": seed}
        info = self._info(request.request_id, uid, request.namespace)
        if self.create_pending:
            info.runtime.state = pb2.RUNTIME_STATE_CREATING
            info.data_plane.state = pb2.DATA_PLANE_STATE_PENDING
            info.ready = False
        return pb2.CreateSandboxResponse(
            sandbox=info,
            generation=1,
            completion=request.completion,
        )

    def InvokeHandlerAction(self, request, context):
        self.last_relay = request
        name = request.sandbox.namespaced_name.name
        record = self.policy_records.get(name)
        if request.path.startswith("/policy/"):
            if request.method == "GET":
                if record is None:
                    return pb2.InvokeHandlerActionResponse(
                        result=pb2.HandlerActionResult(status=404, body=b"no recorded policy")
                    )
                return pb2.InvokeHandlerActionResponse(
                    result=pb2.HandlerActionResult(status=200, body=json.dumps(record).encode())
                )
            if request.method == "PUT":
                document = json.loads(request.payload)
                expected = document.get("expectedRevision")
                current = record["revision"] if record else 0
                if expected is not None and expected != current:
                    return pb2.InvokeHandlerActionResponse(
                        result=pb2.HandlerActionResult(status=412, body=b"revision mismatch")
                    )
                updated = {"revision": current + 1, "policy": document["policy"]}
                self.policy_records[name] = updated
                return pb2.InvokeHandlerActionResponse(
                    result=pb2.HandlerActionResult(
                        status=200 if record else 201, body=json.dumps(updated).encode()
                    )
                )
            if request.method == "DELETE":
                if record is None:
                    return pb2.InvokeHandlerActionResponse(
                        result=pb2.HandlerActionResult(status=404, body=b"no recorded policy")
                    )
                tombstone = {
                    "revision": record["revision"] + 1,
                    "policy": {"defaultAction": "deny", "egress": []},
                }
                self.policy_records[name] = tombstone
                return pb2.InvokeHandlerActionResponse(
                    result=pb2.HandlerActionResult(status=204, body=b"")
                )
        return pb2.InvokeHandlerActionResponse(
            result=pb2.HandlerActionResult(status=404, body=b"unknown relay path")
        )

    def GetSandbox(self, request, context):
        self.last_get = request
        namespace = request.sandbox.namespaced_name.namespace
        name = request.sandbox.namespaced_name.name
        if error := self.get_error_by_namespace.get(namespace):
            context.abort(error, "scripted get failure")
        current = self.sandboxes.get((namespace, name))
        if current is None:
            return context.abort(grpc.StatusCode.NOT_FOUND, "not found")
        uid, generation = current
        if request.sandbox.expected_uid and request.sandbox.expected_uid != uid:
            context.abort(grpc.StatusCode.ABORTED, "uid fence rejected")
        if request.expected_generation and request.expected_generation != generation:
            context.abort(grpc.StatusCode.ABORTED, "generation fence rejected")
        return pb2.GetSandboxResponse(
            sandbox=self._info(name, uid, namespace), generation=generation
        )

    def UpdateSandbox(self, request, context):
        self.last_update = request
        if self.abort_update_with is not None:
            context.abort(self.abort_update_with, "scripted update failure")
        namespace = request.sandbox.namespaced_name.namespace
        name = request.sandbox.namespaced_name.name
        current = self.sandboxes.get((namespace, name))
        if current is None:
            return context.abort(grpc.StatusCode.NOT_FOUND, "not found")
        uid, generation = current
        if request.sandbox.expected_uid != uid or (
            request.expected_generation and request.expected_generation != generation
        ):
            context.abort(grpc.StatusCode.ABORTED, "fence rejected")
        self.sandboxes[(namespace, name)] = (uid, generation + 1)
        cr = self.crs[(namespace, name)]
        cr["metadata"]["generation"] = generation + 1
        cr["metadata"]["resourceVersion"] = str(generation + 1)
        if request.HasField("expires_at_unix_seconds"):
            cr["spec"]["expireTime"] = datetime.fromtimestamp(
                request.expires_at_unix_seconds, timezone.utc
            ).isoformat()
        if request.HasField("action_bindings"):
            cr["spec"]["actionBindings"] = [
                {"handler": b.handler, "input": b.input} for b in request.action_bindings.items
            ]
        for key, value in request.metadata_upsert.items():
            cr["metadata"]["labels"][METADATA_PREFIX + key] = value
        for key in request.metadata_delete_keys:
            cr["metadata"]["labels"].pop(METADATA_PREFIX + key, None)
        return pb2.UpdateSandboxResponse(
            sandbox=pb2.SandboxIdentity(uid=uid, name=name, namespace=namespace),
            committed_generation=generation + 1,
        )

    def DeleteSandbox(self, request, context):
        self.last_delete = request
        namespace = request.sandbox.namespaced_name.namespace
        name = request.sandbox.namespaced_name.name
        current = self.sandboxes.get((namespace, name))
        if current is None:
            return context.abort(grpc.StatusCode.NOT_FOUND, "not found")
        uid, _ = current
        if request.sandbox.expected_uid != uid:
            context.abort(grpc.StatusCode.ABORTED, "uid fence rejected")
        self.sandboxes.pop((namespace, name))
        self.crs.pop((namespace, name), None)
        return pb2.DeleteResponse()

    def _cr_runtime_state(self, namespace: str, name: str) -> str:
        cr = self.crs.get((namespace, name)) or {}
        return (cr.get("status") or {}).get("runtime", {}).get("state", "")

    def PauseSandbox(self, request, context):
        self.pause_requests.append(request)
        namespace = request.sandbox.namespaced_name.namespace
        name = request.sandbox.namespaced_name.name
        current = self.sandboxes.get((namespace, name))
        if current is None:
            return context.abort(grpc.StatusCode.NOT_FOUND, "not found")
        uid, generation = current
        if request.sandbox.expected_uid and request.sandbox.expected_uid != uid:
            context.abort(grpc.StatusCode.ABORTED, "uid fence rejected")
        # Only a terminal runtime refuses the pause.
        if self._cr_runtime_state(namespace, name) in ("Stopped", "Stopping", "Failed"):
            context.abort(
                grpc.StatusCode.FAILED_PRECONDITION,
                "only a Ready runtime can be paused",
            )
        if self.abort_pause_with is not None:
            context.abort(self.abort_pause_with, "scripted pause failure")
        if (namespace, name) in self.crs:
            self.crs[(namespace, name)]["status"]["runtime"]["state"] = "Pausing"
        return pb2.PauseSandboxResponse(
            sandbox=self._info(name, uid, namespace), generation=generation
        )

    def ResumeSandbox(self, request, context):
        self.resume_requests.append(request)
        namespace = request.sandbox.namespaced_name.namespace
        name = request.sandbox.namespaced_name.name
        current = self.sandboxes.get((namespace, name))
        if current is None:
            return context.abort(grpc.StatusCode.NOT_FOUND, "not found")
        uid, generation = current
        if request.sandbox.expected_uid and request.sandbox.expected_uid != uid:
            context.abort(grpc.StatusCode.ABORTED, "uid fence rejected")
        # A durably Paused sandbox without a checkpoint is unresumable.
        cr = self.crs.get((namespace, name)) or {}
        runtime_status = (cr.get("status") or {}).get("runtime") or {}
        if runtime_status.get("state") == "Paused" and not runtime_status.get("checkpoint"):
            context.abort(
                grpc.StatusCode.FAILED_PRECONDITION,
                "no recorded checkpoint; resume is impossible",
            )
        if self.abort_resume_with is not None:
            context.abort(self.abort_resume_with, "scripted resume failure")
        if (namespace, name) in self.crs:
            self.crs[(namespace, name)]["status"]["runtime"]["state"] = "Resuming"
        return pb2.ResumeSandboxResponse(
            sandbox=self._info(name, uid, namespace), generation=generation
        )

    def GetSandboxDiagnostics(self, request, context):
        sandbox = self.sandboxes.get((request.namespace, request.sandbox_name))
        if sandbox is None:
            return context.abort(grpc.StatusCode.NOT_FOUND, "not found")
        uid, _ = sandbox
        info = self._info(request.sandbox_name, uid, request.namespace)
        info.runtime.state = self.diagnostic_runtime_state
        return pb2.SandboxDiagnosticsResponse(
            sandbox=info,
            assignment_state="assigned",
            events=[
                pb2.SandboxDiagnosticEvent(
                    timestamp_unix_nano=1,
                    level="Info",
                    source="fastlet",
                    phase="Ready",
                    message="sandbox ready",
                )
            ],
        )

    def GetPool(self, request, context):
        if not self.pool_has_resources:
            return pb2.PoolInfo(namespace=request.namespace, name=request.pool_name)
        return pb2.PoolInfo(
            namespace=request.namespace,
            name=request.pool_name,
            sandbox_cpu="500m",
            sandbox_memory="512Mi",
        )


@pytest.fixture
def http_fsb(monkeypatch):
    fake = _FakeFastPathService()
    grpc_server = grpc.server(futures.ThreadPoolExecutor(max_workers=4))
    pb2_grpc.add_FastPathServiceServicer_to_server(fake, grpc_server)
    port = grpc_server.add_insecure_port("127.0.0.1:0")
    grpc_server.start()

    config = AppConfig(
        server=ServerConfig(
            host="0.0.0.0",
            port=8080,
            api_key="x",
            max_sandbox_timeout_seconds=7200,
        ),
        runtime=RuntimeConfig(type="kubernetes", execd_image="ghcr.io/opensandbox/execd:latest"),
        kubernetes=KubernetesRuntimeConfig(namespace="ns-1"),
    )
    fastpath = FastPathClient(endpoint=f"127.0.0.1:{port}")
    with patch.object(K8sClient, "_load_config"):
        k8s = K8sClient(KubernetesRuntimeConfig())
    monkeypatch.setattr(WorkloadInformer, "start", lambda self: None)
    api = Mock(spec=CustomObjectsApi)
    api.get_api_resources.return_value = V1APIResourceList(
        group_version="sandbox.fast.io/v1alpha2",
        resources=[
            V1APIResource(
                name="sandboxes",
                singular_name="sandbox",
                kind="Sandbox",
                namespaced=True,
                verbs=["get", "list", "watch"],
            )
        ],
    )

    def get_cr(**kwargs):
        cr = fake.crs.get((kwargs["namespace"], kwargs["name"]))
        if cr is None:
            raise ApiException(status=404)
        return deepcopy(cr)

    def list_crs(**kwargs):
        return {
            "metadata": {"resourceVersion": "1"},
            "items": [
                deepcopy(cr)
                for (namespace, _), cr in fake.crs.items()
                if namespace == kwargs["namespace"]
            ],
        }

    api.get_namespaced_custom_object.side_effect = get_cr
    api.list_namespaced_custom_object.side_effect = list_crs
    k8s._custom_objects_api = api
    service = FastSandboxService(config, fastpath_client=fastpath, k8s_client=k8s)
    monkeypatch.setattr(lifecycle, "sandbox_service", service)

    app = FastAPI()
    app.add_middleware(RequestIdMiddleware)
    app.include_router(lifecycle.router, prefix="/v1")
    app.include_router(network_policy.router, prefix="/v1")
    try:
        with TestClient(app) as client:
            yield client, fake, service
    finally:
        service.close()
        grpc_server.stop(None)


def test_http_create_calls_fastpath_with_ready_completion(http_fsb):
    client, fake, service = http_fsb
    response = client.post(
        "/v1/sandboxes",
        json={
            "image": {"uri": "python:3.11"},
            "entrypoint": ["python", "-m", "http.server"],
            "timeout": 3600,
            "resourceLimits": {"cpu": "500m", "memory": "512Mi"},
            "metadata": {"team": "agents"},
        },
    )

    assert response.status_code == 202
    body = response.json()
    assert body["id"].startswith("fsb-")
    assert body["status"]["state"] == "Running"
    assert body["metadata"] == {"team": "agents"}
    assert fake.last_create.request_id == body["id"]
    assert fake.last_create.completion == pb2.CREATE_COMPLETION_READY
    assert fake.create_time_remaining > service._k8s.fastpath_wait_ready_seconds


def _annotation_api(service: FastSandboxService) -> Mock:
    return service._cr_reader._kubernetes()._custom_objects_api  # type: ignore[return-value]


def test_http_create_persists_idle_policy_annotation(http_fsb):
    client, fake, service = http_fsb
    api = _annotation_api(service)

    response = client.post(
        "/v1/sandboxes",
        json={
            "image": {"uri": "python:3.11"},
            "entrypoint": ["python"],
            "timeout": 3600,
            "resourceLimits": {"cpu": "500m", "memory": "512Mi"},
            "lifecycle": {
                "idlePolicy": {"idleTimeoutSeconds": 300, "onIdle": "pause"},
            },
        },
    )

    assert response.status_code == 202
    sandbox_id = response.json()["id"]
    api.patch_namespaced_custom_object.assert_called_once()
    kwargs = api.patch_namespaced_custom_object.call_args.kwargs
    assert kwargs["group"] == "sandbox.fast.io"
    assert kwargs["version"] == "v1alpha2"
    assert kwargs["namespace"] == "ns-1"
    assert kwargs["name"] == sandbox_id
    annotation = kwargs["body"]["metadata"]["annotations"]["sandbox.opensandbox.io/idle-policy"]
    assert json.loads(annotation) == {
        "idleTimeoutSeconds": 300,
        "onIdle": "pause",
        "wakeOnAccess": True,
    }


def test_http_create_wake_only_policy_annotation(http_fsb):
    client, fake, service = http_fsb
    api = _annotation_api(service)

    response = client.post(
        "/v1/sandboxes",
        json={
            "image": {"uri": "python:3.11"},
            "entrypoint": ["python"],
            "timeout": 3600,
            "resourceLimits": {"cpu": "500m", "memory": "512Mi"},
            "lifecycle": {"idlePolicy": {"wakeOnAccess": False}},
        },
    )

    assert response.status_code == 202
    annotation = (
        api.patch_namespaced_custom_object.call_args.kwargs["body"]["metadata"][
            "annotations"
        ]["sandbox.opensandbox.io/idle-policy"]
    )
    assert json.loads(annotation) == {"wakeOnAccess": False}


def test_http_create_probe_policy_annotation_applies_default_timeout(http_fsb):
    client, fake, service = http_fsb
    api = _annotation_api(service)

    response = client.post(
        "/v1/sandboxes",
        json={
            "image": {"uri": "python:3.11"},
            "entrypoint": ["python"],
            "timeout": 3600,
            "resourceLimits": {"cpu": "500m", "memory": "512Mi"},
            "lifecycle": {
                "idlePolicy": {
                    "idleTimeoutSeconds": 300,
                    "onIdle": "pause",
                    "activeProbe": {"port": 8080, "path": "/active"},
                },
            },
        },
    )

    assert response.status_code == 202
    annotation = (
        api.patch_namespaced_custom_object.call_args.kwargs["body"]["metadata"][
            "annotations"
        ]["sandbox.opensandbox.io/idle-policy"]
    )
    assert json.loads(annotation) == {
        "idleTimeoutSeconds": 300,
        "onIdle": "pause",
        "wakeOnAccess": True,
        "activeProbe": {"port": 8080, "path": "/active", "timeoutSeconds": 2},
    }


def test_http_create_without_idle_policy_skips_annotation_write(http_fsb):
    client, fake, service = http_fsb
    api = _annotation_api(service)

    response = client.post(
        "/v1/sandboxes",
        json={
            "image": {"uri": "python:3.11"},
            "entrypoint": ["python"],
            "timeout": 3600,
            "resourceLimits": {"cpu": "500m", "memory": "512Mi"},
        },
    )

    assert response.status_code == 202
    api.patch_namespaced_custom_object.assert_not_called()


def test_http_create_rejects_idle_timeout_above_activity_ttl(http_fsb):
    client, fake, service = http_fsb
    api = _annotation_api(service)

    response = client.post(
        "/v1/sandboxes",
        json={
            "image": {"uri": "python:3.11"},
            "entrypoint": ["python"],
            "timeout": 3600,
            "resourceLimits": {"cpu": "500m", "memory": "512Mi"},
            "lifecycle": {
                "idlePolicy": {"idleTimeoutSeconds": 3000, "onIdle": "pause"},
            },
        },
    )

    assert response.status_code == 400
    assert "activity_ttl_seconds" in response.json()["detail"]["message"]
    assert fake.last_create is None  # rejected before any side effect
    api.patch_namespaced_custom_object.assert_not_called()


def test_http_create_survives_idle_policy_annotation_write_failure(http_fsb):
    client, fake, service = http_fsb
    api = _annotation_api(service)
    api.patch_namespaced_custom_object.side_effect = ApiException(status=403)

    response = client.post(
        "/v1/sandboxes",
        json={
            "image": {"uri": "python:3.11"},
            "entrypoint": ["python"],
            "timeout": 3600,
            "resourceLimits": {"cpu": "500m", "memory": "512Mi"},
            "lifecycle": {
                "idlePolicy": {"idleTimeoutSeconds": 300, "onIdle": "pause"},
            },
        },
    )

    # A failed policy write must not fail the create; sandbox stays unpolicied.
    assert response.status_code == 202
    assert fake.last_create is not None


def test_http_create_recovers_an_ambiguous_post_persistence_failure(http_fsb):
    client, fake, _ = http_fsb
    fake.abort_create_with = grpc.StatusCode.INTERNAL

    response = client.post(
        "/v1/sandboxes",
        json={
            "image": {"uri": "python:3.11"},
            "entrypoint": ["python"],
            "timeout": 3600,
            "resourceLimits": {"cpu": "500m", "memory": "512Mi"},
        },
    )

    assert response.status_code == 202
    sandbox_id = response.json()["id"]
    assert sandbox_id.startswith("fsb-")
    assert fake.last_get is not None
    assert ("ns-1", sandbox_id) in fake.sandboxes


def test_http_create_rejects_timeout_pool_mismatch_and_unreadable_extension(http_fsb):
    client, fake, _ = http_fsb
    base = {
        "image": {"uri": "python:3.11"},
        "entrypoint": ["python"],
        "timeout": 3600,
        "resourceLimits": {"cpu": "500m", "memory": "512Mi"},
    }

    too_long = client.post("/v1/sandboxes", json={**base, "timeout": 10**9})
    pool_mismatch = client.post(
        "/v1/sandboxes",
        json={**base, "resourceLimits": {"cpu": "1", "memory": "512Mi"}},
    )
    fake.pool_has_resources = False
    empty_pool_profile = client.post("/v1/sandboxes", json=base)

    assert too_long.status_code == 400
    assert pool_mismatch.status_code == 400
    assert "resourceLimits" in pool_mismatch.json()["detail"]["message"]
    assert empty_pool_profile.status_code == 400
    assert (
        "not defined by the selected SandboxPool" in empty_pool_profile.json()["detail"]["message"]
    )


def test_http_read_and_metadata_patch_use_cr_fields(http_fsb):
    client, fake, _ = http_fsb
    created = client.post(
        "/v1/sandboxes",
        json={
            "image": {"uri": "python:3.11"},
            "entrypoint": ["python", "-m", "http.server"],
            "timeout": 3600,
            "resourceLimits": {"cpu": "500m", "memory": "512Mi"},
            "metadata": {"team": "agents", "remove-me": "yes"},
        },
    ).json()
    sandbox_id = created["id"]

    responses = [
        client.get(f"/v1/sandboxes/{sandbox_id}"),
        client.patch(
            f"/v1/sandboxes/{sandbox_id}/metadata",
            json={"team": "platform", "remove-me": None},
        ),
        client.get("/v1/sandboxes", params={"pageSize": 10}),
    ]

    assert [response.status_code for response in responses] == [200, 200, 200]
    fetched, patched, listed = [response.json() for response in responses]
    cr = fake.crs[("ns-1", sandbox_id)]
    assert fetched["image"] == {"uri": "python:3.11"}
    assert fetched["entrypoint"] == ["python", "-m", "http.server"]
    assert datetime.fromisoformat(
        fetched["createdAt"].replace("Z", "+00:00")
    ) == datetime.fromisoformat(cr["metadata"]["creationTimestamp"])
    assert fetched["metadata"] == {"team": "agents", "remove-me": "yes"}
    assert patched["metadata"] == {"team": "platform"}
    assert fake.last_update.sandbox.expected_uid == cr["metadata"]["uid"]
    assert fake.last_update.expected_generation == 1
    assert listed["items"][0]["metadata"] == patched["metadata"]
    assert listed["pagination"]["totalItems"] == 1


@pytest.fixture
def persisted_fsb(http_fsb):
    client, fake, service = http_fsb
    response = client.post(
        "/v1/sandboxes",
        json={
            "image": {"uri": "python:3.11"},
            "entrypoint": ["python"],
            "timeout": 3600,
            "resourceLimits": {"cpu": "500m", "memory": "512Mi"},
            "metadata": {"team": "agents"},
        },
    )
    assert response.status_code == 202
    sandbox_id = response.json()["id"]
    return client, fake, service, sandbox_id


def test_cr_reads_do_not_depend_on_fastpath_and_remain_tenant_scoped(persisted_fsb):
    client, fake, service, sandbox_id = persisted_fsb
    fake.get_error_by_namespace["ns-1"] = grpc.StatusCode.UNAVAILABLE
    other = deepcopy(fake.crs[("ns-1", sandbox_id)])
    other["metadata"]["namespace"] = "tenant-a"
    other["metadata"]["labels"][METADATA_PREFIX + "team"] = "other"
    fake.crs[("tenant-a", sandbox_id)] = other
    assert client.get(f"/v1/sandboxes/{sandbox_id}").status_code == 200
    previous = get_current_tenant()
    try:
        set_current_tenant(TenantEntry(name="tenant-a", namespace="tenant-a"))
        assert client.get(f"/v1/sandboxes/{sandbox_id}").json()["metadata"] == {"team": "other"}
        listed = client.get("/v1/sandboxes").json()
        assert listed["pagination"]["totalItems"] == 1
        assert listed["items"][0]["metadata"] == {"team": "other"}
        set_current_tenant(TenantEntry(name="empty", namespace="empty"))
        assert client.get(f"/v1/sandboxes/{sandbox_id}").status_code == 404
        assert client.get("/v1/sandboxes").json()["items"] == []
    finally:
        set_current_tenant(previous)


def test_cr_watch_and_fastpath_mutations_refresh_http_reads(persisted_fsb, monkeypatch):
    client, fake, service, sandbox_id = persisted_fsb
    k8s = service._cr_reader._client
    monkeypatch.setattr(WorkloadInformer, "start", lambda self: None)
    url = f"/v1/sandboxes/{sandbox_id}"
    assert client.get(url).json()["status"]["state"] == "Running"
    informer = k8s._lookup_informer("sandbox.fast.io", "v1alpha2", "sandboxes", "ns-1")
    assert informer._full_resync()

    cr = fake.crs[("ns-1", sandbox_id)]
    cr["status"]["dataPlane"]["state"] = "Pending"
    informer._handle_event({"type": "MODIFIED", "object": deepcopy(cr)})
    assert client.get(url).json()["status"]["state"] == "Pending"

    patched = client.patch(url + "/metadata", json={"team": "platform"})
    assert patched.status_code == 200
    assert patched.json()["metadata"] == {"team": "platform"}
    assert client.get("/v1/sandboxes").json()["items"][0]["metadata"] == {"team": "platform"}

    assert informer._full_resync()
    expires = (datetime.now(timezone.utc) + timedelta(hours=2)).replace(microsecond=0)
    assert (
        client.post(url + "/renew-expiration", json={"expiresAt": expires.isoformat()}).status_code
        == 200
    )
    assert (
        datetime.fromisoformat(client.get(url).json()["expiresAt"].replace("Z", "+00:00"))
        == expires
    )

    cr["status"]["runtime"]["state"] = "Stopped"
    assert informer._full_resync()
    assert client.get(url).json()["status"]["state"] == "Terminated"
    fake.crs.pop(("ns-1", sandbox_id))
    informer._handle_event({"type": "DELETED", "object": deepcopy(cr)})
    assert client.get(url).status_code == 404
    assert client.get("/v1/sandboxes").json()["items"] == []


def test_stale_ready_generation_is_not_running(persisted_fsb):
    client, fake, _, sandbox_id = persisted_fsb
    cr = fake.crs[("ns-1", sandbox_id)]
    cr["metadata"]["generation"] = 2
    assert client.get(f"/v1/sandboxes/{sandbox_id}").json()["status"]["state"] == "Pending"


def test_delete_uses_cr_identity_when_runtime_observation_is_gone(persisted_fsb):
    client, fake, _, sandbox_id = persisted_fsb
    uid = fake.crs[("ns-1", sandbox_id)]["metadata"]["uid"]
    fake.get_error_by_namespace["ns-1"] = grpc.StatusCode.UNAVAILABLE
    url = f"/v1/sandboxes/{sandbox_id}"
    assert client.delete(url).status_code == 204
    assert fake.last_delete.sandbox.expected_uid == uid
    assert client.delete(url).status_code == 404


def test_upstream_proxy_config_rejects_network_policy_create(http_fsb):
    client, fake, service = http_fsb
    service._app_config.egress = EgressConfig(
        image="opensandbox/egress:v1.1.7",
        mode=EGRESS_MODE_DNS_NFT,
        upstream_proxy=EgressUpstreamProxyConfig(url="http://proxy.local:3128"),
    )

    response = client.post(
        "/v1/sandboxes",
        json={
            "image": {"uri": "python:3.11"},
            "entrypoint": ["python"],
            "timeout": 3600,
            "resourceLimits": {"cpu": "500m", "memory": "512Mi"},
            "networkPolicy": {"defaultAction": "deny", "egress": []},
        },
    )

    assert response.status_code == 400
    assert response.json()["detail"]["code"] == "SANDBOX::INVALID_PARAMETER"
    assert "egress.upstream_proxy" in response.json()["detail"]["message"]
    assert fake.last_create is None


def test_upstream_proxy_config_rejects_network_policy_mutation(persisted_fsb):
    client, fake, service, sandbox_id = persisted_fsb
    service._app_config.egress = EgressConfig(
        image="opensandbox/egress:v1.1.7",
        mode=EGRESS_MODE_DNS_NFT,
        upstream_proxy=EgressUpstreamProxyConfig(url="http://proxy.local:3128"),
    )

    response = client.put(
        f"/v1/sandboxes/{sandbox_id}/networkpolicy",
        json={"defaultAction": "deny", "egress": []},
    )

    assert response.status_code == 400
    assert response.json()["detail"]["code"] == "SANDBOX::INVALID_PARAMETER"
    assert fake.last_update is None


def test_upstream_proxy_config_allows_delete_network_policy_rules(persisted_fsb):
    client, fake, service, sandbox_id = persisted_fsb
    service._app_config.egress = EgressConfig(
        image="opensandbox/egress:v1.1.7",
        mode=EGRESS_MODE_DNS_NFT,
        upstream_proxy=EgressUpstreamProxyConfig(url="http://proxy.local:3128"),
    )
    url = f"/v1/sandboxes/{sandbox_id}/networkpolicy"

    # Deleting rules stays available: teardown is not chaining-sensitive.
    response = client.request("DELETE", url, json=["a.com"])

    assert response.status_code == 200
    assert fake.last_relay is not None and fake.last_relay.path.startswith("/policy/")


def test_create_without_network_policy_still_registers_egress_binding(persisted_fsb):
    """Every fsb sandbox registers an egress subject at create — the relay
    surfaces (/policy/{uid}, /vault/{uid}) attach to it. A sandbox created
    without a networkPolicy seeds default-deny and can adopt a policy later
    through the relay."""
    client, fake, _, sandbox_id = persisted_fsb
    url = f"/v1/sandboxes/{sandbox_id}/networkpolicy"

    # The create-time binding seeded a no-op default-allow record: GET
    # serves it, and the CR carries the egress binding.
    assert client.get(url).json()["policy"] == {"defaultAction": "allow", "egress": []}
    cr = fake.crs[("ns-1", sandbox_id)]
    bindings = cr["spec"]["actionBindings"]
    assert [b["handler"] for b in bindings] == ["egress"]
    assert json.loads(bindings[0]["input"]) == {"defaultAction": "allow", "egress": []}

    # The sandbox can adopt a policy later through the relay.
    policy = {"defaultAction": "allow", "egress": [{"action": "allow", "target": "a.com"}]}
    response = client.put(url, json=policy)
    assert response.status_code == 200
    assert response.json()["policy"] == policy
    assert client.get(url).json()["policy"] == policy
    # The CR binding input stays the create-time bootstrap.
    assert json.loads(cr["spec"]["actionBindings"][0]["input"]) == {
        "defaultAction": "allow",
        "egress": [],
    }


def test_upstream_proxy_config_allows_create_without_network_policy(http_fsb):
    client, fake, service = http_fsb
    service._app_config.egress = EgressConfig(
        image="opensandbox/egress:v1.1.7",
        mode=EGRESS_MODE_DNS_NFT,
        upstream_proxy=EgressUpstreamProxyConfig(url="http://proxy.local:3128"),
    )

    response = client.post(
        "/v1/sandboxes",
        json={
            "image": {"uri": "python:3.11"},
            "entrypoint": ["python"],
            "timeout": 3600,
            "resourceLimits": {"cpu": "500m", "memory": "512Mi"},
        },
    )

    assert response.status_code == 202
    assert fake.last_create is not None


def test_http_policy_replace_relays_and_leaves_bindings_untouched(persisted_fsb):
    """Runtime policy changes ride the handler relay: the CRD binding input is
    the create-time bootstrap only, and a relay PUT never rewrites it."""
    client, fake, _, sandbox_id = persisted_fsb
    url = f"/v1/sandboxes/{sandbox_id}/networkpolicy"
    # The create-time egress binding seeds the record (a no-op
    # default-allow for a create without networkPolicy).
    assert client.get(url).json()["policy"] == {"defaultAction": "allow", "egress": []}
    bindings_before = [
        {"handler": "audit", "input": '{"enabled":true}'},
        {"handler": "egress", "input": '{"defaultAction":"deny"}'},
    ]
    cr = fake.crs[("ns-1", sandbox_id)]
    cr["spec"]["actionBindings"] = bindings_before

    policy = {"defaultAction": "allow", "egress": []}
    response = client.put(url, json=policy)
    assert response.status_code == 200
    assert response.json()["policy"] == policy
    assert client.get(url).json()["policy"] == policy

    # The CRD binding input is untouched: the relay owned the change.
    assert cr["spec"]["actionBindings"] == bindings_before
    assert fake.last_update is None

    # The relayed policy is the record: the (stale) binding input never
    # resurfaces through a read.
    cr["spec"]["actionBindings"][1]["input"] = '{"defaultAction":"deny","egress":[{"action":"allow","target":"stale.com"}]}'
    assert client.get(url).json()["policy"] == policy


def test_http_policy_patch_and_delete_match_sidecar_semantics(persisted_fsb):
    client, fake, _, sandbox_id = persisted_fsb
    url = f"/v1/sandboxes/{sandbox_id}/networkpolicy"
    client.put(url, json={
        "defaultAction": "allow",
        "egress": [
            {"action": "deny", "target": "a.com"},
            {"action": "allow", "target": "b.com"},
        ],
    })

    # PATCH: incoming replaces same-target in place, first-wins, others kept,
    # defaultAction preserved.
    patched = client.patch(url, json=[
        {"action": "allow", "target": "a.com"},
        {"action": "deny", "target": "c.com"},
        {"action": "allow", "target": "c.com"},
    ])
    assert patched.status_code == 200
    assert patched.json()["policy"] == {
        "defaultAction": "allow",
        "egress": [
            {"action": "allow", "target": "a.com"},
            {"action": "allow", "target": "b.com"},
            {"action": "deny", "target": "c.com"},
        ],
    }

    # DELETE: idempotent by target, defaultAction preserved.
    deleted = client.request("DELETE", url, json=["a.com", "missing.com"])
    assert deleted.status_code == 200
    assert deleted.json()["policy"] == {
        "defaultAction": "allow",
        "egress": [
            {"action": "allow", "target": "b.com"},
            {"action": "deny", "target": "c.com"},
        ],
    }
    assert client.request("DELETE", url, json=["a.com"]).status_code == 200

    # Conflict protection still applies to merged commits: the merged PUT
    # carries the read revision as expectedRevision — a concurrent relay
    # write landing between the merge-read and the commit turns the stale
    # merge into a 409 (the egress 412 is remapped to the public conflict
    # status).
    from opensandbox_server.services.fast_sandbox import policy_relay as policy_relay_module

    real_get_full = policy_relay_module._relay_get_full

    def racy_get_full(service, sid):
        state = real_get_full(service, sid)
        fake.policy_records[sid]["revision"] += 1
        return state

    policy_relay_module._relay_get_full = racy_get_full
    try:
        response = client.patch(url, json=[{"action": "deny", "target": "d.com"}])
    finally:
        policy_relay_module._relay_get_full = real_get_full
    assert response.status_code == 409


def test_policy_rejects_invalid_input_and_other_tenant(persisted_fsb):
    client, fake, _, sandbox_id = persisted_fsb
    url = f"/v1/sandboxes/{sandbox_id}/networkpolicy"
    # defaultAction is validated by the fsb service (400); a blank target is
    # now rejected earlier by the NetworkRule schema (422).
    assert client.put(url, json={"defaultAction": "invalid"}).status_code == 400
    assert client.put(url, json={"egress": [{"action": "allow", "target": " "}]}).status_code == 422
    assert fake.last_update is None
    previous = get_current_tenant()
    try:
        set_current_tenant(TenantEntry(name="empty", namespace="empty"))
        assert client.get(url).status_code == 404
        assert client.put(url, json={"defaultAction": "allow"}).status_code == 404
        assert fake.last_update is None
    finally:
        set_current_tenant(previous)


def test_policy_rejects_port_scoped_rules(persisted_fsb):
    """Fast Sandbox enforcement does not cover port-scoped rules yet; accepting
    them would silently drop deny rules (fail-open), so both PUT and PATCH
    reject them until the fastlet path catches up."""
    client, fake, _, sandbox_id = persisted_fsb
    url = f"/v1/sandboxes/{sandbox_id}/networkpolicy"
    port_only = {"action": "deny", "ports": [25]}
    scoped = {"action": "allow", "target": "10.0.0.5", "ports": [22]}
    assert client.put(url, json={"defaultAction": "deny", "egress": [port_only]}).status_code == 400
    assert client.put(url, json={"defaultAction": "deny", "egress": [scoped]}).status_code == 400
    assert client.patch(url, json=[port_only]).status_code == 400
    assert client.patch(url, json=[scoped]).status_code == 400
    assert fake.last_update is None
    assert fake.last_relay is None or fake.last_relay.method == "GET"


def test_legacy_policy_route_preserves_body_for_existing_proxy(http_fsb, monkeypatch):
    from starlette.responses import JSONResponse

    client, _, _ = http_fsb

    async def proxy(request, sandbox_id, port, path, **kwargs):
        assert (sandbox_id, port, path) == ("legacy-id", 18080, "policy")
        body = b"".join([part async for part in request.stream()])
        return JSONResponse(json.loads(body) if body else {"status": "ok"})

    monkeypatch.setattr(network_policy, "_proxy_http_request", proxy)
    assert client.get("/v1/sandboxes/legacy-id/networkpolicy").json() == {"status": "ok"}
    policy = {"defaultAction": "allow", "egress": []}
    assert client.put("/v1/sandboxes/legacy-id/networkpolicy", json=policy).json() == policy


@pytest.mark.parametrize(
    "k8s_namespace,expected_namespace",
    [
        ("legacy", "legacy"),
        (None, "default"),
    ],
)
def test_kubernetes_configuration_automatically_composes_fsb(
    persisted_fsb, k8s_namespace, expected_namespace
):
    _, _, fsb, _ = persisted_fsb
    config = fsb._app_config.model_copy(
        update={
            "runtime": RuntimeConfig(type="kubernetes", execd_image="execd:test"),
            "kubernetes": KubernetesRuntimeConfig(namespace=k8s_namespace),
        }
    )
    with patch("opensandbox_server.services.factory.KubernetesSandboxService") as constructor:
        constructor.return_value.k8s_client = fsb._cr_reader._client
        service = create_sandbox_service(config=config)
    try:
        assert isinstance(service, CompositeSandboxService)
        assert service._fsb._resolve_namespace() == expected_namespace
        assert service._fsb._cr_reader._client is constructor.return_value.k8s_client
    finally:
        service.close()


def test_mixed_list_globally_filters_sorts_and_pages(persisted_fsb, monkeypatch):
    client, fake, fsb, sandbox_id = persisted_fsb
    base = fsb.get_sandbox(sandbox_id)
    cr = fake.crs[("ns-1", sandbox_id)]
    cr["metadata"]["creationTimestamp"] = "2026-01-02T00:00:00Z"
    legacy = Mock(spec=KubernetesSandboxService)
    legacy.list_sandbox_objects.return_value = [
        base.model_copy(
            update={"id": "legacy-new", "created_at": datetime(2026, 1, 3, tzinfo=timezone.utc)}
        ),
        base.model_copy(
            update={"id": "legacy-tie", "created_at": datetime(2026, 1, 2, tzinfo=timezone.utc)}
        ),
        base.model_copy(update={"id": "legacy-filtered", "metadata": {"team": "other"}}),
    ]
    monkeypatch.setattr(lifecycle, "sandbox_service", CompositeSandboxService(legacy, fsb))
    params = {"pageSize": 2, "metadata": "team=agents", "state": "Running"}
    first = client.get("/v1/sandboxes", params=params).json()
    second = client.get("/v1/sandboxes", params={**params, "page": 2}).json()
    assert [item["id"] for item in first["items"]] == ["legacy-new", sandbox_id]
    assert [item["id"] for item in second["items"]] == ["legacy-tie"]
    assert first["pagination"]["totalItems"] == second["pagination"]["totalItems"] == 3
    assert first["pagination"]["hasNextPage"] is True
    assert second["pagination"]["hasNextPage"] is False

    legacy.get_sandbox.return_value = base.model_copy(update={"id": "legacy-new"})
    assert client.get("/v1/sandboxes/legacy-new").status_code == 200
    assert client.get("/v1/sandboxes/fsb-missing").status_code == 404
    legacy.get_sandbox.assert_called_once_with("legacy-new")


@pytest.mark.parametrize(
    "failure,expected", [("list404", 200), ("legacy", 503)]
)
def test_mixed_list_never_hides_a_backend_failure(persisted_fsb, monkeypatch, failure, expected):
    client, _, fsb, sandbox_id = persisted_fsb
    legacy = Mock(spec=KubernetesSandboxService)
    legacy.list_sandbox_objects.return_value = [
        fsb.get_sandbox(sandbox_id).model_copy(update={"id": "legacy"})
    ]
    api = fsb._cr_reader._client.get_custom_objects_api()
    if failure == "list404":
        api.list_namespaced_custom_object.side_effect = ApiException(status=404)
    else:
        legacy.list_sandbox_objects.side_effect = ApiException(status=503)
    monkeypatch.setattr(lifecycle, "sandbox_service", CompositeSandboxService(legacy, fsb))
    response = client.get("/v1/sandboxes")
    assert response.status_code == expected
    if expected == 200:
        listed_ids = [item["id"] for item in response.json()["items"]]
        assert "legacy" in listed_ids
        if failure == "list404":
            assert not [i for i in listed_ids if i.startswith("fsb-")]


@pytest.mark.asyncio
async def test_mixed_create_keeps_legacy_semantics(persisted_fsb):
    _, _, fsb, _ = persisted_fsb
    legacy = Mock(spec=KubernetesSandboxService)
    service = CompositeSandboxService(legacy, fsb)
    request = Mock(template_id=None)
    assert await service.create_sandbox(request) is legacy.create_sandbox.return_value
    legacy.create_sandbox.assert_awaited_once_with(request)


@pytest.mark.parametrize(
    "delta_seconds",
    [-600, 0, 600, 10800],
    ids=["shorten", "unchanged", "extend", "above-create-limit"],
)
def test_http_renew_sets_future_timestamp_without_create_limit(http_fsb, delta_seconds):
    client, fake, _ = http_fsb
    response = client.post(
        "/v1/sandboxes",
        json={
            "image": {"uri": "python:3.11"},
            "entrypoint": ["python"],
            "timeout": 1800,
            "resourceLimits": {"cpu": "500m", "memory": "512Mi"},
        },
    )
    assert response.status_code == 202
    sandbox_id = response.json()["id"]
    cr = fake.crs[("ns-1", sandbox_id)]
    current_expiration = datetime.fromisoformat(cr["spec"]["expireTime"])
    new_expiration = current_expiration + timedelta(seconds=delta_seconds)

    renewed = client.post(
        f"/v1/sandboxes/{sandbox_id}/renew-expiration",
        json={"expiresAt": new_expiration.isoformat()},
    )

    assert renewed.status_code == 200
    assert datetime.fromisoformat(
        renewed.json()["expiresAt"].replace("Z", "+00:00")
    ) == new_expiration
    assert fake.last_update.expires_at_unix_seconds == int(new_expiration.timestamp())
    assert datetime.fromisoformat(cr["spec"]["expireTime"]) == new_expiration


def test_http_renew_and_delete_use_uid_fences(http_fsb):
    client, fake, _ = http_fsb
    created = client.post(
        "/v1/sandboxes",
        json={
            "image": {"uri": "python:3.11"},
            "entrypoint": ["python"],
            "timeout": 3600,
            "resourceLimits": {"cpu": "500m", "memory": "512Mi"},
        },
    ).json()
    sandbox_id = created["id"]
    expires_at = datetime.now(timezone.utc) + timedelta(hours=2)

    renewed = client.post(
        f"/v1/sandboxes/{sandbox_id}/renew-expiration",
        json={"expiresAt": expires_at.isoformat()},
    )
    past = client.post(
        f"/v1/sandboxes/{sandbox_id}/renew-expiration",
        json={"expiresAt": (datetime.now(timezone.utc) - timedelta(seconds=1)).isoformat()},
    )
    deleted = client.delete(f"/v1/sandboxes/{sandbox_id}")

    assert renewed.status_code == 200
    assert fake.last_update.sandbox.expected_uid == f"uid-{sandbox_id}"
    assert fake.last_update.expected_generation == 1
    assert past.status_code == 400
    assert deleted.status_code == 204
    assert fake.last_delete.sandbox.expected_uid == f"uid-{sandbox_id}"


def test_http_renew_maps_fence_conflict_and_missing_sandbox(http_fsb):
    client, fake, _ = http_fsb
    created = client.post(
        "/v1/sandboxes",
        json={
            "image": {"uri": "python:3.11"},
            "entrypoint": ["python"],
            "timeout": 3600,
            "resourceLimits": {"cpu": "500m", "memory": "512Mi"},
        },
    ).json()
    expires_at = datetime.now(timezone.utc) + timedelta(hours=2)
    fake.abort_update_with = grpc.StatusCode.ABORTED

    conflict = client.post(
        f"/v1/sandboxes/{created['id']}/renew-expiration",
        json={"expiresAt": expires_at.isoformat()},
    )
    missing_renew = client.post(
        "/v1/sandboxes/fsb-missing/renew-expiration",
        json={"expiresAt": expires_at.isoformat()},
    )
    missing_delete = client.delete("/v1/sandboxes/fsb-missing")

    assert conflict.status_code == 409
    assert missing_renew.status_code == 404
    assert missing_delete.status_code == 404


def _create_fsb_sandbox(client) -> str:
    return client.post(
        "/v1/sandboxes",
        json={
            "image": {"uri": "python:3.11"},
            "entrypoint": ["python"],
            "timeout": 3600,
            "resourceLimits": {"cpu": "500m", "memory": "512Mi"},
        },
    ).json()["id"]


def test_http_pause_and_resume_use_uid_fences_and_return_accepted(http_fsb):
    client, fake, _ = http_fsb
    sandbox_id = _create_fsb_sandbox(client)

    paused = client.post(f"/v1/sandboxes/{sandbox_id}/pause")
    resumed = client.post(f"/v1/sandboxes/{sandbox_id}/resume")

    assert paused.status_code == 202
    assert resumed.status_code == 202
    assert len(fake.pause_requests) == 1
    assert fake.pause_requests[0].sandbox.expected_uid == f"uid-{sandbox_id}"
    assert fake.pause_requests[0].sandbox.namespaced_name.name == sandbox_id
    assert fake.pause_requests[0].request_id != ""
    assert fake.resume_requests[0].sandbox.expected_uid == f"uid-{sandbox_id}"


def test_http_pause_maps_precondition_to_conflict_and_missing_to_not_found(http_fsb):
    client, fake, _ = http_fsb
    sandbox_id = _create_fsb_sandbox(client)
    fake.abort_pause_with = grpc.StatusCode.FAILED_PRECONDITION

    conflict = client.post(f"/v1/sandboxes/{sandbox_id}/pause")
    missing = client.post("/v1/sandboxes/fsb-missing/pause")

    assert conflict.status_code == 409
    assert missing.status_code == 404


def test_http_resume_maps_precondition_to_conflict(http_fsb):
    client, fake, _ = http_fsb
    sandbox_id = _create_fsb_sandbox(client)
    fake.abort_resume_with = grpc.StatusCode.FAILED_PRECONDITION

    conflict = client.post(f"/v1/sandboxes/{sandbox_id}/resume")
    detail = conflict.json()["detail"]

    assert conflict.status_code == 409
    assert detail["code"] == "FSB::API_ERROR"


def test_http_pause_rejects_terminal_runtime_state(http_fsb):
    client, fake, _ = http_fsb
    sandbox_id = _create_fsb_sandbox(client)
    fake.crs[("ns-1", sandbox_id)]["status"]["runtime"]["state"] = "Failed"

    response = client.post(f"/v1/sandboxes/{sandbox_id}/pause")

    assert response.status_code == 409


def test_http_resume_rejects_paused_without_checkpoint(http_fsb):
    client, fake, _ = http_fsb
    sandbox_id = _create_fsb_sandbox(client)
    fake.crs[("ns-1", sandbox_id)]["status"]["runtime"] = {"state": "Paused"}

    response = client.post(f"/v1/sandboxes/{sandbox_id}/resume")

    assert response.status_code == 409


def test_get_reports_pause_lifecycle_states(http_fsb):
    client, fake, _ = http_fsb
    sandbox_id = _create_fsb_sandbox(client)
    cr_status = fake.crs[("ns-1", sandbox_id)]["status"]

    for cr_state, expected in (
        ("Pausing", "Pausing"),
        ("Paused", "Paused"),
        ("Resuming", "Resuming"),
        ("Ready", "Running"),
    ):
        cr_status["runtime"]["state"] = cr_state
        cr_status["dataPlane"]["state"] = "Ready" if cr_state == "Ready" else "Unavailable"
        cr_status["conditions"] = [
            {
                "type": "Ready",
                "status": "True" if cr_state == "Ready" else "False",
                "observedGeneration": 1,
            }
        ]
        state = client.get(f"/v1/sandboxes/{sandbox_id}").json()["status"]["state"]
        assert state == expected, f"CR runtime {cr_state} must map to {expected}"


def test_diagnostics_use_new_structured_state(http_fsb):
    client, fake, service = http_fsb
    sandbox_id = client.post(
        "/v1/sandboxes",
        json={
            "image": {"uri": "python:3.11"},
            "entrypoint": ["python"],
            "timeout": 3600,
            "resourceLimits": {"cpu": "500m", "memory": "512Mi"},
        },
    ).json()["id"]
    content = service.get_sandbox_events(sandbox_id)

    assert '"runtime_state": "RUNTIME_STATE_READY"' in content
    assert '"data_plane_state": "DATA_PLANE_STATE_READY"' in content
    assert "sandbox ready" in content

    fake.diagnostic_runtime_state = 99
    assert '"runtime_state": "99"' in service.get_sandbox_events(sandbox_id)


def test_event_diagnostics_enforce_stable_scope_contract(http_fsb):
    _, _, service = http_fsb

    with pytest.raises(HTTPException) as exc_info:
        service.get_sandbox_event_diagnostics("fsb-1", "lifecycle")

    assert exc_info.value.status_code == 400
    assert exc_info.value.detail == {
        "code": "DIAGNOSTICS_SCOPE_UNSUPPORTED",
        "message": (
            "Unsupported events diagnostics scope 'lifecycle'. Supported scopes: runtime, all."
        ),
    }


def test_unsupported_fsb_operations_are_explicit(http_fsb):
    _, _, service = http_fsb

    with pytest.raises(HTTPException) as exc_info:
        service.get_sandbox_logs("fsb-1")

    assert exc_info.value.status_code == 501


@pytest.mark.parametrize("pending", [False, True])
def test_http_create_handles_capacity_rejection_and_accepted_pending(http_fsb, pending):
    client, fake, _ = http_fsb
    fake.create_pending = pending
    if not pending:
        fake.reject_create_with = grpc.StatusCode.RESOURCE_EXHAUSTED
    response = client.post(
        "/v1/sandboxes",
        json={
            "image": {"uri": "python:3.11"},
            "entrypoint": ["python"],
            "timeout": 60,
            "resourceLimits": {"cpu": "500m", "memory": "512Mi"},
        },
    )
    if pending:
        assert response.status_code == 202
        assert response.json()["status"]["state"] == "Pending"
    else:
        assert response.status_code == 429
        assert response.headers["Retry-After"] == "1"
        detail = response.json()["detail"]
        assert detail["message"] == "scripted rejection before persistence"
        assert detail["cause"] == "FastPath pool capacity is temporarily unavailable."


class _CapturingHandler(logging.Handler):
    """Collect records directly on the target logger.

    The app's dictConfig disables propagation, so pytest's caplog cannot
    see app records; attach our own (pattern from test_renew_intent_restart).
    """

    def __init__(self) -> None:
        super().__init__(level=logging.WARNING)
        self.records: list[logging.LogRecord] = []

    def emit(self, record: logging.LogRecord) -> None:
        self.records.append(record)


_SERVICE_LOGGER = "opensandbox_server.services.fast_sandbox.service"


def test_capacity_mapping_propagates_and_logs_underlying_fastpath_error(http_fsb):
    _, _, service = http_fsb
    exc = FastPathResourceExhausted(
        "RESOURCE_EXHAUSTED",
        "all Fastlet candidates rejected admission: load Firecracker snapshot: "
        "Error creating KVM object: No such device (os error 19)",
    )

    handler = _CapturingHandler()
    service_logger = logging.getLogger(_SERVICE_LOGGER)
    service_logger.addHandler(handler)
    try:
        error = service._fastpath_http_error(exc)
    finally:
        service_logger.removeHandler(handler)

    assert error.status_code == 429
    assert error.headers == {"Retry-After": "1"}
    assert error.detail["code"] == SandboxErrorCodes.FSB_API_ERROR
    assert error.detail["message"] == exc.message
    assert error.detail["cause"] == "FastPath pool capacity is temporarily unavailable."
    messages = [record.getMessage() for record in handler.records]
    assert messages, "expected a WARNING carrying the underlying FastPath message"
    assert "Error creating KVM object: No such device" in messages[0]


def test_unavailable_mapping_logs_underlying_fastpath_error(http_fsb):
    _, _, service = http_fsb
    exc = FastPathUnavailable("UNAVAILABLE", "gRPC deadline exceeded after 30s")

    handler = _CapturingHandler()
    service_logger = logging.getLogger(_SERVICE_LOGGER)
    service_logger.addHandler(handler)
    try:
        error = service._fastpath_http_error(exc)
    finally:
        service_logger.removeHandler(handler)

    assert error.status_code == 503
    assert error.detail["message"] == "FastPath backend unavailable."
    messages = [record.getMessage() for record in handler.records]
    assert messages, "expected a WARNING carrying the underlying FastPath message"
    assert "gRPC deadline exceeded after 30s" in messages[0]


def _gateway_config(mode="header"):
    return IngressConfig.model_validate(
        {
            "mode": "gateway",
            "gateway": {
                "address": "*.example.com" if mode == "wildcard" else "ingress.example.com",
                "route": {"mode": mode},
            },
            "secure_access": {
                "active_key": "k",
                "keys": [{"key_id": "k", "key": "c2hhcmVkLXNlY3JldA=="}],
            },
        }
    )


@pytest.mark.parametrize("mode", ["header", "uri"])
def test_http_endpoint_matches_go_scope_without_fastpath_lookup(http_fsb, monkeypatch, mode):
    client, _, service = http_fsb
    service._app_config.ingress = _gateway_config(mode)
    fastpath = Mock(spec=FastPathClient)
    monkeypatch.setattr(service, "_fastpath", fastpath)
    previous = get_current_tenant()
    set_current_tenant(TenantEntry(name="tenant-a", namespace="tenant-a"))
    try:
        response = client.get(
            "/v1/sandboxes/sandbox-123/endpoints/44772",
            headers={"X-Namespace": "attacker"},
        )
    finally:
        set_current_tenant(previous)
    assert response.status_code == 200
    scope = "f1.dGVuYW50LWE.c2FuZGJveC0xMjM.44772.k.gtJzW337dCO-kStxh2GPfA"
    body = response.json()
    if mode == "header":
        assert body == {
            "endpoint": "ingress.example.com",
            "headers": {"OpenSandbox-Ingress-To": scope},
        }
    else:
        assert body["endpoint"] == f"ingress.example.com/{scope}"
        assert not body.get("headers")
    assert fastpath.mock_calls == []


@pytest.mark.parametrize("port", [8080, 18080])
def test_endpoint_binds_port_and_default_namespace(http_fsb, port):
    client, _, service = http_fsb
    service._app_config.ingress = _gateway_config()
    response = client.get(f"/v1/sandboxes/fsb-123/endpoints/{port}")
    assert response.status_code == 200
    scope = response.json()["headers"]["OpenSandbox-Ingress-To"]
    assert scope.startswith(f"f1.bnMtMQ.ZnNiLTEyMw.{port}.k.")


@pytest.mark.parametrize(
    "invalid", ["no_gateway", "no_keys", "wildcard", "expires", "port", "identity"]
)
def test_endpoint_rejects_unsupported_or_invalid_routes(http_fsb, invalid):
    _, _, service = http_fsb
    config = service._app_config
    config.ingress = _gateway_config("wildcard" if invalid == "wildcard" else "header")
    if invalid == "no_gateway":
        config.ingress = None
    elif invalid == "no_keys":
        config.ingress.secure_access = None
    with pytest.raises(HTTPException) as exc_info:
        service.get_endpoint(
            "bad\nidentity" if invalid == "identity" else "fsb-123",
            0 if invalid == "port" else 44772,
            expires=2_000_000_000 if invalid == "expires" else None,
        )
    assert exc_info.value.status_code == 400


def test_fsb_snapshot_operations_rejected(http_fsb):
    """Snapshot operations on fsb (fsb-) sandboxes are explicitly rejected
    at the service layer even though the kubernetes snapshot runtime is
    available for container-sandbox workloads."""
    client, fake, service = http_fsb
    sandbox_id = client.post(
        "/v1/sandboxes",
        json={
            "image": {"uri": "python:3.11"},
            "entrypoint": ["python"],
            "timeout": 3600,
            "resourceLimits": {"cpu": "500m", "memory": "512Mi"},
        },
    ).json()["id"]

    response = client.post(f"/v1/sandboxes/{sandbox_id}/snapshots")
    assert response.status_code in (400, 404, 501)


def test_background_renew_resolves_tenant_namespace(http_fsb):
    client, fake, service = http_fsb
    sandbox_id = client.post(
        "/v1/sandboxes",
        json={
            "image": {"uri": "python:3.11"},
            "entrypoint": ["python"],
            "timeout": 3600,
            "resourceLimits": {"cpu": "500m", "memory": "512Mi"},
        },
    ).json()["id"]
    sandbox = fake.sandboxes.pop(("ns-1", sandbox_id))
    fake.sandboxes[("tenant-a", sandbox_id)] = sandbox
    cr = fake.crs.pop(("ns-1", sandbox_id))
    cr["metadata"]["namespace"] = "tenant-a"
    fake.crs[("tenant-a", sandbox_id)] = cr
    service.set_tenant_provider(
        SimpleNamespace(list_tenants=lambda: [SimpleNamespace(namespace="tenant-a")])
    )

    service.renew_expiration(
        sandbox_id,
        request=RenewSandboxExpirationRequest(
            expiresAt=datetime.now(timezone.utc) + timedelta(hours=2)
        ),
    )

    assert fake.last_get.sandbox.namespaced_name.namespace == "tenant-a"
    assert fake.last_update.sandbox.namespaced_name.namespace == "tenant-a"


def test_background_lookup_does_not_treat_fastpath_failure_as_namespace_miss(http_fsb):
    _, fake, service = http_fsb
    fake.get_error_by_namespace["ns-1"] = grpc.StatusCode.UNAVAILABLE
    service.set_tenant_provider(
        SimpleNamespace(list_tenants=lambda: [SimpleNamespace(namespace="tenant-a")])
    )

    with pytest.raises(HTTPException) as exc_info:
        service.renew_expiration(
            "fsb-1",
            request=RenewSandboxExpirationRequest(
                expiresAt=datetime.now(timezone.utc) + timedelta(hours=2)
            ),
        )

    assert exc_info.value.status_code == 503


def test_http_create_returns_503_when_fastpath_is_unavailable(monkeypatch):
    config = AppConfig(
        server=ServerConfig(host="0.0.0.0", port=8080, api_key="x"),
        runtime=RuntimeConfig(type="kubernetes", execd_image="ghcr.io/opensandbox/execd:latest"),
        kubernetes=KubernetesRuntimeConfig(
            namespace="ns-1",
            fastpath_endpoint="127.0.0.1:1",
            fastpath_timeout_seconds=1,
        ),
    )
    fastpath = FastPathClient(endpoint="127.0.0.1:1", timeout_seconds=1)
    service = FastSandboxService(config, fastpath_client=fastpath)
    monkeypatch.setattr(lifecycle, "sandbox_service", service)
    app = FastAPI()
    app.include_router(lifecycle.router, prefix="/v1")

    try:
        with TestClient(app) as client:
            response = client.post(
                "/v1/sandboxes",
                json={
                    "image": {"uri": "python:3.11"},
                    "entrypoint": ["python"],
                    "timeout": 3600,
                    "resourceLimits": {"cpu": "500m", "memory": "512Mi"},
                },
            )
            assert response.status_code == 503
    finally:
        fastpath.close()


@pytest.mark.parametrize("sandbox_id,is_fsb", [("fsb-123", True), ("flt-123", False)])
def test_composite_routes_only_fsb_prefix(sandbox_id, is_fsb):
    kubernetes = Mock(spec=KubernetesSandboxService)
    fsb = Mock(spec=FastSandboxService)
    service = CompositeSandboxService(kubernetes, fsb)
    expected, other = (fsb, kubernetes) if is_fsb else (kubernetes, fsb)

    assert service.get_sandbox(sandbox_id) is expected.get_sandbox.return_value
    expected.get_sandbox.assert_called_once_with(sandbox_id)
    other.get_sandbox.assert_not_called()


def test_fsb_list_excludes_old_prefix(persisted_fsb):
    client, fake, _, sandbox_id = persisted_fsb
    old = deepcopy(fake.crs[("ns-1", sandbox_id)])
    old["metadata"]["name"] = "flt-123"
    fake.crs[("ns-1", "flt-123")] = old

    response = client.get("/v1/sandboxes")
    assert response.status_code == 200
    assert [item["id"] for item in response.json()["items"]] == [sandbox_id]