Secure, Fast, and Extensible Sandbox runtime for AI agents.
# Copyright 2026 The OpenSandbox Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import annotations
import asyncio
import time
from datetime import timedelta
from typing import Any, cast
import httpx
import pytest
from opensandbox._async_pool_reconciler import run_async_reconcile_tick
from opensandbox.config import ConnectionConfig
from opensandbox.exceptions import (
PoolAcquireFailedException,
PoolDestroyedException,
PoolEmptyException,
PoolNotRunningException,
)
from opensandbox.models.sandboxes import (
CredentialProxyConfig,
NetworkPolicy,
PlatformSpec,
)
from opensandbox.pool import (
AcquirePolicy,
AsyncPoolConfig,
InMemoryAsyncPoolStateStore,
PoolCreationSpec,
PooledSandboxCreateContext,
PooledSandboxCreateReason,
SandboxPoolAsync,
)
from opensandbox.pool_types import PoolLifecycleState
@pytest.mark.asyncio
@pytest.mark.parametrize(
"stage",
["get_destroy_state", "manager", "set_idle_entry_ttl", "set_max_idle"],
)
async def test_async_cancelled_start_cleans_up_and_can_restart(
stage: str,
monkeypatch: pytest.MonkeyPatch,
) -> None:
store = InMemoryAsyncPoolStateStore()
entered = asyncio.Event()
release = asyncio.Event()
blocked = False
managers: list[FakeAsyncManager] = []
closed_transports: list[httpx.AsyncHTTPTransport] = []
original_close = httpx.AsyncHTTPTransport.aclose
async def record_close(transport: httpx.AsyncHTTPTransport) -> None:
closed_transports.append(transport)
await original_close(transport)
monkeypatch.setattr(httpx.AsyncHTTPTransport, "aclose", record_close)
async def pause_once() -> None:
nonlocal blocked
if not blocked:
blocked = True
entered.set()
await release.wait()
if stage != "manager":
original_store_call = getattr(store, stage)
async def blocking_store_call(*args: Any, **kwargs: Any) -> Any:
await pause_once()
return await original_store_call(*args, **kwargs)
monkeypatch.setattr(store, stage, blocking_store_call)
async def manager_factory(config: ConnectionConfig) -> Any:
if stage == "manager":
await pause_once()
manager = FakeAsyncManager()
managers.append(manager)
return manager
pool = SandboxPoolAsync(
pool_name="pool",
max_idle=0,
state_store=store,
connection_config=ConnectionConfig(),
creation_spec=PoolCreationSpec(image="ubuntu:22.04"),
sandbox_manager_factory=manager_factory,
sandbox_factory=FakeAsyncSandbox, # type: ignore[arg-type]
)
startup = asyncio.create_task(pool.start())
try:
await asyncio.wait_for(entered.wait(), timeout=2)
startup.cancel()
with pytest.raises(asyncio.CancelledError):
await asyncio.wait_for(startup, timeout=2)
assert (await pool.snapshot()).lifecycle_state == PoolLifecycleState.STOPPED
assert all(manager.closed for manager in managers)
assert len(closed_transports) == (0 if stage == "get_destroy_state" else 1)
await asyncio.wait_for(pool.start(), timeout=2)
assert (await pool.snapshot()).lifecycle_state == PoolLifecycleState.RUNNING
assert not managers[-1].closed
with pytest.raises(PoolEmptyException):
await pool.acquire(policy=AcquirePolicy.FAIL_FAST)
finally:
release.set()
if not startup.done():
startup.cancel()
await asyncio.gather(startup, return_exceptions=True)
await asyncio.wait_for(pool.shutdown(False), timeout=5)
@pytest.mark.asyncio
@pytest.mark.parametrize(
"cleanup_stage", ["release_primary_lock", "manager", "transport"]
)
@pytest.mark.parametrize("startup_failure", ["cancelled", "exception"])
@pytest.mark.parametrize("extra_cancellations", [0, 3])
async def test_async_failed_start_finishes_cleanup_before_propagating(
cleanup_stage: str,
startup_failure: str,
extra_cancellations: int,
monkeypatch: pytest.MonkeyPatch,
) -> None:
store = InMemoryAsyncPoolStateStore()
start_entered = asyncio.Event()
fail_start = asyncio.Event()
cleanup_entered = asyncio.Event()
finish_cleanup = asyncio.Event()
blocked = False
managers: list[FakeAsyncManager] = []
closed_transports: list[httpx.AsyncHTTPTransport] = []
startup_error = RuntimeError("state store unavailable")
original_set_max_idle = store.set_max_idle
original_release_lock = store.release_primary_lock
original_close = httpx.AsyncHTTPTransport.aclose
async def pause_cleanup_once() -> None:
nonlocal blocked
if not blocked:
blocked = True
cleanup_entered.set()
await finish_cleanup.wait()
async def failing_set_max_idle(*args: Any, **kwargs: Any) -> None:
if not start_entered.is_set():
start_entered.set()
await fail_start.wait()
raise startup_error
await original_set_max_idle(*args, **kwargs)
async def release_lock(*args: Any, **kwargs: Any) -> None:
if cleanup_stage == "release_primary_lock":
await pause_cleanup_once()
return await original_release_lock(*args, **kwargs)
async def close_transport(transport: httpx.AsyncHTTPTransport) -> None:
if cleanup_stage == "transport":
await pause_cleanup_once()
await original_close(transport)
closed_transports.append(transport)
class BlockingManager(FakeAsyncManager):
async def close(self) -> None:
if cleanup_stage == "manager":
await pause_cleanup_once()
await super().close()
async def manager_factory(config: ConnectionConfig) -> Any:
manager = BlockingManager()
managers.append(manager)
return manager
monkeypatch.setattr(store, "set_max_idle", failing_set_max_idle)
monkeypatch.setattr(store, "release_primary_lock", release_lock)
monkeypatch.setattr(httpx.AsyncHTTPTransport, "aclose", close_transport)
pool = SandboxPoolAsync(
pool_name="pool",
max_idle=0,
state_store=store,
connection_config=ConnectionConfig(),
creation_spec=PoolCreationSpec(image="ubuntu:22.04"),
sandbox_manager_factory=manager_factory,
sandbox_factory=FakeAsyncSandbox, # type: ignore[arg-type]
)
startup = asyncio.create_task(pool.start())
try:
await asyncio.wait_for(start_entered.wait(), timeout=2)
if startup_failure == "cancelled":
startup.cancel()
else:
fail_start.set()
await asyncio.wait_for(cleanup_entered.wait(), timeout=2)
for _ in range(extra_cancellations):
startup.cancel()
await asyncio.sleep(0)
assert not startup.done(), "startup exited before cleanup finished"
finish_cleanup.set()
if startup_failure == "cancelled" or extra_cancellations:
with pytest.raises(asyncio.CancelledError):
await asyncio.wait_for(startup, timeout=2)
else:
with pytest.raises(RuntimeError) as raised:
await asyncio.wait_for(startup, timeout=2)
assert raised.value is startup_error
assert (await pool.snapshot()).lifecycle_state == PoolLifecycleState.STOPPED
assert managers[0].closed
assert len(closed_transports) == 1
await asyncio.wait_for(pool.start(), timeout=2)
assert (await pool.snapshot()).lifecycle_state == PoolLifecycleState.RUNNING
assert not managers[-1].closed
with pytest.raises(PoolEmptyException):
await pool.acquire(policy=AcquirePolicy.FAIL_FAST)
finally:
fail_start.set()
finish_cleanup.set()
if not startup.done():
startup.cancel()
await asyncio.gather(startup, return_exceptions=True)
await asyncio.wait_for(pool.shutdown(False), timeout=5)
@pytest.mark.asyncio
@pytest.mark.parametrize("cleanup_stage", ["reconcile", "manager", "transport"])
@pytest.mark.parametrize("startup_failure", ["cancelled", "exception"])
@pytest.mark.parametrize(
"cleanup_failure_order",
["no_cancel", "after_cancel", "before_cancel_resumes", "timeout"],
)
async def test_async_failed_start_preserves_cancellation_when_cleanup_fails(
cleanup_stage: str,
startup_failure: str,
cleanup_failure_order: str,
monkeypatch: pytest.MonkeyPatch,
) -> None:
store = InMemoryAsyncPoolStateStore()
start_entered = asyncio.Event()
fail_start = asyncio.Event()
cleanup_entered = asyncio.Event()
finish_cleanup = asyncio.Event()
startup_error = RuntimeError("state store unavailable")
cleanup_error = RuntimeError(f"{cleanup_stage} cleanup failed")
original_set_max_idle = store.set_max_idle
original_close = ConnectionConfig.close_transport_if_owned
managers: list[FakeAsyncManager] = []
async def failing_set_max_idle(*args: Any, **kwargs: Any) -> None:
if start_entered.is_set():
await original_set_max_idle(*args, **kwargs)
return
start_entered.set()
await fail_start.wait()
raise startup_error
async def fail_cleanup_once() -> None:
if not cleanup_entered.is_set():
cleanup_entered.set()
await finish_cleanup.wait()
raise cleanup_error
async def close_transport(config: ConnectionConfig) -> None:
if cleanup_stage == "transport":
await fail_cleanup_once()
await original_close(config)
class FailingManager(FakeAsyncManager):
async def close(self) -> None:
if cleanup_stage == "manager":
await fail_cleanup_once()
await super().close()
async def manager_factory(config: ConnectionConfig) -> Any:
manager = FailingManager()
managers.append(manager)
return manager
monkeypatch.setattr(store, "set_max_idle", failing_set_max_idle)
monkeypatch.setattr(ConnectionConfig, "close_transport_if_owned", close_transport)
pool = SandboxPoolAsync(
pool_name="pool",
max_idle=0,
state_store=store,
connection_config=ConnectionConfig(),
creation_spec=PoolCreationSpec(image="ubuntu:22.04"),
sandbox_manager_factory=manager_factory,
sandbox_factory=FakeAsyncSandbox, # type: ignore[arg-type]
)
original_stop_reconcile = pool._stop_reconcile
async def stop_reconcile(*args: Any, **kwargs: Any) -> None:
await original_stop_reconcile(*args, **kwargs)
if cleanup_stage == "reconcile":
await fail_cleanup_once()
monkeypatch.setattr(pool, "_stop_reconcile", stop_reconcile)
propagated_cancellations: list[asyncio.CancelledError] = []
async def start_pool() -> None:
try:
await pool.start()
except asyncio.CancelledError as exc:
# Python 3.10 wait_for can copy this exception and drop its cause.
propagated_cancellations.append(exc)
raise
startup = asyncio.create_task(start_pool())
timeout_waiter: asyncio.Task[None] | None = None
transport_owner: ConnectionConfig | None = None
try:
await asyncio.wait_for(start_entered.wait(), timeout=2)
transport_owner = pool._pool_transport_owner
if startup_failure == "cancelled":
startup.cancel()
else:
fail_start.set()
await asyncio.wait_for(cleanup_entered.wait(), timeout=2)
if cleanup_failure_order in ("after_cancel", "before_cancel_resumes"):
for _ in range(2):
startup.cancel("cancel during cleanup")
await asyncio.sleep(0)
assert not startup.done(), "startup exited before cleanup finished"
if cleanup_failure_order == "before_cancel_resumes":
# Queue cleanup first so it fails before the caller resumes.
finish_cleanup.set()
startup.cancel("cancel during cleanup")
if cleanup_failure_order == "after_cancel":
await asyncio.sleep(0)
assert not startup.done(), "startup exited before cleanup finished"
elif cleanup_failure_order == "timeout":
timeout_waiter = asyncio.create_task(asyncio.wait_for(startup, timeout=0))
await asyncio.sleep(0)
await asyncio.sleep(0)
assert not timeout_waiter.done()
finish_cleanup.set()
if cleanup_failure_order == "no_cancel":
with pytest.raises(RuntimeError) as raised_error:
await asyncio.wait_for(startup, timeout=2)
assert raised_error.value is cleanup_error
elif timeout_waiter is not None:
with pytest.raises(asyncio.TimeoutError) as raised_timeout:
await asyncio.wait_for(timeout_waiter, timeout=2)
assert isinstance(raised_timeout.value.__cause__, asyncio.CancelledError)
else:
with pytest.raises(asyncio.CancelledError):
await asyncio.wait_for(startup, timeout=2)
if cleanup_failure_order != "no_cancel":
assert len(propagated_cancellations) == 1
cancellation = propagated_cancellations[0]
assert cancellation.__cause__ is cleanup_error
if cleanup_failure_order != "timeout":
assert cancellation.args == ("cancel during cleanup",)
assert startup.cancelled()
assert (await pool.snapshot()).lifecycle_state == PoolLifecycleState.STOPPED
await asyncio.wait_for(pool.start(), timeout=2)
assert (await pool.snapshot()).lifecycle_state == PoolLifecycleState.RUNNING
assert len(managers) == 2
assert not managers[-1].closed
with pytest.raises(PoolEmptyException):
await pool.acquire(policy=AcquirePolicy.FAIL_FAST)
finally:
fail_start.set()
finish_cleanup.set()
if not startup.done():
startup.cancel()
await asyncio.gather(startup, return_exceptions=True)
if timeout_waiter is not None:
await asyncio.gather(timeout_waiter, return_exceptions=True)
await asyncio.wait_for(pool.shutdown(False), timeout=5)
# A failed close can leave the first run's resources open; the fixture
# owns their final cleanup even after a restart replaces the provider.
for manager in managers:
await manager.close()
if transport_owner is not None:
await transport_owner.close_transport_if_owned()
@pytest.mark.asyncio
async def test_async_acquire_fail_fast_empty_raises_pool_empty() -> None:
pool = _create_pool(max_idle=0)
await pool.start()
try:
with pytest.raises(PoolEmptyException) as exc:
await pool.acquire(policy=AcquirePolicy.FAIL_FAST)
assert exc.value.error.code == "POOL_EMPTY"
finally:
await pool.shutdown(False)
@pytest.mark.asyncio
async def test_release_all_idle_preserves_serial_behavior() -> None:
store = InMemoryAsyncPoolStateStore()
for index in range(3):
await store.put_idle("pool", f"idle-{index}")
class TrackingManager(FakeAsyncManager):
def __init__(self) -> None:
super().__init__()
self.active = 0
self.max_active = 0
async def kill_sandbox(self, sandbox_id: str) -> None:
self.active += 1
self.max_active = max(self.max_active, self.active)
await asyncio.sleep(0)
self.killed.append(sandbox_id)
self.active -= 1
manager = TrackingManager()
pool = _create_pool(max_idle=0, store=store, manager=manager)
released = await pool.release_all_idle()
assert released == 3
assert manager.max_active == 1
assert len(manager.killed) == 3
assert manager.closed
@pytest.mark.asyncio
async def test_release_all_idle_parallel_rejects_nonpositive_workers() -> None:
pool = _create_pool(max_idle=0)
with pytest.raises(ValueError, match="max_workers must be positive"):
await pool.release_all_idle_parallel(0)
@pytest.mark.asyncio
async def test_release_all_idle_bounds_kills_and_cleans_up_before_store_failure() -> (
None
):
class FailingStore(InMemoryAsyncPoolStateStore):
def __init__(self) -> None:
super().__init__()
self.takes = 0
async def try_take_idle(self, pool_name: str) -> str | None:
if self.takes == 55:
raise RuntimeError("injected store failure")
self.takes += 1
return await super().try_take_idle(pool_name)
store = FailingStore()
for index in range(55):
await store.put_idle("pool", f"idle-{index}")
class ConcurrentManager(FakeAsyncManager):
def __init__(self) -> None:
super().__init__()
self.active = 0
self.max_active = 0
self.ready = asyncio.Event()
async def kill_sandbox(self, sandbox_id: str) -> None:
self.active += 1
self.max_active = max(self.max_active, self.active)
if self.active == 50:
self.ready.set()
await self.ready.wait()
self.killed.append(sandbox_id)
self.active -= 1
if sandbox_id == "idle-0":
raise RuntimeError("injected kill failure")
manager = ConcurrentManager()
pool = _create_pool(max_idle=0, store=store, manager=manager)
with pytest.raises(RuntimeError, match="injected store failure"):
await asyncio.wait_for(pool.release_all_idle_parallel(), timeout=2)
assert manager.max_active == 50
assert len(manager.killed) == 55
assert (await store.snapshot_counters("pool")).idle_count == 0
assert manager.closed
@pytest.mark.asyncio
async def test_release_all_idle_parallel_finishes_kills_before_cancellation() -> None:
store = InMemoryAsyncPoolStateStore()
for index in range(55):
await store.put_idle("pool", f"idle-{index}")
class BlockingManager(FakeAsyncManager):
def __init__(self) -> None:
super().__init__()
self.started = 0
self.first_batch_started = asyncio.Event()
self.release_kills = asyncio.Event()
async def kill_sandbox(self, sandbox_id: str) -> None:
self.started += 1
if self.started == 50:
self.first_batch_started.set()
await self.release_kills.wait()
self.killed.append(sandbox_id)
manager = BlockingManager()
pool = _create_pool(max_idle=0, store=store, manager=manager)
release_task = asyncio.create_task(pool.release_all_idle_parallel())
await asyncio.wait_for(manager.first_batch_started.wait(), timeout=2)
try:
release_task.cancel()
await asyncio.sleep(0)
assert not release_task.done()
manager.release_kills.set()
with pytest.raises(asyncio.CancelledError):
await release_task
finally:
manager.release_kills.set()
await asyncio.gather(release_task, return_exceptions=True)
assert len(manager.killed) == 55
assert (await store.snapshot_counters("pool")).idle_count == 0
assert manager.closed
@pytest.mark.asyncio
async def test_async_reconcile_submits_at_most_warmup_create_qps() -> None:
store = InMemoryAsyncPoolStateStore()
config = AsyncPoolConfig(
pool_name="pool",
owner_id="owner-1",
max_idle=10,
warmup_concurrency=10,
state_store=store,
connection_config=ConnectionConfig(),
creation_spec=PoolCreationSpec(image="ubuntu:22.04"),
)
submitted: list[int] = []
await run_async_reconcile_tick(
config=config,
state_store=store,
on_discard_sandbox=_noop_discard,
warming_count=0,
submit_warmups=submitted.append,
)
assert submitted == [10]
@pytest.mark.asyncio
async def test_async_reconcile_accounts_for_warming_before_admission() -> None:
store = InMemoryAsyncPoolStateStore()
config = AsyncPoolConfig(
pool_name="pool",
owner_id="owner-1",
max_idle=2,
warmup_concurrency=2,
state_store=store,
connection_config=ConnectionConfig(),
creation_spec=PoolCreationSpec(image="ubuntu:22.04"),
)
submitted: list[int] = []
await run_async_reconcile_tick(
config=config,
state_store=store,
on_discard_sandbox=_noop_discard,
warming_count=1,
submit_warmups=submitted.append,
)
assert submitted == [1]
@pytest.mark.asyncio
async def test_async_reconcile_returns_without_waiting_for_submitted_warmup() -> None:
store = InMemoryAsyncPoolStateStore()
config = AsyncPoolConfig(
pool_name="pool",
owner_id="owner-1",
max_idle=2,
warmup_concurrency=2,
state_store=store,
connection_config=ConnectionConfig(),
creation_spec=PoolCreationSpec(image="ubuntu:22.04"),
)
called = asyncio.Event()
def submit(count: int) -> None:
assert count == 2
called.set()
await run_async_reconcile_tick(
config=config,
state_store=store,
on_discard_sandbox=_noop_discard,
warming_count=0,
submit_warmups=submit,
)
assert called.is_set()
@pytest.mark.asyncio
async def test_async_acquire_fail_fast_stale_idle_raises_and_kills_candidate() -> None:
store = InMemoryAsyncPoolStateStore()
await store.put_idle("pool", "stale-1")
manager = FakeAsyncManager()
pool = _create_pool(max_idle=0, store=store, manager=manager)
await pool.start()
try:
with pytest.raises(PoolAcquireFailedException) as exc:
await pool.acquire(policy=AcquirePolicy.FAIL_FAST)
assert exc.value.error.code == "POOL_ACQUIRE_FAILED"
assert (await store.snapshot_counters("pool")).idle_count == 0
# Kill is now fire-and-forget (retry-loop must not block on slow DELETE) so wait for
# the background task to observe the kill.
async def _killed_stale_1() -> bool:
return manager.killed == ["stale-1"]
await _eventually(_killed_stale_1)
finally:
await pool.shutdown(False)
@pytest.mark.asyncio
async def test_async_acquire_direct_create_when_empty() -> None:
FakeAsyncSandbox.reset()
pool = _create_pool(max_idle=0)
await pool.start()
try:
sandbox = await pool.acquire(sandbox_timeout=timedelta(minutes=5))
fake_sandbox = cast(FakeAsyncSandbox, sandbox)
assert sandbox.id == "created-1"
assert fake_sandbox.renewed == [timedelta(minutes=5)]
finally:
await pool.shutdown(False)
@pytest.mark.asyncio
async def test_async_acquire_does_not_direct_create_when_pool_namespace_is_destroying() -> (
None
):
FakeAsyncSandbox.reset()
store = InMemoryAsyncPoolStateStore()
pool = _create_pool(max_idle=0, store=store)
await pool.start()
try:
await store.begin_destroy("pool", "destroyer")
with pytest.raises(PoolDestroyedException):
await pool.acquire()
assert FakeAsyncSandbox.created_count == 0
finally:
await pool.shutdown(False)
@pytest.mark.asyncio
async def test_async_acquire_idle_destroy_race_raises_pool_destroyed() -> None:
store = InMemoryAsyncPoolStateStore()
await store.put_idle("pool", "id-1")
connected: list[FakeAsyncSandbox] = []
class FencingAsyncSandbox(FakeAsyncSandbox):
@classmethod
async def connect(
cls, sandbox_id: str, *args: Any, **kwargs: Any
) -> FakeAsyncSandbox:
sandbox = cls(sandbox_id)
connected.append(sandbox)
await store.begin_destroy("pool", "destroyer")
return sandbox
pool = SandboxPoolAsync(
pool_name="pool",
owner_id="owner-1",
max_idle=0,
state_store=store,
connection_config=ConnectionConfig(),
creation_spec=PoolCreationSpec(image="ubuntu:22.04"),
sandbox_manager_factory=lambda config: _manager_factory(FakeAsyncManager()),
sandbox_factory=FencingAsyncSandbox, # type: ignore[arg-type]
)
await pool.start()
try:
with pytest.raises(PoolDestroyedException):
await pool.acquire(policy=AcquirePolicy.DIRECT_CREATE)
assert connected[0].killed
assert connected[0].closed
finally:
await pool.shutdown(False)
@pytest.mark.asyncio
async def test_async_acquire_stopped_destroyed_pool_raises_pool_destroyed() -> None:
store = InMemoryAsyncPoolStateStore()
pool = _create_pool(max_idle=0, store=store)
await pool.start()
await store.begin_destroy("pool", "destroyer")
await pool.shutdown(False)
with pytest.raises(PoolDestroyedException):
await pool.acquire()
@pytest.mark.asyncio
async def test_async_acquire_destroy_race_preserves_pool_destroyed_when_cleanup_fails() -> (
None
):
store = InMemoryAsyncPoolStateStore()
await store.put_idle("pool", "id-1")
class CleanupFailingAsyncSandbox(FakeAsyncSandbox):
@classmethod
async def connect(
cls, sandbox_id: str, *args: Any, **kwargs: Any
) -> FakeAsyncSandbox:
await store.begin_destroy("pool", "destroyer")
return cls(sandbox_id)
async def kill(self) -> None:
raise RuntimeError("kill failed")
async def close(self) -> None:
raise RuntimeError("close failed")
pool = SandboxPoolAsync(
pool_name="pool",
owner_id="owner-1",
max_idle=0,
state_store=store,
connection_config=ConnectionConfig(),
creation_spec=PoolCreationSpec(image="ubuntu:22.04"),
sandbox_manager_factory=lambda config: _manager_factory(FakeAsyncManager()),
sandbox_factory=CleanupFailingAsyncSandbox, # type: ignore[arg-type]
)
await pool.start()
try:
with pytest.raises(PoolDestroyedException):
await pool.acquire(policy=AcquirePolicy.DIRECT_CREATE)
finally:
await pool.shutdown(False)
@pytest.mark.asyncio
async def test_async_acquire_direct_create_forwards_pool_creation_platform() -> None:
captured_kwargs: dict[str, Any] = {}
class CapturingAsyncSandbox(FakeAsyncSandbox):
@classmethod
async def create(cls, *args: Any, **kwargs: Any) -> CapturingAsyncSandbox:
captured_kwargs.update(kwargs)
return cls("created-with-platform")
pool = SandboxPoolAsync(
pool_name="pool",
owner_id="owner-1",
max_idle=0,
state_store=InMemoryAsyncPoolStateStore(),
connection_config=ConnectionConfig(),
creation_spec=PoolCreationSpec(
image="ubuntu:22.04",
platform=PlatformSpec(os="linux", arch="arm64"),
),
sandbox_factory=CapturingAsyncSandbox, # type: ignore[arg-type]
)
await pool.start()
try:
await pool.acquire()
assert captured_kwargs["platform"] == PlatformSpec(os="linux", arch="arm64")
finally:
await pool.shutdown(False)
@pytest.mark.asyncio
@pytest.mark.parametrize("warmup", [False, True], ids=["direct", "warmup"])
@pytest.mark.parametrize(
"credential_proxy",
[None, CredentialProxyConfig(enabled=False), CredentialProxyConfig(enabled=True)],
ids=["omitted", "disabled", "enabled"],
)
async def test_async_pool_creation_forwards_credential_proxy(
warmup: bool, credential_proxy: CredentialProxyConfig | None
) -> None:
captured_kwargs: dict[str, Any] = {}
class CapturingAsyncSandbox(FakeAsyncSandbox):
@classmethod
async def create(cls, *args: Any, **kwargs: Any) -> CapturingAsyncSandbox:
captured_kwargs.update(kwargs)
return cls("created-with-credential-proxy")
network_policy = NetworkPolicy(defaultAction="deny")
spec_kwargs = {"credential_proxy": credential_proxy} if credential_proxy else {}
pool = SandboxPoolAsync(
pool_name="pool",
max_idle=1 if warmup else 0,
state_store=InMemoryAsyncPoolStateStore(),
connection_config=ConnectionConfig(),
creation_spec=PoolCreationSpec(
image="ubuntu:22.04", network_policy=network_policy, **spec_kwargs
),
sandbox_manager_factory=lambda config: _manager_factory(FakeAsyncManager()), # type: ignore[arg-type]
sandbox_factory=CapturingAsyncSandbox, # type: ignore[arg-type]
)
await pool.start()
try:
if warmup:
await _eventually(lambda: _idle_count_equals(pool, 1))
else:
await pool.acquire()
if credential_proxy is None:
assert "credential_proxy" not in captured_kwargs
else:
assert captured_kwargs["credential_proxy"] is credential_proxy
assert captured_kwargs["network_policy"] is network_policy
finally:
await pool.shutdown(False)
@pytest.mark.asyncio
@pytest.mark.parametrize("warmup", [False, True], ids=["direct", "warmup"])
async def test_async_pool_creation_supports_legacy_factory(warmup: bool) -> None:
class LegacySandbox(FakeAsyncSandbox):
@classmethod
async def create(
cls,
image: Any,
*,
timeout: Any,
ready_timeout: Any,
env: Any,
metadata: Any,
resource: Any,
network_policy: Any,
platform: Any,
extensions: Any,
secure_access: Any,
entrypoint: Any,
volumes: Any,
connection_config: Any,
health_check: Any,
health_check_polling_interval: Any,
skip_health_check: Any,
) -> LegacySandbox:
return cls("legacy-factory-sandbox")
pool = SandboxPoolAsync(
pool_name="pool",
max_idle=1 if warmup else 0,
state_store=InMemoryAsyncPoolStateStore(),
connection_config=ConnectionConfig(),
creation_spec=PoolCreationSpec(image="ubuntu:22.04"),
sandbox_manager_factory=lambda config: _manager_factory(FakeAsyncManager()), # type: ignore[arg-type]
sandbox_factory=LegacySandbox, # type: ignore[arg-type]
)
await pool.start()
try:
if warmup:
await _eventually(lambda: _idle_count_equals(pool, 1))
else:
sandbox = await pool.acquire()
assert sandbox.id == "legacy-factory-sandbox"
finally:
await pool.shutdown(False)
@pytest.mark.asyncio
async def test_async_acquire_direct_create_kills_and_closes_when_renew_fails() -> None:
FakeAsyncSandbox.reset()
FakeAsyncSandbox.fail_renew = True
pool = _create_pool(max_idle=0)
await pool.start()
try:
with pytest.raises(RuntimeError, match="renew failed"):
await pool.acquire(sandbox_timeout=timedelta(minutes=5))
assert FakeAsyncSandbox.last_created is not None
assert FakeAsyncSandbox.last_created.killed
assert FakeAsyncSandbox.last_created.closed
finally:
FakeAsyncSandbox.fail_renew = False
await pool.shutdown(False)
@pytest.mark.asyncio
@pytest.mark.parametrize("source", ["idle", "direct", "creator"])
@pytest.mark.parametrize("phase", ["renew", "fence"])
async def test_acquire_cancellation_finishes_owned_sandbox_cleanup(
source: str,
phase: str,
) -> None:
entered = asyncio.Event()
kill_started = asyncio.Event()
finish_kill = asyncio.Event()
never = asyncio.Event()
class BlockingSandbox(FakeAsyncSandbox):
def __init__(self) -> None:
super().__init__("owned-sandbox")
self.client = httpx.AsyncClient()
async def renew(self, timeout: timedelta) -> None:
if phase == "renew":
entered.set()
await never.wait()
async def kill(self) -> None:
kill_started.set()
await finish_kill.wait()
self.killed = True
async def close(self) -> None:
await self.client.aclose()
self.closed = True
sandbox = BlockingSandbox()
class BlockingStore(InMemoryAsyncPoolStateStore):
async def get_destroy_state(self, pool_name: str) -> Any:
if phase == "fence" and returned_sandbox:
entered.set()
await never.wait()
return await super().get_destroy_state(pool_name)
returned_sandbox = False
async def provide(*args: Any, **kwargs: Any) -> BlockingSandbox:
nonlocal returned_sandbox
returned_sandbox = True
return sandbox
class Factory:
create = staticmethod(provide)
connect = staticmethod(provide)
store = BlockingStore()
pool = SandboxPoolAsync(
pool_name="pool",
max_idle=0,
state_store=store,
connection_config=ConnectionConfig(),
creation_spec=PoolCreationSpec(image="ubuntu:22.04"),
sandbox_creator=provide if source == "creator" else None,
sandbox_factory=Factory, # type: ignore[arg-type]
sandbox_manager_factory=lambda config: _manager_factory(FakeAsyncManager()),
)
await pool.start()
if source == "idle":
await store.put_idle("pool", sandbox.id)
acquisition = asyncio.create_task(
pool.acquire(sandbox_timeout=timedelta(minutes=5))
)
try:
await asyncio.wait_for(entered.wait(), timeout=2)
acquisition.cancel()
await asyncio.wait_for(kill_started.wait(), timeout=2)
acquisition.cancel()
finish_kill.set()
with pytest.raises(asyncio.CancelledError):
await acquisition
assert sandbox.killed
assert sandbox.client.is_closed
assert (await store.snapshot_counters("pool")).idle_count == 0
finally:
returned_sandbox = False
finish_kill.set()
acquisition.cancel()
await asyncio.gather(acquisition, return_exceptions=True)
await sandbox.client.aclose()
await pool.shutdown(False)
@pytest.mark.asyncio
async def test_async_direct_create_failure_after_run_retired_is_pool_not_running() -> (
None
):
pool = _create_pool(max_idle=0)
await pool.start()
async def fail_after_retire(*args: Any, **kwargs: Any) -> FakeAsyncSandbox:
pool._lifecycle_state = PoolLifecycleState.STOPPED
raise RuntimeError("transport closed during shutdown")
pool._direct_create = fail_after_retire # type: ignore[method-assign]
try:
with pytest.raises(PoolNotRunningException) as raised:
await pool.acquire()
assert isinstance(raised.value.__cause__, RuntimeError)
finally:
await pool.shutdown(False)
@pytest.mark.asyncio
async def test_async_acquire_direct_create_uses_sandbox_creator() -> None:
contexts: list[PooledSandboxCreateContext] = []
async def creator(context: PooledSandboxCreateContext) -> FakeAsyncSandbox:
contexts.append(context)
return FakeAsyncSandbox("created-by-hook")
pool = SandboxPoolAsync(
pool_name="pool",
owner_id="owner-1",
max_idle=0,
state_store=InMemoryAsyncPoolStateStore(),
connection_config=ConnectionConfig(),
creation_spec=PoolCreationSpec(image="ubuntu:22.04"),
idle_timeout=timedelta(minutes=10),
sandbox_creator=creator,
sandbox_manager_factory=lambda config: _manager_factory(FakeAsyncManager()),
sandbox_factory=FakeAsyncSandbox, # type: ignore[arg-type]
)
await pool.start()
try:
sandbox = await pool.acquire(sandbox_timeout=timedelta(minutes=5))
fake_sandbox = cast(FakeAsyncSandbox, sandbox)
assert sandbox.id == "created-by-hook"
assert fake_sandbox.renewed == [timedelta(minutes=5)]
assert len(contexts) == 1
assert contexts[0].pool_name == "pool"
assert contexts[0].owner_id == "owner-1"
assert contexts[0].idle_timeout == timedelta(minutes=10)
assert contexts[0].reason is PooledSandboxCreateReason.DIRECT_CREATE
assert contexts[0].ready_timeout == pool._config.acquire_ready_timeout
assert (
contexts[0].health_check_polling_interval
== pool._config.acquire_health_check_polling_interval
)
assert contexts[0].skip_health_check is False
assert contexts[0].health_check is None
assert isinstance(contexts[0].connection_config, ConnectionConfig)
finally:
await pool.shutdown(False)
@pytest.mark.asyncio
async def test_async_acquire_when_stopped_raises_pool_not_running() -> None:
pool = _create_pool(max_idle=0)
with pytest.raises(PoolNotRunningException) as exc:
await pool.acquire(policy=AcquirePolicy.FAIL_FAST)
assert exc.value.error.code == "POOL_NOT_RUNNING"
@pytest.mark.asyncio
async def test_async_start_warms_idle_and_resize_zero_shrinks() -> None:
FakeAsyncSandbox.reset()
store = InMemoryAsyncPoolStateStore()
manager = FakeAsyncManager()
pool = _create_pool(max_idle=2, store=store, manager=manager)
await pool.start()
try:
await _eventually(lambda: _idle_count_equals(pool, 2))
await pool.resize(0)
await _eventually(lambda: _idle_count_equals(pool, 0))
assert len(manager.killed) >= 2
finally:
await pool.shutdown(False)
@pytest.mark.asyncio
async def test_async_start_overwrites_shared_max_idle_with_user_config() -> None:
store = SharedAsyncMaxIdleStore(initial_max_idle=0)
pool = _create_pool(max_idle=3, store=store)
await pool.start()
try:
assert store.max_idle_by_pool["pool"] == 3
assert store.set_max_idle_calls == [("pool", 3)]
assert (await pool.snapshot()).max_idle == 3
finally:
await pool.shutdown(False)
@pytest.mark.asyncio
async def test_async_resize_only_updates_target_without_immediate_reconcile_trigger() -> (
None
):
pool = SandboxPoolAsync(
pool_name="pool",
owner_id="owner-1",
max_idle=0,
state_store=InMemoryAsyncPoolStateStore(),
connection_config=ConnectionConfig(),
creation_spec=PoolCreationSpec(image="ubuntu:22.04"),
sandbox_manager_factory=lambda config: _manager_factory(FakeAsyncManager()),
sandbox_factory=FakeAsyncSandbox, # type: ignore[arg-type]
)
await pool.start()
calls = 0
async def record_reconcile() -> None:
nonlocal calls
calls += 1
pool._run_reconcile_tick = record_reconcile # type: ignore[method-assign]
try:
await pool.resize(1)
await asyncio.sleep(0.05)
assert calls == 0
assert (await pool.snapshot()).max_idle == 1
finally:
await pool.shutdown(False)
@pytest.mark.asyncio
async def test_async_graceful_shutdown_waits_for_running_warmup_before_stop() -> None:
FakeAsyncSandbox.reset()
entered_preparer = asyncio.Event()
release_preparer = asyncio.Event()
async def blocking_preparer(sandbox: FakeAsyncSandbox) -> None:
entered_preparer.set()
await release_preparer.wait()
pool = SandboxPoolAsync(
pool_name="pool",
owner_id="owner-1",
max_idle=1,
warmup_concurrency=1,
state_store=InMemoryAsyncPoolStateStore(),
connection_config=ConnectionConfig(),
creation_spec=PoolCreationSpec(image="ubuntu:22.04"),
primary_lock_ttl=timedelta(seconds=5),
drain_timeout=timedelta(milliseconds=50),
warmup_sandbox_preparer=blocking_preparer, # type: ignore[arg-type]
sandbox_manager_factory=lambda config: _manager_factory(FakeAsyncManager()),
sandbox_factory=FakeAsyncSandbox, # type: ignore[arg-type]
)
await pool.start()
try:
await asyncio.wait_for(entered_preparer.wait(), timeout=2)
async def release_after_delay() -> None:
await asyncio.sleep(0.05)
release_preparer.set()
release_task = asyncio.create_task(release_after_delay())
started = time.monotonic()
await pool.shutdown(graceful=True)
elapsed = time.monotonic() - started
await release_task
assert elapsed >= 0.04
assert (await pool.snapshot()).lifecycle_state.value == "STOPPED"
finally:
release_preparer.set()
await pool.shutdown(False)
@pytest.mark.asyncio
async def test_async_graceful_shutdown_restart_does_not_reuse_stop_event() -> None:
pool = _create_pool(max_idle=0)
await pool.start()
first_stop_event = pool._stop_event
try:
await pool.shutdown(graceful=True)
assert first_stop_event.is_set()
await pool.start()
assert pool._stop_event is not first_stop_event
assert first_stop_event.is_set()
finally:
await pool.shutdown(False)
@pytest.mark.asyncio
async def test_async_user_managed_transport_is_preserved_for_pool_resources() -> None:
transport = _AsyncTransport()
connection_config = ConnectionConfig(transport=transport)
manager_configs: list[ConnectionConfig] = []
sandbox_configs: list[ConnectionConfig] = []
class CapturingAsyncSandbox(FakeAsyncSandbox):
@classmethod
async def create(cls, *args: Any, **kwargs: Any) -> CapturingAsyncSandbox:
sandbox_configs.append(kwargs["connection_config"])
return cls("created-with-custom-transport")
async def manager_factory(config: ConnectionConfig) -> FakeAsyncManager:
manager_configs.append(config)
return FakeAsyncManager()
pool = SandboxPoolAsync(
pool_name="pool",
owner_id="owner-1",
max_idle=0,
state_store=InMemoryAsyncPoolStateStore(),
connection_config=connection_config,
creation_spec=PoolCreationSpec(image="ubuntu:22.04"),
sandbox_manager_factory=manager_factory, # type: ignore[arg-type]
sandbox_factory=CapturingAsyncSandbox, # type: ignore[arg-type]
)
await pool.start()
try:
await pool.acquire()
assert manager_configs[0].transport is transport
assert not manager_configs[0]._owns_transport
assert sandbox_configs[0].transport is transport
assert not sandbox_configs[0]._owns_transport
finally:
await pool.shutdown(False)
@pytest.mark.asyncio
async def test_async_default_warmup_owns_transport_while_manager_uses_shared(
monkeypatch: pytest.MonkeyPatch,
) -> None:
manager_configs: list[ConnectionConfig] = []
sandbox_configs: list[ConnectionConfig] = []
transport_limits: dict[str, object] = {}
original_with_transport = ConnectionConfig.with_transport_if_missing
def capture_transport_limits(
config: ConnectionConfig, **kwargs: object
) -> ConnectionConfig:
transport_limits.update(kwargs)
return original_with_transport(config, **kwargs) # type: ignore[arg-type]
monkeypatch.setattr(
ConnectionConfig, "with_transport_if_missing", capture_transport_limits
)
class CapturingAsyncSandbox(FakeAsyncSandbox):
@classmethod
async def create(cls, *args: Any, **kwargs: Any) -> CapturingAsyncSandbox:
sandbox_configs.append(kwargs["connection_config"])
return cls("created-with-shared-transport")
async def manager_factory(config: ConnectionConfig) -> FakeAsyncManager:
manager_configs.append(config)
return FakeAsyncManager()
pool = SandboxPoolAsync(
pool_name="pool",
owner_id="owner-1",
max_idle=1,
state_store=InMemoryAsyncPoolStateStore(),
connection_config=ConnectionConfig(),
creation_spec=PoolCreationSpec(image="ubuntu:22.04"),
sandbox_manager_factory=manager_factory, # type: ignore[arg-type]
sandbox_factory=CapturingAsyncSandbox, # type: ignore[arg-type]
)
await pool.start()
try:
await _eventually(lambda: _idle_count_equals(pool, 1))
assert manager_configs[0].transport is not None
assert transport_limits["max_connections"] is None
assert transport_limits["max_keepalive_connections"] == 128
assert sandbox_configs[0].transport is None
assert manager_configs[0]._owns_transport
assert sandbox_configs[0]._owns_transport
assert sandbox_configs[0].retry_policy.max_retries == 0
assert sandbox_configs[0].disable_metrics is True
another_warmup_config = pool._connection_for_warmup_create()
assert another_warmup_config is not sandbox_configs[0]
assert another_warmup_config.transport is None
assert another_warmup_config._owns_transport
assert another_warmup_config.disable_metrics is True
finally:
await pool.shutdown(False)
@pytest.mark.asyncio
async def test_async_warmup_connection_disables_metrics_with_shared_transport() -> None:
transport = _AsyncTransport()
connection_config = ConnectionConfig(transport=transport)
pool = SandboxPoolAsync(
pool_name="pool",
owner_id="owner-1",
max_idle=0,
state_store=InMemoryAsyncPoolStateStore(),
connection_config=connection_config,
creation_spec=PoolCreationSpec(image="ubuntu:22.04"),
sandbox_manager_factory=lambda config: _manager_factory(FakeAsyncManager()),
sandbox_factory=FakeAsyncSandbox, # type: ignore[arg-type]
)
await pool.start()
try:
warmup_config = pool._connection_for_warmup_create()
assert warmup_config is not connection_config
assert warmup_config.disable_metrics is True
assert warmup_config.transport is transport
assert connection_config.disable_metrics is False
finally:
await pool.shutdown(False)
async def test_async_staged_warmup_runs_in_order() -> None:
events: list[str] = []
class StagedSandbox(FakeAsyncSandbox):
@classmethod
async def create(cls, *args: Any, **kwargs: Any) -> StagedSandbox:
assert kwargs["skip_health_check"] is True
events.append("create")
return cls("staged-1")
async def is_healthy(self) -> bool:
events.append("readiness")
return True
async def renew(self, timeout: timedelta) -> None:
events.append("renew")
await super().renew(timeout)
async def prepare(sandbox: FakeAsyncSandbox) -> None:
events.append("prepare")
async def post_prepare(sandbox: FakeAsyncSandbox) -> bool:
events.append("post-prepare")
return True
async def manager_factory(config: ConnectionConfig) -> FakeAsyncManager:
return FakeAsyncManager()
pool = SandboxPoolAsync(
pool_name="staged",
max_idle=1,
warmup_create_qps=1,
warmup_concurrency=1,
state_store=InMemoryAsyncPoolStateStore(),
connection_config=ConnectionConfig(),
creation_spec=PoolCreationSpec(image="ubuntu:22.04"),
warmup_sandbox_preparer=prepare, # type: ignore[arg-type]
warmup_post_prepare_health_check=post_prepare, # type: ignore[arg-type]
sandbox_manager_factory=manager_factory, # type: ignore[arg-type]
sandbox_factory=StagedSandbox, # type: ignore[arg-type]
)
await pool.start()
try:
await _eventually(lambda: _idle_count_equals(pool, 1))
assert events[:5] == [
"create",
"readiness",
"prepare",
"post-prepare",
"renew",
]
finally:
await pool.shutdown(False)
async def test_async_warmup_polling_delay_does_not_hold_concurrency_slot() -> None:
created = 0
attempts: dict[str, int] = {}
first_round: list[str] = []
class PollingSandbox(FakeAsyncSandbox):
@classmethod
async def create(cls, *args: Any, **kwargs: Any) -> PollingSandbox:
nonlocal created
created += 1
return cls(f"polling-{created}")
async def health(sandbox: FakeAsyncSandbox) -> bool:
attempts[sandbox.id] = attempts.get(sandbox.id, 0) + 1
if attempts[sandbox.id] == 1:
first_round.append(sandbox.id)
return attempts[sandbox.id] >= 2
async def manager_factory(config: ConnectionConfig) -> FakeAsyncManager:
return FakeAsyncManager()
pool = SandboxPoolAsync(
pool_name="polling-slots",
max_idle=2,
warmup_create_qps=2,
warmup_concurrency=1,
state_store=InMemoryAsyncPoolStateStore(),
connection_config=ConnectionConfig(),
creation_spec=PoolCreationSpec(image="ubuntu:22.04"),
warmup_health_check=health, # type: ignore[arg-type]
warmup_health_check_polling_interval=timedelta(milliseconds=100),
sandbox_manager_factory=manager_factory, # type: ignore[arg-type]
sandbox_factory=PollingSandbox, # type: ignore[arg-type]
)
await pool.start()
try:
await _eventually(lambda: _idle_count_equals(pool, 2))
assert len(set(first_round[:2])) == 2
finally:
await pool.shutdown(False)
async def test_async_primary_heartbeat_continues_while_preparer_is_blocked() -> None:
class CountingStore(InMemoryAsyncPoolStateStore):
def __init__(self) -> None:
super().__init__()
self.renew_calls = 0
async def renew_primary_lock(
self, pool_name: str, owner_id: str, ttl: timedelta
) -> bool:
self.renew_calls += 1
return await super().renew_primary_lock(pool_name, owner_id, ttl)
store = CountingStore()
entered = asyncio.Event()
release = asyncio.Event()
async def prepare(sandbox: FakeAsyncSandbox) -> None:
entered.set()
await release.wait()
async def manager_factory(config: ConnectionConfig) -> FakeAsyncManager:
return FakeAsyncManager()
pool = SandboxPoolAsync(
pool_name="heartbeat",
owner_id="owner-1",
max_idle=1,
state_store=store,
connection_config=ConnectionConfig(),
creation_spec=PoolCreationSpec(image="ubuntu:22.04"),
primary_lock_ttl=timedelta(milliseconds=90),
warmup_sandbox_preparer=prepare, # type: ignore[arg-type]
sandbox_manager_factory=manager_factory, # type: ignore[arg-type]
sandbox_factory=FakeAsyncSandbox, # type: ignore[arg-type]
)
await pool.start()
try:
await asyncio.wait_for(entered.wait(), timeout=2)
await asyncio.sleep(0.2)
assert store.renew_calls >= 2
release.set()
await _eventually(lambda: _idle_count_equals(pool, 1))
finally:
release.set()
await pool.shutdown(False)
async def test_async_retired_acquire_cannot_consume_restarted_run_idle() -> None:
store = InMemoryAsyncPoolStateStore()
await store.put_idle("pool", "old-run")
entered = asyncio.Event()
release = asyncio.Event()
class BlockingFactory(FakeAsyncSandbox):
@classmethod
async def connect(
cls, sandbox_id: str, *args: Any, **kwargs: Any
) -> FakeAsyncSandbox:
if sandbox_id == "old-run":
entered.set()
await release.wait()
raise RuntimeError("old candidate failed")
return cls(sandbox_id)
async def manager_factory(config: ConnectionConfig) -> FakeAsyncManager:
return FakeAsyncManager()
pool = SandboxPoolAsync(
pool_name="pool",
owner_id="owner-1",
max_idle=0,
max_acquire_retries=2,
state_store=store,
connection_config=ConnectionConfig(),
creation_spec=PoolCreationSpec(image="ubuntu:22.04"),
sandbox_manager_factory=manager_factory, # type: ignore[arg-type]
sandbox_factory=BlockingFactory, # type: ignore[arg-type]
)
await pool.start()
acquire = asyncio.create_task(pool.acquire(policy=AcquirePolicy.RETRY_NEXT_IDLE))
await asyncio.wait_for(entered.wait(), timeout=1)
await pool.shutdown(False)
await pool.start()
await store.put_idle("pool", "new-run")
release.set()
with pytest.raises(PoolNotRunningException):
await asyncio.wait_for(acquire, timeout=2)
entries = await store.snapshot_idle_entries("pool")
assert entries[0].sandbox_id == "new-run"
await pool.shutdown(False)
async def test_async_acquire_assertion_error_cleans_popped_idle() -> None:
store = InMemoryAsyncPoolStateStore()
await store.put_idle("pool", "broken-check")
manager = FakeAsyncManager()
class AssertionFactory(FakeAsyncSandbox):
@classmethod
async def connect(
cls, sandbox_id: str, *args: Any, **kwargs: Any
) -> FakeAsyncSandbox:
raise AssertionError("user health check failed")
async def manager_factory(config: ConnectionConfig) -> FakeAsyncManager:
return manager
pool = SandboxPoolAsync(
pool_name="pool",
owner_id="owner-1",
max_idle=0,
state_store=store,
connection_config=ConnectionConfig(),
creation_spec=PoolCreationSpec(image="ubuntu:22.04"),
sandbox_manager_factory=manager_factory, # type: ignore[arg-type]
sandbox_factory=AssertionFactory, # type: ignore[arg-type]
)
await pool.start()
try:
with pytest.raises(AssertionError, match="user health check failed"):
await pool.acquire(policy=AcquirePolicy.FAIL_FAST)
async def killed() -> bool:
return manager.killed == ["broken-check"]
await _eventually(killed)
assert (await store.snapshot_counters("pool")).idle_count == 0
finally:
await pool.shutdown(False)
def _create_pool(
*,
max_idle: int,
store: InMemoryAsyncPoolStateStore | None = None,
manager: FakeAsyncManager | None = None,
max_acquire_retries: int = 3,
) -> SandboxPoolAsync:
return SandboxPoolAsync(
pool_name="pool",
owner_id="owner-1",
max_idle=max_idle,
warmup_concurrency=2,
state_store=store or InMemoryAsyncPoolStateStore(),
connection_config=ConnectionConfig(),
creation_spec=PoolCreationSpec(image="ubuntu:22.04"),
primary_lock_ttl=timedelta(seconds=5),
drain_timeout=timedelta(milliseconds=50),
max_acquire_retries=max_acquire_retries,
sandbox_manager_factory=lambda config: _manager_factory(
manager or FakeAsyncManager()
),
sandbox_factory=FakeAsyncSandbox, # type: ignore[arg-type]
)
async def test_async_graceful_shutdown_superseded_by_start_keeps_new_run_intact(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""Regression: a graceful shutdown releases _lifecycle_lock while draining
in-flight operations. A start() that installs a newer run inside that
window must survive: the superseded shutdown may not cancel the new run's
scheduler/heartbeat tasks, tear down its warmup tasks, or stomp its
RUNNING state to STOPPED.
"""
pool = _create_pool(max_idle=0)
await pool.start()
old_generation = pool._run_generation
drain_started = asyncio.Event()
release_drain = asyncio.Event()
original_drain = pool._await_in_flight_drain
async def paused_drain(timeout: timedelta) -> bool:
drain_started.set()
await release_drain.wait()
return await original_drain(timeout)
monkeypatch.setattr(pool, "_await_in_flight_drain", paused_drain)
shutdown_task = asyncio.create_task(pool.shutdown(graceful=True))
await drain_started.wait()
await pool.start()
release_drain.set()
await shutdown_task
assert pool._run_generation == old_generation + 1
assert pool._lifecycle_state == PoolLifecycleState.RUNNING
assert pool._scheduler_task is not None and not pool._scheduler_task.done()
assert pool._heartbeat_task is not None and not pool._heartbeat_task.done()
await pool.shutdown(graceful=False)
async def test_async_acquire_retry_next_idle_empty_raises_pool_empty() -> None:
pool = _create_pool(max_idle=0)
await pool.start()
try:
with pytest.raises(PoolEmptyException) as exc:
await pool.acquire(policy=AcquirePolicy.RETRY_NEXT_IDLE)
assert "RETRY_NEXT_IDLE" in str(exc.value)
finally:
await pool.shutdown(False)
async def test_async_acquire_retry_next_idle_all_stale_bounds_retries_and_raises() -> (
None
):
store = InMemoryAsyncPoolStateStore()
manager = FakeAsyncManager()
for i in range(5):
await store.put_idle("pool", f"stale-{i}")
pool = _create_pool(max_idle=0, store=store, manager=manager, max_acquire_retries=3)
await pool.start()
try:
with pytest.raises(PoolAcquireFailedException):
await pool.acquire(policy=AcquirePolicy.RETRY_NEXT_IDLE)
counters = await store.snapshot_counters("pool")
assert counters.idle_count == 2
# Kills for stale candidates are fire-and-forget (Codex review: retry loop must not
# block on slow DELETEs). Wait for the background tasks to observe all three.
async def _killed_three() -> bool:
return sorted(manager.killed) == ["stale-0", "stale-1", "stale-2"]
await _eventually(_killed_three)
finally:
await pool.shutdown(False)
async def test_async_acquire_retry_next_idle_drained_mid_loop_raises_pool_acquire_failed() -> (
None
):
store = InMemoryAsyncPoolStateStore()
await store.put_idle("pool", "stale-a")
await store.put_idle("pool", "stale-b")
pool = _create_pool(max_idle=0, store=store, max_acquire_retries=5)
await pool.start()
try:
with pytest.raises(PoolAcquireFailedException) as exc:
await pool.acquire(policy=AcquirePolicy.RETRY_NEXT_IDLE)
assert "drained" in str(exc.value)
counters = await store.snapshot_counters("pool")
assert counters.idle_count == 0
finally:
await pool.shutdown(False)
async def test_async_acquire_retry_next_idle_then_create_falls_through_after_exhaustion() -> (
None
):
FakeAsyncSandbox.reset()
store = InMemoryAsyncPoolStateStore()
for i in range(3):
await store.put_idle("pool", f"stale-{i}")
pool = _create_pool(max_idle=0, store=store, max_acquire_retries=3)
await pool.start()
try:
sandbox = await pool.acquire(policy=AcquirePolicy.RETRY_NEXT_IDLE_THEN_CREATE)
assert sandbox.id.startswith("created-")
counters = await store.snapshot_counters("pool")
assert counters.idle_count == 0
finally:
await pool.shutdown(False)
async def test_async_acquire_retry_next_idle_returns_first_healthy_candidate() -> None:
store = InMemoryAsyncPoolStateStore()
await store.put_idle("pool", "stale-a")
await store.put_idle("pool", "stale-b")
await store.put_idle("pool", "healthy-x")
pool = _create_pool(max_idle=0, store=store, max_acquire_retries=5)
await pool.start()
try:
sandbox = await pool.acquire(policy=AcquirePolicy.RETRY_NEXT_IDLE)
assert sandbox.id == "healthy-x"
counters = await store.snapshot_counters("pool")
assert counters.idle_count == 0
finally:
await pool.shutdown(False)
async def test_async_acquire_retry_next_idle_then_create_empty_falls_through_immediately() -> (
None
):
FakeAsyncSandbox.reset()
pool = _create_pool(max_idle=0)
await pool.start()
try:
sandbox = await pool.acquire(policy=AcquirePolicy.RETRY_NEXT_IDLE_THEN_CREATE)
assert sandbox.id.startswith("created-")
finally:
await pool.shutdown(False)
async def test_async_acquire_retry_next_idle_renew_failure_kills_remote_without_retrying() -> (
None
):
"""Regression: renew failure against a healthy connected sandbox must NOT trigger the
retry loop to drain more idle candidates. But the connected sandbox MUST be killed on
the remote side, since try_take_idle already popped its id out of the pool store —
otherwise it leaks alive-but-untracked until server-side TTL expires.
"""
connected: list[FakeAsyncSandbox] = []
class TrackingAsyncSandbox(FakeAsyncSandbox):
@classmethod
async def connect(
cls, sandbox_id: str, *args: Any, **kwargs: Any
) -> FakeAsyncSandbox:
sb = await super().connect(sandbox_id, *args, **kwargs)
sb.fail_renew = True # per-instance renew failure
connected.append(sb)
return sb
store = InMemoryAsyncPoolStateStore()
manager = FakeAsyncManager()
for i in range(3):
await store.put_idle("pool", f"healthy-{i}")
pool = SandboxPoolAsync(
pool_name="pool",
owner_id="owner-1",
max_idle=0,
warmup_concurrency=2,
state_store=store,
connection_config=ConnectionConfig(),
creation_spec=PoolCreationSpec(image="ubuntu:22.04"),
primary_lock_ttl=timedelta(seconds=5),
drain_timeout=timedelta(milliseconds=50),
max_acquire_retries=5,
sandbox_manager_factory=lambda config: _manager_factory(manager),
sandbox_factory=TrackingAsyncSandbox, # type: ignore[arg-type]
)
await pool.start()
try:
with pytest.raises(RuntimeError, match="renew failed"):
await pool.acquire(
sandbox_timeout=timedelta(minutes=5),
policy=AcquirePolicy.RETRY_NEXT_IDLE,
)
assert len(connected) == 1
counters = await store.snapshot_counters("pool")
assert counters.idle_count == 2
assert manager.killed == []
assert connected[0].killed, (
"renew failure must trigger sandbox.kill() to release remote resources; "
"otherwise the sandbox leaks alive-but-untracked until server-side TTL expiry"
)
assert connected[0].closed
finally:
await pool.shutdown(False)
async def test_async_acquire_retry_next_idle_does_not_block_on_slow_stale_kill() -> (
None
):
"""Async counterpart of the sync slow-kill regression: stale-candidate kill must be
scheduled as a background task so a slow DELETE does not stall the retry loop.
"""
slow_kill_seconds = 2.0
class SlowKillManager(FakeAsyncManager):
async def kill_sandbox(self, sandbox_id: str) -> None:
await asyncio.sleep(slow_kill_seconds)
await super().kill_sandbox(sandbox_id)
store = InMemoryAsyncPoolStateStore()
await store.put_idle("pool", "stale-a")
await store.put_idle("pool", "stale-b")
await store.put_idle("pool", "healthy-x")
manager = SlowKillManager()
pool = _create_pool(max_idle=0, store=store, manager=manager, max_acquire_retries=5)
await pool.start()
try:
start = time.monotonic()
sandbox = await pool.acquire(policy=AcquirePolicy.RETRY_NEXT_IDLE)
elapsed = time.monotonic() - start
assert sandbox.id == "healthy-x"
assert elapsed < slow_kill_seconds, (
f"acquire took {elapsed:.2f}s; expected retry loop to not block on the "
f"slow stale kill (each blocks {slow_kill_seconds:.2f}s)"
)
finally:
await pool.shutdown(False)
async def test_async_acquire_retry_next_idle_then_create_falls_through_on_state_store_outage() -> (
None
):
"""Regression: PoolStateStoreUnavailableException during try_take_idle must degrade to
direct-create under RETRY_NEXT_IDLE_THEN_CREATE (per OSEP-0005).
"""
from opensandbox.exceptions import PoolStateStoreUnavailableException
FakeAsyncSandbox.reset()
class OutageStore(InMemoryAsyncPoolStateStore):
async def try_take_idle(self, pool_name: str) -> str | None:
raise PoolStateStoreUnavailableException(
"TryTakeIdle", RuntimeError("redis unavailable")
)
async def try_take_idle_min_ttl( # type: ignore[override]
self, pool_name: str, min_remaining_ttl: object
) -> object:
raise PoolStateStoreUnavailableException(
"TryTakeIdleWithMinTTL", RuntimeError("redis unavailable")
)
store = OutageStore()
pool = _create_pool(max_idle=0, store=store)
await pool.start()
try:
sandbox = await pool.acquire(policy=AcquirePolicy.RETRY_NEXT_IDLE_THEN_CREATE)
assert sandbox.id.startswith("created-")
finally:
await pool.shutdown(False)
async def test_async_acquire_retry_next_idle_raises_on_state_store_outage() -> None:
from opensandbox.exceptions import PoolStateStoreUnavailableException
class OutageStore(InMemoryAsyncPoolStateStore):
async def try_take_idle(self, pool_name: str) -> str | None:
raise PoolStateStoreUnavailableException(
"TryTakeIdle", RuntimeError("redis unavailable")
)
async def try_take_idle_min_ttl( # type: ignore[override]
self, pool_name: str, min_remaining_ttl: object
) -> object:
raise PoolStateStoreUnavailableException(
"TryTakeIdleWithMinTTL", RuntimeError("redis unavailable")
)
store = OutageStore()
pool = _create_pool(max_idle=0, store=store)
await pool.start()
try:
with pytest.raises(PoolStateStoreUnavailableException):
await pool.acquire(policy=AcquirePolicy.RETRY_NEXT_IDLE)
finally:
await pool.shutdown(False)
async def test_async_acquire_then_create_falls_through_when_full_state_store_outage_also_fails_namespace_check() -> (
None
):
"""Regression for Codex round-5 P2: previously, when the full state store was down
(Redis outage affecting *all* methods, not just try_take_idle), acquire aborted at
the pre-loop `_ensure_pool_namespace_active` call before the fallthrough branch
could run. RETRY_NEXT_IDLE_THEN_CREATE is documented to degrade to direct-create
during store outages (OSEP-0005); this test proves the namespace check no longer
breaks that guarantee.
"""
from opensandbox.exceptions import PoolStateStoreUnavailableException
FakeAsyncSandbox.reset()
class OutageStore(InMemoryAsyncPoolStateStore):
def __init__(self) -> None:
super().__init__()
# Only start raising after pool.start() completes so setup still works;
# this mirrors a real Redis instance that crashes after the pool warms.
self._outage = False
async def try_take_idle(self, pool_name: str) -> str | None:
if self._outage:
raise PoolStateStoreUnavailableException(
"TryTakeIdle", RuntimeError("redis unavailable")
)
return await super().try_take_idle(pool_name)
async def try_take_idle_min_ttl( # type: ignore[override]
self, pool_name: str, min_remaining_ttl: object
) -> object:
if self._outage:
raise PoolStateStoreUnavailableException(
"TryTakeIdleWithMinTTL", RuntimeError("redis unavailable")
)
return await super().try_take_idle_min_ttl(pool_name, min_remaining_ttl) # type: ignore[arg-type]
async def get_destroy_state(self, pool_name: str): # type: ignore[override]
if self._outage:
raise PoolStateStoreUnavailableException(
"GetDestroyState", RuntimeError("redis unavailable")
)
return await super().get_destroy_state(pool_name)
store = OutageStore()
pool = _create_pool(max_idle=0, store=store)
await pool.start()
store._outage = True
try:
sandbox = await pool.acquire(policy=AcquirePolicy.RETRY_NEXT_IDLE_THEN_CREATE)
assert sandbox.id.startswith("created-")
finally:
store._outage = False
await pool.shutdown(False)
async def test_async_acquire_retry_next_idle_raises_when_full_state_store_outage_also_fails_namespace_check() -> (
None
):
"""Non-fallthrough counterpart: full state-store outage under RETRY_NEXT_IDLE must
still surface PoolStateStoreUnavailableException (fail-closed)."""
from opensandbox.exceptions import PoolStateStoreUnavailableException
FakeAsyncSandbox.reset()
class OutageStore(InMemoryAsyncPoolStateStore):
def __init__(self) -> None:
super().__init__()
self._outage = False
async def try_take_idle(self, pool_name: str) -> str | None:
if self._outage:
raise PoolStateStoreUnavailableException(
"TryTakeIdle", RuntimeError("redis unavailable")
)
return await super().try_take_idle(pool_name)
async def try_take_idle_min_ttl( # type: ignore[override]
self, pool_name: str, min_remaining_ttl: object
) -> object:
if self._outage:
raise PoolStateStoreUnavailableException(
"TryTakeIdleWithMinTTL", RuntimeError("redis unavailable")
)
return await super().try_take_idle_min_ttl(pool_name, min_remaining_ttl) # type: ignore[arg-type]
async def get_destroy_state(self, pool_name: str): # type: ignore[override]
if self._outage:
raise PoolStateStoreUnavailableException(
"GetDestroyState", RuntimeError("redis unavailable")
)
return await super().get_destroy_state(pool_name)
store = OutageStore()
pool = _create_pool(max_idle=0, store=store)
await pool.start()
store._outage = True
try:
with pytest.raises(PoolStateStoreUnavailableException):
await pool.acquire(policy=AcquirePolicy.RETRY_NEXT_IDLE)
finally:
store._outage = False
await pool.shutdown(False)
async def test_async_pool_config_rejects_max_acquire_retries_below_one() -> None:
from opensandbox.pool_types import AsyncPoolConfig
with pytest.raises(ValueError, match="max_acquire_retries must be >= 1"):
AsyncPoolConfig(
pool_name="pool",
owner_id="owner-1",
max_idle=1,
state_store=InMemoryAsyncPoolStateStore(),
connection_config=ConnectionConfig(),
creation_spec=PoolCreationSpec(image="ubuntu:22.04"),
max_acquire_retries=0,
)
async def _manager_factory(manager: FakeAsyncManager) -> FakeAsyncManager:
return manager
async def _noop_discard(_sandbox_id: str) -> None:
return None
async def _idle_count_equals(pool: SandboxPoolAsync, expected: int) -> bool:
return (await pool.snapshot()).idle_count == expected
async def _eventually(condition: Any, timeout: float = 2.0) -> None:
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
if await condition():
return
await asyncio.sleep(0.01)
raise AssertionError("condition did not become true")
class FakeAsyncManager:
def __init__(self) -> None:
self.killed: list[str] = []
self.closed = False
async def kill_sandbox(self, sandbox_id: str) -> None:
self.killed.append(sandbox_id)
async def close(self) -> None:
self.closed = True
class FakeAsyncSandbox:
created_count = 0
fail_renew = False
last_created: FakeAsyncSandbox | None = None
def __init__(self, sandbox_id: str) -> None:
self.id = sandbox_id
self.renewed: list[timedelta] = []
self.closed = False
self.killed = False
@classmethod
def reset(cls) -> None:
cls.created_count = 0
cls.fail_renew = False
cls.last_created = None
@classmethod
async def create(cls, *args: Any, **kwargs: Any) -> FakeAsyncSandbox:
cls.created_count += 1
sandbox = cls(f"created-{cls.created_count}")
cls.last_created = sandbox
return sandbox
@classmethod
async def connect(
cls, sandbox_id: str, *args: Any, **kwargs: Any
) -> FakeAsyncSandbox:
if sandbox_id.startswith("stale"):
raise RuntimeError("stale sandbox")
return cls(sandbox_id)
async def renew(self, timeout: timedelta) -> None:
if self.fail_renew:
raise RuntimeError("renew failed")
self.renewed.append(timeout)
async def is_healthy(self) -> bool:
return True
async def kill(self) -> None:
self.killed = True
async def close(self) -> None:
self.closed = True
class SharedAsyncMaxIdleStore(InMemoryAsyncPoolStateStore):
def __init__(self, initial_max_idle: int | None = None) -> None:
super().__init__()
self.max_idle_by_pool: dict[str, int] = {}
self.set_max_idle_calls: list[tuple[str, int]] = []
if initial_max_idle is not None:
self.max_idle_by_pool["pool"] = initial_max_idle
async def get_max_idle(self, pool_name: str) -> int | None:
return self.max_idle_by_pool.get(pool_name)
async def set_max_idle(self, pool_name: str, max_idle: int) -> None:
self.set_max_idle_calls.append((pool_name, max_idle))
self.max_idle_by_pool[pool_name] = max_idle
class _AsyncTransport(httpx.AsyncBaseTransport):
async def handle_async_request(self, request: httpx.Request) -> httpx.Response:
return httpx.Response(200, request=request)