Enterprise agent platform for building, deploying, and operating AI teammates.
"""Tests for task-scoped sandbox lifecycle handling in AgentServiceManager.
The active-task ref-count and per-key lifecycle locks live in
``SandboxManager`` (attach/release); these tests cover the web layer's
integration with that API: exactly-once worker cleanup and AgentService
eviction when the last task releases a sandbox.
"""
import asyncio
from unittest.mock import AsyncMock
import pytest
from xagent.core.execution_scope import ExecutionScope
from xagent.sandbox.base import (
SandboxRecoveryRequiredError,
SandboxRuntimeConflictError,
)
from xagent.web.sandbox_manager import SandboxCapacityError, SandboxManager
from xagent.web.services.agent_service_manager import AgentServiceManager
class _FakeAgentService:
def __init__(
self,
*,
result: dict | None = None,
started: asyncio.Event | None = None,
release: asyncio.Event | None = None,
) -> None:
self.result = result or {"success": True}
self.started = started
self.release = release
async def execute_task(self, **_kwargs):
if self.started is not None:
self.started.set()
if self.release is not None:
await self.release.wait()
return self.result
def _make_sandbox_manager() -> SandboxManager:
service = AsyncMock()
service.list_sandboxes = AsyncMock(return_value=[])
service.delete = AsyncMock()
return SandboxManager(service)
async def _seed_attached_provider(
sandbox_mgr: SandboxManager, owner_id: int, provider: object
) -> None:
"""Cache a lease provider and attach one active task for an owner."""
sandbox_mgr._lease_providers[f"user::{owner_id}"] = provider
assert await sandbox_mgr.attach("user", str(owner_id))
@pytest.fixture
def sandbox_mgr(monkeypatch) -> SandboxManager:
manager = _make_sandbox_manager()
monkeypatch.setattr(
"xagent.web.sandbox_manager.get_sandbox_manager",
lambda: manager,
)
return manager
@pytest.mark.asyncio
async def test_worker_cleanup_does_not_block_other_users(sandbox_mgr) -> None:
"""Slow worker cleanup for one user must not serialize other users."""
manager = AgentServiceManager()
cleanup_started = asyncio.Event()
cleanup_release = asyncio.Event()
async def slow_list_sandboxes() -> list:
cleanup_started.set()
await cleanup_release.wait()
return []
sandbox_mgr._service.list_sandboxes.side_effect = slow_list_sandboxes
await _seed_attached_provider(sandbox_mgr, 7, AsyncMock())
sandbox_mgr._lease_providers["user::8"] = AsyncMock()
manager._agent_owner_ids[2] = 8
manager._agent_sandbox_keys[2] = "user:8"
release_task = asyncio.create_task(manager._release_sandbox_task("user:7"))
await cleanup_started.wait()
try:
sandbox_key = await asyncio.wait_for(
manager._acquire_sandbox_task("2"),
timeout=30,
)
finally:
cleanup_release.set()
await release_task
assert sandbox_key == "user:8"
assert sandbox_mgr.ref_count("user", "8") == 1
@pytest.mark.asyncio
async def test_worker_cleanup_removes_sandbox_lifecycle_lock_entry(
sandbox_mgr,
) -> None:
"""Lifecycle lock entries should not leak after a provider is removed."""
manager = AgentServiceManager()
await _seed_attached_provider(sandbox_mgr, 7, AsyncMock())
await manager._release_sandbox_task("user:7")
assert sandbox_mgr._lifecycle_locks == {}
assert "user::7" not in sandbox_mgr._lease_providers
@pytest.mark.asyncio
async def test_worker_cleanup_blocks_same_user_provider_recreate(sandbox_mgr) -> None:
"""The same user must not recreate same-named workers during cleanup."""
manager = AgentServiceManager()
new_provider = AsyncMock()
cleanup_started = asyncio.Event()
cleanup_release = asyncio.Event()
create_called = asyncio.Event()
async def slow_list_sandboxes() -> list:
cleanup_started.set()
await cleanup_release.wait()
return []
async def create_provider(*_args, **_kwargs):
create_called.set()
return new_provider
sandbox_mgr._service.list_sandboxes.side_effect = slow_list_sandboxes
sandbox_mgr._create_lease_provider_locked = AsyncMock(side_effect=create_provider)
await _seed_attached_provider(sandbox_mgr, 7, AsyncMock())
release_task = asyncio.create_task(manager._release_sandbox_task("user:7"))
await cleanup_started.wait()
create_task = asyncio.create_task(
manager._get_or_create_task_sandbox(
task_id=3,
workspace_owner_id=7,
mount_intent=None,
)
)
try:
with pytest.raises(asyncio.TimeoutError):
await asyncio.wait_for(create_called.wait(), timeout=0.05)
cleanup_release.set()
sandbox = await asyncio.wait_for(create_task, timeout=30)
await release_task
finally:
cleanup_release.set()
await asyncio.gather(release_task, create_task, return_exceptions=True)
assert sandbox is new_provider
sandbox_mgr._create_lease_provider_locked.assert_awaited_once()
@pytest.mark.asyncio
async def test_execute_task_releases_sandbox_workers_after_task(sandbox_mgr) -> None:
"""Terminal task execution should release cached worker sandboxes."""
manager = AgentServiceManager()
agent_service = _FakeAgentService()
sandbox_mgr._lease_providers["user::7"] = AsyncMock()
sandbox_mgr.delete_worker_sandboxes = AsyncMock()
manager._agents[1] = agent_service
manager._agent_owner_ids[1] = 7
manager._agent_sandbox_keys[1] = "user:7"
result = await manager.execute_task(
agent_service=agent_service,
task="run",
task_id="1",
manage_task_lease=False,
)
assert result == {"success": True}
assert "user::7" not in sandbox_mgr._lease_providers
sandbox_mgr.delete_worker_sandboxes.assert_awaited_once_with("user", "7")
@pytest.mark.asyncio
async def test_execute_task_evicts_agents_for_released_sandbox_provider(
sandbox_mgr,
) -> None:
"""Cached agents must not retain a provider after its sandbox is released."""
manager = AgentServiceManager()
agent_service = _FakeAgentService()
other_agent_service = _FakeAgentService()
sandbox_mgr._lease_providers["user::7"] = AsyncMock()
manager._agents[1] = agent_service
manager._agents[2] = other_agent_service
manager._agent_owner_ids[1] = 7
manager._agent_owner_ids[2] = 7
manager._agent_sandbox_keys[1] = "user:7"
manager._agent_sandbox_keys[2] = "user:7"
await manager.execute_task(
agent_service=agent_service,
task="run",
task_id="1",
manage_task_lease=False,
)
assert "user::7" not in sandbox_mgr._lease_providers
assert 1 not in manager._agents
assert 2 not in manager._agents
assert 1 not in manager._agent_owner_ids
assert 2 not in manager._agent_owner_ids
assert 1 not in manager._agent_sandbox_keys
assert 2 not in manager._agent_sandbox_keys
@pytest.mark.asyncio
async def test_execute_task_keeps_workers_until_last_same_user_task_finishes(
sandbox_mgr,
) -> None:
"""One task finishing must not delete workers still shared by another task."""
manager = AgentServiceManager()
first_agent_service = _FakeAgentService()
second_agent_service = _FakeAgentService()
sandbox_mgr._lease_providers["user::7"] = AsyncMock()
sandbox_mgr.delete_worker_sandboxes = AsyncMock()
manager._agents[1] = first_agent_service
manager._agents[2] = second_agent_service
manager._agent_owner_ids[1] = 7
manager._agent_owner_ids[2] = 7
manager._agent_sandbox_keys[1] = "user:7"
manager._agent_sandbox_keys[2] = "user:7"
first_started = asyncio.Event()
first_release = asyncio.Event()
second_started = asyncio.Event()
second_release = asyncio.Event()
first_agent_service.started = first_started
first_agent_service.release = first_release
second_agent_service.started = second_started
second_agent_service.release = second_release
first_task = asyncio.create_task(
manager.execute_task(
agent_service=first_agent_service,
task="first",
task_id="1",
manage_task_lease=False,
)
)
await first_started.wait()
second_task = asyncio.create_task(
manager.execute_task(
agent_service=second_agent_service,
task="second",
task_id="2",
manage_task_lease=False,
)
)
await second_started.wait()
first_release.set()
await first_task
assert "user::7" in sandbox_mgr._lease_providers
assert 1 in manager._agents
assert 2 in manager._agents
sandbox_mgr.delete_worker_sandboxes.assert_not_awaited()
second_release.set()
await second_task
assert "user::7" not in sandbox_mgr._lease_providers
assert 1 not in manager._agents
assert 2 not in manager._agents
sandbox_mgr.delete_worker_sandboxes.assert_awaited_once()
@pytest.mark.asyncio
async def test_acquire_with_reclaimed_provider_raises_and_evicts_agent(
sandbox_mgr,
) -> None:
"""An agent built with a provider that was since reclaimed must fail
clearly (and drop its stale cache) instead of running against deleted
containers."""
manager = AgentServiceManager()
manager._agents[1] = AsyncMock()
manager._agent_owner_ids[1] = 7
manager._agent_sandbox_keys[1] = "user:7"
with pytest.raises(RuntimeError, match="reclaimed before"):
await manager._acquire_sandbox_task("1")
assert 1 not in manager._agents
assert 1 not in manager._agent_owner_ids
assert 1 not in manager._agent_sandbox_keys
assert sandbox_mgr.ref_count("user", "7") == 0
@pytest.mark.asyncio
async def test_acquire_without_recorded_key_returns_none(
sandbox_mgr,
) -> None:
"""A task with no recorded sandbox key (e.g. local execution) returns
None, never raises. The key is never re-derived from the owner id —
the owner-only reconstruction fallback was removed with #757 scoping,
since it would silently miss a scope-suffixed sandbox."""
manager = AgentServiceManager()
manager._agent_owner_ids[1] = 7
assert await manager._acquire_sandbox_task("1") is None
assert sandbox_mgr.ref_count("user", "7") == 0
@pytest.mark.asyncio
async def test_capacity_error_rejects_task(sandbox_mgr, monkeypatch) -> None:
"""Capacity exhaustion must reject the task, not silently run locally."""
monkeypatch.delenv("XAGENT_SANDBOX_ALLOW_LOCAL_FALLBACK_ON_CAPACITY", raising=False)
manager = AgentServiceManager()
sandbox_mgr.get_or_create_lease_provider = AsyncMock(
side_effect=SandboxCapacityError(cap=2, in_use=2)
)
with pytest.raises(SandboxCapacityError):
await manager._get_or_create_task_sandbox(
task_id=1,
workspace_owner_id=7,
mount_intent=None,
)
assert 1 not in manager._agent_sandbox_keys
@pytest.mark.asyncio
async def test_capacity_error_ignores_removed_fallback_flag(
sandbox_mgr, monkeypatch
) -> None:
"""The removed legacy flag cannot re-enable host-local execution."""
monkeypatch.setenv("XAGENT_SANDBOX_ALLOW_LOCAL_FALLBACK_ON_CAPACITY", "true")
manager = AgentServiceManager()
sandbox_mgr.get_or_create_lease_provider = AsyncMock(
side_effect=SandboxCapacityError(cap=2, in_use=2)
)
with pytest.raises(SandboxCapacityError):
await manager._get_or_create_task_sandbox(
task_id=1,
workspace_owner_id=7,
mount_intent=None,
)
assert 1 not in manager._agent_sandbox_keys
@pytest.mark.asyncio
async def test_sandbox_unavailability_fails_closed(sandbox_mgr, monkeypatch) -> None:
"""Runtime failures must never execute the task on the host."""
monkeypatch.delenv("XAGENT_SANDBOX_ALLOW_LOCAL_FALLBACK_ON_CAPACITY", raising=False)
manager = AgentServiceManager()
sandbox_mgr.get_or_create_lease_provider = AsyncMock(
side_effect=RuntimeError("docker daemon unreachable")
)
with pytest.raises(RuntimeError, match="docker daemon unreachable"):
await manager._get_or_create_task_sandbox(
task_id=1,
workspace_owner_id=7,
mount_intent=None,
)
assert 1 not in manager._agent_sandbox_keys
_SCOPE = ExecutionScope(
sandbox_key_suffix="client-3",
workspace_segments=("clients", "3", "end_users", "7"),
)
# A scope that sets no ``sandbox_key_suffix`` resolves to the same
# ``user:{owner}`` lifecycle key as unscoped execution. Both forms fail closed.
_SUFFIXLESS_SCOPE = ExecutionScope(
workspace_segments=("clients", "3", "end_users", "7"),
)
@pytest.mark.asyncio
@pytest.mark.parametrize(
"fallback_flag, side_effect, expected_exc",
[
pytest.param(
"true",
SandboxCapacityError(cap=2, in_use=2),
SandboxCapacityError,
id="capacity-with-flag-on",
),
pytest.param(
None,
RuntimeError("docker daemon unreachable"),
RuntimeError,
id="non-capacity-outage",
),
],
)
async def test_suffix_scoped_execution_fails_closed(
sandbox_mgr, monkeypatch, fallback_flag, side_effect, expected_exc
) -> None:
"""A suffix-scoped task never takes the local fallback.
Local fallback would run an isolated/untrusted scoped workload outside its
container, so both capacity exhaustion (even with the opt-in flag on) and a
non-capacity sandbox outage must fail closed rather than downgrade to
local execution.
"""
if fallback_flag is None:
monkeypatch.delenv(
"XAGENT_SANDBOX_ALLOW_LOCAL_FALLBACK_ON_CAPACITY", raising=False
)
else:
monkeypatch.setenv(
"XAGENT_SANDBOX_ALLOW_LOCAL_FALLBACK_ON_CAPACITY", fallback_flag
)
manager = AgentServiceManager()
sandbox_mgr.get_or_create_lease_provider = AsyncMock(side_effect=side_effect)
with pytest.raises(expected_exc):
await manager._get_or_create_task_sandbox(
task_id=1,
workspace_owner_id=7,
mount_intent=None,
scope=_SCOPE,
)
assert 1 not in manager._agent_sandbox_keys
@pytest.mark.asyncio
@pytest.mark.parametrize(
"fallback_flag, side_effect",
[
pytest.param(
"true", SandboxCapacityError(cap=2, in_use=2), id="capacity-with-flag-on"
),
pytest.param(
None, RuntimeError("docker daemon unreachable"), id="non-capacity-outage"
),
],
)
async def test_suffixless_scope_fails_closed(
sandbox_mgr, monkeypatch, fallback_flag, side_effect
) -> None:
"""A scope without a suffix still cannot downgrade to host execution."""
if fallback_flag is None:
monkeypatch.delenv(
"XAGENT_SANDBOX_ALLOW_LOCAL_FALLBACK_ON_CAPACITY", raising=False
)
else:
monkeypatch.setenv(
"XAGENT_SANDBOX_ALLOW_LOCAL_FALLBACK_ON_CAPACITY", fallback_flag
)
manager = AgentServiceManager()
sandbox_mgr.get_or_create_lease_provider = AsyncMock(side_effect=side_effect)
with pytest.raises(type(side_effect)):
await manager._get_or_create_task_sandbox(
task_id=1,
workspace_owner_id=7,
mount_intent=None,
scope=_SUFFIXLESS_SCOPE,
)
assert 1 not in manager._agent_sandbox_keys
@pytest.mark.asyncio
@pytest.mark.parametrize(
"scope",
[
pytest.param(None, id="unscoped"),
pytest.param(_SUFFIXLESS_SCOPE, id="suffixless-scope"),
pytest.param(_SCOPE, id="scoped"),
],
)
@pytest.mark.parametrize(
"contract_error",
[
pytest.param(
SandboxRuntimeConflictError("runtime configuration drifted"),
id="runtime-conflict",
),
pytest.param(
SandboxRecoveryRequiredError("needs recovery before use"),
id="recovery-required",
),
],
)
async def test_contract_error_always_fails_closed(
sandbox_mgr, monkeypatch, scope, contract_error
) -> None:
"""A sandbox lifecycle contract violation never falls back to local
execution, for unscoped, suffixless-scope, or suffix-scoped tasks alike.
A ``SandboxContractError`` means the manager explicitly refused to honor
the desired mount/runtime spec. Silently running the task on the host
would defeat that refusal rather than surface it. The removed legacy
capacity fallback flag must not change this.
"""
monkeypatch.setenv("XAGENT_SANDBOX_ALLOW_LOCAL_FALLBACK_ON_CAPACITY", "true")
manager = AgentServiceManager()
manager._agent_sandbox_keys[1] = "stale-key"
manager._agent_sandbox_providers[1] = object()
sandbox_mgr.get_or_create_lease_provider = AsyncMock(side_effect=contract_error)
with pytest.raises(type(contract_error)):
await manager._get_or_create_task_sandbox(
task_id=1,
workspace_owner_id=7,
mount_intent=None,
scope=scope,
)
assert 1 not in manager._agent_sandbox_keys
assert 1 not in manager._agent_sandbox_providers
@pytest.mark.asyncio
async def test_capacity_semantics_unchanged_by_contract_classification(
sandbox_mgr, monkeypatch
) -> None:
"""SandboxCapacityError is not a SandboxContractError: adding the
contract-error branch must not allow the removed legacy flag to bypass
fail-closed capacity handling."""
monkeypatch.setenv("XAGENT_SANDBOX_ALLOW_LOCAL_FALLBACK_ON_CAPACITY", "true")
manager = AgentServiceManager()
sandbox_mgr.get_or_create_lease_provider = AsyncMock(
side_effect=SandboxCapacityError(cap=2, in_use=2)
)
with pytest.raises(SandboxCapacityError):
await manager._get_or_create_task_sandbox(
task_id=1,
workspace_owner_id=7,
mount_intent=None,
)
assert 1 not in manager._agent_sandbox_keys
assert 1 not in manager._agent_sandbox_providers
@pytest.mark.asyncio
async def test_get_or_create_task_sandbox_records_provider(sandbox_mgr) -> None:
"""A successful sandbox build must record its provider object (keyed
the same as the sandbox key) so ``_acquire_sandbox_task`` can later
attach by identity instead of key existence alone."""
manager = AgentServiceManager()
provider = AsyncMock()
sandbox_mgr.get_or_create_lease_provider = AsyncMock(return_value=provider)
sandbox = await manager._get_or_create_task_sandbox(
task_id=1,
workspace_owner_id=7,
mount_intent=None,
)
assert sandbox is provider
assert manager._agent_sandbox_providers[1] is provider
@pytest.mark.asyncio
async def test_acquire_falls_back_to_attach_when_no_provider_recorded(
sandbox_mgr,
) -> None:
"""A cache entry with a recorded key but no recorded provider (e.g. a
pre-ABA cache state) keeps the existence-only ``attach()`` behavior."""
manager = AgentServiceManager()
manager._agent_owner_ids[1] = 7
manager._agent_sandbox_keys[1] = "user:7"
sandbox_mgr._lease_providers["user::7"] = AsyncMock()
sandbox_key = await manager._acquire_sandbox_task("1")
assert sandbox_key == "user:7"
assert sandbox_mgr.ref_count("user", "7") == 1
@pytest.mark.asyncio
async def test_acquire_detects_provider_replaced_since_build_time(
sandbox_mgr,
) -> None:
"""ABA: the key still resolves to a live provider, but it is not the
same object the cached agent's tools were built against (a rebuild
replaced it) -- ``attach_provider`` must catch this by identity and the
caller must evict and rebuild exactly like the existence-only case."""
manager = AgentServiceManager()
old_provider = object()
manager._agents[1] = AsyncMock()
manager._agent_owner_ids[1] = 7
manager._agent_sandbox_keys[1] = "user:7"
manager._agent_sandbox_providers[1] = old_provider
# A rebuild since then installed a genuinely different provider object
# for the same lifecycle key.
sandbox_mgr._lease_providers["user::7"] = object()
with pytest.raises(RuntimeError, match="reclaimed before"):
await manager._acquire_sandbox_task("1")
assert 1 not in manager._agents
assert 1 not in manager._agent_owner_ids
assert 1 not in manager._agent_sandbox_keys
assert 1 not in manager._agent_sandbox_providers
assert sandbox_mgr.ref_count("user", "7") == 0