Skip to content

Enterprise agent platform for building, deploying, and operating AI teammates.

18.1 KB
Raw Blame
"""
Unit tests for DockerSandboxService's per-name lifecycle lock (_named_lock)
and control-object construction (_get_live_control). Pure asyncio tests
against a minimal fake Docker client; no Docker daemon required.
"""

from __future__ import annotations

import ast
import asyncio
import threading
import time
from pathlib import Path

import pytest

import xagent.sandbox.docker_sandbox as docker_sandbox_module
from xagent.sandbox.docker_sandbox import (
    LABEL_SANDBOX_NAME,
    DockerSandboxService,
    MemDockerStore,
    _SandboxControl,
)


class _FakeContainerCollection:
    """Minimal Docker container collection stub: always reports no containers."""

    def list(self, *args, **kwargs):
        return []


class _FakeDockerClient:
    """Minimal Docker client stub sufficient for service construction and delete()."""

    def __init__(self) -> None:
        self.containers = _FakeContainerCollection()

    def ping(self):
        return True


def _make_service() -> DockerSandboxService:
    return DockerSandboxService(
        MemDockerStore(),
        namespace="test",
        network="xagent-test-sandbox-egress",
        client=_FakeDockerClient(),
    )


class _StoppableContainer:
    """Container stub whose ``stop()`` can be held open from the test.

    ``stop()`` runs on a worker thread (both callers reach it through
    ``asyncio.to_thread``), so the occupancy counter is guarded by a real
    ``threading.Lock`` and the gate is a ``threading.Event``.
    """

    def __init__(self, name: str) -> None:
        self.name = name
        self.labels = {LABEL_SANDBOX_NAME: name}
        self.attrs: dict = {
            "Config": {"Image": "busybox:latest", "WorkingDir": "/home"},
            "HostConfig": {},
            "State": {"Status": "running"},
            "NetworkSettings": {"Networks": {"xagent-test-sandbox-egress": {}}},
            "Created": "2026-01-01T00:00:00Z",
        }
        self.stop_calls = 0
        self.start_calls = 0
        self.max_concurrent_stops = 0
        self.stop_entered = threading.Event()
        self.release_stop = threading.Event()
        self._occupancy = 0
        self._guard = threading.Lock()

    def reload(self):
        return None

    def start(self):
        self.start_calls += 1

    def stop(self):
        with self._guard:
            self.stop_calls += 1
            self._occupancy += 1
            self.max_concurrent_stops = max(self.max_concurrent_stops, self._occupancy)
        self.stop_entered.set()
        if not self.release_stop.wait(timeout=10):
            raise AssertionError("stop() gate was never released")
        with self._guard:
            self._occupancy -= 1


class _FakeClientWithContainer:
    """Docker client stub that always resolves one managed container."""

    def __init__(self, container: _StoppableContainer) -> None:
        self.containers = _SingleContainerCollection(container)

    def ping(self):
        return True


class _SingleContainerCollection:
    def __init__(self, container: _StoppableContainer) -> None:
        self._container = container

    def list(self, *args, **kwargs):
        return [self._container]


async def _await_flag(flag: threading.Event, timeout: float = 5.0) -> None:
    """Await a thread-set flag from the event loop without blocking it."""
    deadline = time.monotonic() + timeout
    while not flag.is_set():
        if time.monotonic() > deadline:
            raise AssertionError("timed out waiting for the worker thread")
        await asyncio.sleep(0.01)


class TestHandleStopSharesTheServiceLifecycleLock:
    """Pin that ``DockerSandbox.stop()`` and the service's own lifecycle
    methods are mutually exclusive for one container name.

    ``stop()`` used to take only the sandbox's own ``exclusive_access``
    barrier, which drains in-flight ``operation()`` work and tracks no
    exclusive holder -- and neither ``container.stop()`` nor
    ``container.start()`` registers as an ``operation()``, so a stop could
    interleave with ``stop_existing``/``start_existing``/``delete``/
    ``create_snapshot`` for the same name.
    """

    @pytest.mark.asyncio
    async def test_handles_are_built_with_the_services_lock_registry(self):
        container = _StoppableContainer("wired")
        service = DockerSandboxService(
            MemDockerStore(),
            namespace="test",
            network="xagent-test-sandbox-egress",
            client=_FakeClientWithContainer(container),
        )

        from_start_existing = await service.start_existing("wired")
        from_get_or_create = await service.get_or_create("wired")

        # Same registry object, not a private copy: sharing the registry is
        # the whole mechanism behind the mutual exclusion below.
        assert from_start_existing._locks is service._locks
        assert from_get_or_create._locks is service._locks

    @pytest.mark.asyncio
    async def test_stop_and_stop_existing_cannot_overlap_for_one_name(self):
        name = "contended"
        container = _StoppableContainer(name)
        service = DockerSandboxService(
            MemDockerStore(),
            namespace="test",
            network="xagent-test-sandbox-egress",
            client=_FakeClientWithContainer(container),
        )
        handle = await service.start_existing(name)

        # The handle's stop() enters container.stop() and parks there.
        stop_task = asyncio.create_task(handle.stop())
        await _await_flag(container.stop_entered)
        assert service._locks[name].lock.locked()
        assert service._locks[name].waiters == 1

        # A concurrent lifecycle transition for the same name must queue on
        # the *same* lock entry rather than proceeding in parallel.
        stop_existing_task = asyncio.create_task(service.stop_existing(name))
        for _ in range(5):
            await asyncio.sleep(0)
        assert service._locks[name].waiters == 2, (
            "stop_existing() must queue on the same keyed lock entry stop() holds"
        )
        assert not stop_existing_task.done()
        assert container.stop_calls == 1, (
            "stop_existing() must not reach container.stop() while stop() holds the lock"
        )

        container.release_stop.set()
        await asyncio.wait_for(stop_task, timeout=5)
        await asyncio.wait_for(stop_existing_task, timeout=5)

        assert container.stop_calls == 2
        assert container.max_concurrent_stops == 1, (
            "the two stop paths overlapped inside the critical section"
        )
        assert name not in service._locks

    @pytest.mark.asyncio
    async def test_lock_holding_paths_do_not_self_deadlock(self):
        """Regression pin for the fix's own failure mode.

        The keyed lock is not reentrant, so a lock-holding path that reached
        ``DockerSandbox.stop()`` would deadlock on itself. ``stop_existing()``
        stops its container through the raw ``Container.stop`` API for exactly
        that reason; this asserts the real call completes rather than hanging.
        """
        name = "no-deadlock"
        container = _StoppableContainer(name)
        container.release_stop.set()
        service = DockerSandboxService(
            MemDockerStore(),
            namespace="test",
            network="xagent-test-sandbox-egress",
            client=_FakeClientWithContainer(container),
        )

        handle = await service.start_existing(name)
        await asyncio.wait_for(service.stop_existing(name), timeout=5)
        await asyncio.wait_for(handle.stop(), timeout=5)

        assert container.stop_calls == 2
        assert name not in service._locks


class TestNamedLockIdentityAndMutualExclusion:
    """Pin the split-brain fix: a waiter queued behind a holder must land on
    the same lock entry the holder used, not a freshly-constructed one."""

    @pytest.mark.asyncio
    async def test_named_lock_stays_mutually_exclusive_across_release_and_requeue(
        self,
    ):
        service = _make_service()
        concurrent_holders = 0
        max_concurrent = 0

        entered_b = asyncio.Event()
        release_b = asyncio.Event()
        entered_a = asyncio.Event()
        release_a = asyncio.Event()
        entered_c = asyncio.Event()
        release_c = asyncio.Event()

        async def holder(entered: asyncio.Event, release: asyncio.Event) -> None:
            nonlocal concurrent_holders, max_concurrent
            async with service._named_lock("shared-name"):
                concurrent_holders += 1
                max_concurrent = max(max_concurrent, concurrent_holders)
                entered.set()
                await release.wait()
                concurrent_holders -= 1

        # B takes the lock first.
        task_b = asyncio.create_task(holder(entered_b, release_b))
        await entered_b.wait()

        # A queues behind B while B still holds it.
        task_a = asyncio.create_task(holder(entered_a, release_a))
        await asyncio.sleep(0)
        entry_while_b_holds = service._locks["shared-name"]
        assert entry_while_b_holds.waiters == 2

        # B finishes; the entry must survive (A is still waiting on it) so
        # that A ends up acquiring the SAME entry rather than racing a
        # newly-constructed one against a delete()/other holder.
        release_b.set()
        await task_b
        await entered_a.wait()
        assert service._locks["shared-name"] is entry_while_b_holds

        # A now holds it; C attempts to acquire concurrently and must queue
        # behind A rather than proceeding on an independent lock instance.
        task_c = asyncio.create_task(holder(entered_c, release_c))
        await asyncio.sleep(0)
        assert concurrent_holders == 1
        assert not entered_c.is_set()

        release_a.set()
        await task_a
        await entered_c.wait()

        release_c.set()
        await task_c

        assert max_concurrent == 1
        assert "shared-name" not in service._locks


class TestNamedLockWaiterRecycling:
    """Pin the entry-recycling contract: only evict when unused."""

    @pytest.mark.asyncio
    async def test_entry_recycled_when_no_waiters_remain(self):
        service = _make_service()
        async with service._named_lock("solo"):
            assert "solo" in service._locks
        assert "solo" not in service._locks

    @pytest.mark.asyncio
    async def test_entry_retained_while_a_waiter_is_pending(self):
        service = _make_service()
        entered_holder = asyncio.Event()
        release_holder = asyncio.Event()
        entered_waiter = asyncio.Event()
        release_waiter = asyncio.Event()

        async def holder(entered: asyncio.Event, release: asyncio.Event) -> None:
            async with service._named_lock("busy"):
                entered.set()
                await release.wait()

        task_holder = asyncio.create_task(holder(entered_holder, release_holder))
        await entered_holder.wait()

        task_waiter = asyncio.create_task(holder(entered_waiter, release_waiter))
        await asyncio.sleep(0)
        assert service._locks["busy"].waiters == 2

        # Releasing the holder must not drop the entry: the waiter is still
        # queued on it.
        release_holder.set()
        await task_holder
        assert "busy" in service._locks

        release_waiter.set()
        await task_waiter
        assert "busy" not in service._locks


class TestNamedLockCancellationSafety:
    """Pin that a cancelled waiter rolls back its waiter count and leaks nothing."""

    @pytest.mark.asyncio
    async def test_cancel_while_waiting_rolls_back_waiter_count_and_entry(self):
        service = _make_service()
        entered_holder = asyncio.Event()
        release_holder = asyncio.Event()

        async def holder() -> None:
            async with service._named_lock("cancel-me"):
                entered_holder.set()
                await release_holder.wait()

        task_holder = asyncio.create_task(holder())
        await entered_holder.wait()

        async def waiter() -> None:
            async with service._named_lock("cancel-me"):
                raise AssertionError("cancelled waiter must never enter the body")

        task_waiter = asyncio.create_task(waiter())
        await asyncio.sleep(0)
        assert service._locks["cancel-me"].waiters == 2

        task_waiter.cancel()
        with pytest.raises(asyncio.CancelledError):
            await task_waiter

        # Cancellation rolled the waiter count back; only the holder remains.
        assert service._locks["cancel-me"].waiters == 1

        release_holder.set()
        await task_holder

        # Once the holder also finishes, the entry must not have leaked.
        assert "cancel-me" not in service._locks

    @pytest.mark.asyncio
    async def test_repeated_cancel_of_a_queued_waiter_rolls_back_cleanly(self):
        # Cancelling a queued waiter must roll back its waiter count and, once
        # the holder leaves, evict the entry. Calling cancel() a second time
        # before the task is resumed must not double-decrement the count.
        #
        # Note what this can and cannot reach: the rollback in _named_lock's
        # `except BaseException` branch contains no `await`, so a second
        # cancellation cannot interleave *inside* it -- asyncio delivers at
        # most one CancelledError to a task that has not resumed yet, and the
        # synchronous rollback is indivisible on this event loop. The case of
        # a cancellation arriving while an in-flight operation still holds the
        # lock is a different mechanism (_await_shielded) and is covered in
        # test_docker_lifecycle_api.py's repeated-cancellation test.
        service = _make_service()
        entered_holder = asyncio.Event()
        release_holder = asyncio.Event()

        async def holder() -> None:
            async with service._named_lock("double-cancel"):
                entered_holder.set()
                await release_holder.wait()

        task_holder = asyncio.create_task(holder())
        await entered_holder.wait()

        async def waiter() -> None:
            async with service._named_lock("double-cancel"):
                raise AssertionError("cancelled waiter must never enter the body")

        task_waiter = asyncio.create_task(waiter())
        await asyncio.sleep(0)
        assert service._locks["double-cancel"].waiters == 2

        task_waiter.cancel()
        task_waiter.cancel()
        with pytest.raises(asyncio.CancelledError):
            await task_waiter

        assert service._locks["double-cancel"].waiters == 1

        release_holder.set()
        await task_holder

        assert "double-cancel" not in service._locks


class TestGetLiveControl:
    """Pin _get_live_control as the deleted-aware construction point.

    _get_live_control() asserts that _named_lock(name) is held, so every
    call below happens inside that context, matching its real call sites.
    """

    @pytest.mark.asyncio
    async def test_reuses_the_same_live_control_instance(self):
        service = _make_service()
        async with service._named_lock("box"):
            first = service._get_live_control("box")
            second = service._get_live_control("box")
        assert second is first

    @pytest.mark.asyncio
    async def test_replaces_a_deleted_control_with_a_fresh_instance(self):
        service = _make_service()
        async with service._named_lock("box"):
            first = service._get_live_control("box")
            first.deleted = True

            replaced = service._get_live_control("box")

        assert replaced is not first
        assert replaced.deleted is False
        assert service._controls["box"] is replaced

    def test_asserts_when_called_without_holding_the_named_lock(self):
        service = _make_service()
        with pytest.raises(AssertionError):
            service._get_live_control("unlocked")


class TestDeleteIdentityCheckedPop:
    """Pin delete()'s identity-checked pop: a replaced control must survive."""

    @pytest.mark.asyncio
    async def test_delete_does_not_evict_a_control_installed_after_its_lookup(
        self, monkeypatch
    ):
        service = _make_service()
        old_control = service._get_control("box")
        new_control = _SandboxControl(name="box")

        real_find_container = service._find_container

        async def find_container_and_swap(name: str):
            # Simulate another in-flight path installing a fresh control
            # object for this name between delete()'s control lookup and its
            # cleanup pop.
            service._controls[name] = new_control
            return await real_find_container(name)

        monkeypatch.setattr(service, "_find_container", find_container_and_swap)

        await service.delete("box")

        assert old_control is not new_control
        assert service._controls.get("box") is new_control


class TestSandboxControlSingleConstructionPoint:
    """Source-level pin: only _get_control and _get_live_control may
    construct a _SandboxControl. A new inline construction elsewhere is a
    regression of the single-construction-point contract.

    Two sanctioned construction points: _get_control (legacy paths,
    deleted-preserving) and _get_live_control (lock-held paths,
    deleted-replacing); no inline construction elsewhere.
    """

    def test_sandbox_control_constructed_in_exactly_two_places(self):
        """Resolved over the AST, not raw text.

        A substring count also matches occurrences in comments and
        docstrings and breaks on reformatting, neither of which has anything
        to do with the contract. What matters is which *functions* contain a
        real call to the constructor.
        """
        source = Path(docker_sandbox_module.__file__).read_text()
        tree = ast.parse(source)

        constructing_functions: list[str] = []
        for node in ast.walk(tree):
            if not isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
                continue
            for inner in ast.walk(node):
                if (
                    isinstance(inner, ast.Call)
                    and isinstance(inner.func, ast.Name)
                    and inner.func.id == "_SandboxControl"
                ):
                    constructing_functions.append(node.name)

        assert sorted(constructing_functions) == ["_get_control", "_get_live_control"]