Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

11.9 KB
Raw Blame
"""Regression tests for bugs found while verifying the sandbox images guide.

Each test here fails against the code as it was before the accompanying fix.
"""

import inspect

import pytest
from cua_sandbox.image import Image


def _build_helpers():
    """Imported lazily so this module still collects against a tree without the fix."""
    from cua_sandbox.builder.build import build_hash, has_build_work

    return build_hash, has_build_work


class TestEnvAndFilesAreBuildInputs:
    """`.env()` and `.copy()` were dropped on the QEMU VM path: the builder only
    looked at `_layers`, so the variables and files never reached the guest."""

    def test_env_alone_counts_as_build_work(self):
        _, has_build_work = _build_helpers()
        assert has_build_work(Image.linux().env(MY_TOKEN="abc123"))

    def test_copy_alone_counts_as_build_work(self):
        _, has_build_work = _build_helpers()
        assert has_build_work(Image.linux().copy("./config.json", "/app/config.json"))

    def test_layers_alone_still_counts(self):
        _, has_build_work = _build_helpers()
        assert has_build_work(Image.linux().apt_install("curl"))

    def test_a_plain_image_has_no_build_work(self):
        _, has_build_work = _build_helpers()
        assert not has_build_work(Image.linux())

    def test_env_participates_in_the_cache_key(self):
        """Two images differing only by an env var must not share a built disk."""
        build_hash, _ = _build_helpers()
        a = Image.linux().apt_install("curl").env(TOKEN="a")
        b = Image.linux().apt_install("curl").env(TOKEN="b")
        assert build_hash(a) != build_hash(b)

    def test_copied_file_contents_participate_in_the_cache_key(self, tmp_path):
        build_hash, _ = _build_helpers()
        src = tmp_path / "config.json"
        src.write_text('{"v": 1}')
        first = build_hash(Image.linux().copy(str(src), "/app/config.json"))
        src.write_text('{"v": 2}')
        second = build_hash(Image.linux().copy(str(src), "/app/config.json"))
        assert first != second, "editing a copied file must rebuild the image"

    def test_layers_only_images_keep_their_historical_cache_key(self):
        """Existing cached user images stay valid — the key only changes when
        env or files are actually present."""
        from cua_sandbox.builder.overlay import layers_hash

        build_hash, _ = _build_helpers()
        img = Image.linux().apt_install("curl").pip_install("requests")
        assert build_hash(img) == layers_hash(list(img._layers))


class TestConcurrentLocalVMsDoNotCollide:
    """A second concurrent bare-metal sandbox died on fixed VNC/QMP ports and
    corrupted the first VM's UEFI variables through a shared efivars.fd."""

    def test_vnc_display_skips_a_busy_port(self):
        import socket

        from cua_sandbox.runtime.qemu import _find_free_vnc_display

        # Occupy a display the test picks itself instead of assuming host port
        # 5900 is free (a real VNC server or VM on this host would break that).
        taken = None
        for display in range(200, 1200):
            candidate = socket.socket()
            try:
                candidate.bind(("", 5900 + display))
            except OSError:
                candidate.close()
                continue
            taken = candidate
            break
        assert taken is not None, "no free VNC-range port to occupy"
        with taken:
            taken.listen(1)
            found = _find_free_vnc_display(display)
            assert found != display
            assert found > display

    def test_efivars_is_per_vm_not_shared(self):
        """Every session disk lives in one directory; a shared efivars.fd would
        be mapped writable into every concurrent UEFI guest at once."""
        from pathlib import Path

        src = inspect.getsource(
            __import__("cua_sandbox.runtime.qemu", fromlist=["x"]).QEMUBaremetalRuntime.start
        )
        assert 'Path(disk_path).parent / "efivars.fd"' not in src
        assert 'with_suffix(".efivars.fd")' in src

        a = Path("/s/alpha.qcow2").with_suffix(".efivars.fd")
        b = Path("/s/beta.qcow2").with_suffix(".efivars.fd")
        assert a != b

    def test_expose_forwards_ports_on_the_local_qemu_path(self):
        """Image.expose() recorded ports the local runtime never read, so an exposed
        port was silently unreachable: the guest listened, nothing forwarded, and
        nothing errored. Verified live against cua-driver's MCP on guest :3000,
        which answered /healthz and an MCP initialize once forwarded."""
        src = inspect.getsource(
            __import__("cua_sandbox.runtime.qemu", fromlist=["x"]).QEMUBaremetalRuntime.start
        )
        assert "image._ports" in src, "expose() ports are never read"
        assert "extra_hostfwd" in src
        assert "{extra_hostfwd}" in src, "forwards are built but never passed to qemu"
        assert "exposed_ports=exposed_ports" in src, "caller cannot find the host port"

    def test_runtime_info_can_report_exposed_ports(self):
        from cua_sandbox.runtime.base import RuntimeInfo

        info = RuntimeInfo(host="localhost", api_port=1, exposed_ports={3000: 41041})
        assert info.exposed_ports == {3000: 41041}
        assert RuntimeInfo(host="localhost", api_port=1).exposed_ports is None

    def test_qmp_port_is_allocated_not_fixed(self):
        src = inspect.getsource(
            __import__("cua_sandbox.runtime.qemu", fromlist=["x"]).QEMUBaremetalRuntime.start
        )
        assert "_find_free_port(self.qmp_port)" in src
        assert 'f"tcp:127.0.0.1:{self.qmp_port},server,nowait"' not in src


class TestLayerExecutorKnowsTheGuestOS:
    """The builder constructed LayerExecutor without os_type, so Windows `run`
    layers were wrapped in `sudo bash -c`."""

    def test_build_user_image_passes_os_type(self):
        from cua_sandbox.builder import build as build_mod

        src = inspect.getsource(build_mod.build_user_image)
        # Assert on the LayerExecutor construction itself — build_user_image also
        # calls Image.from_file(..., os_type=image.os_type), which must not count.
        start = src.index("LayerExecutor(")
        construction = src[start : src.index(")", src.index("api_port", start))]
        assert "os_type=image.os_type" in construction


@pytest.mark.parametrize(
    "image, expected_kind",
    [
        # The canonical Linux image: the resolver picks the variant per backend.
        (Image.linux(), None),
        (Image.linux(kind="vm"), "vm"),
        (Image.linux(kind="container"), "container"),
        (Image.windows(), "vm"),
    ],
)
def test_to_dict_reports_the_real_kind(image, expected_kind):
    """to_dict() reports the kind the image was asked for (None: resolved later)."""
    assert image.to_dict()["kind"] == expected_kind


class TestDeletingNothingIsNotSuccess:
    """`Sandbox.delete(local=True)` dispatched on runtime_type from a state file
    and took no branch when there was none, so deleting a name that never
    existed reported success — and a container from a launch that timed out
    before writing state could not be deleted at all."""

    def test_unknown_name_raises(self, monkeypatch):
        import asyncio
        import importlib

        # cua_sandbox exports a `sandbox()` function that shadows the submodule.
        sandbox_mod = importlib.import_module("cua_sandbox.sandbox")
        state_mod = importlib.import_module("cua_sandbox.sandbox_state")

        monkeypatch.setattr(state_mod, "load", lambda name: None)
        monkeypatch.setattr(sandbox_mod, "_remove_orphan_container", lambda name: False)

        with pytest.raises(ValueError, match="No local sandbox named"):
            asyncio.run(sandbox_mod.Sandbox._delete_local("never-existed"))

    def test_orphan_container_is_removed(self, monkeypatch):
        import asyncio
        import importlib

        # cua_sandbox exports a `sandbox()` function that shadows the submodule.
        sandbox_mod = importlib.import_module("cua_sandbox.sandbox")
        state_mod = importlib.import_module("cua_sandbox.sandbox_state")

        deleted_state = []
        monkeypatch.setattr(state_mod, "load", lambda name: None)
        monkeypatch.setattr(state_mod, "delete", lambda name: deleted_state.append(name))
        monkeypatch.setattr(sandbox_mod, "_remove_orphan_container", lambda name: True)

        asyncio.run(sandbox_mod.Sandbox._delete_local("timed-out-launch"))
        assert deleted_state == ["timed-out-launch"]

    def test_orphan_probe_reports_missing_container(self, monkeypatch):
        import subprocess

        from cua_sandbox.sandbox import _remove_orphan_container

        def fake_run(cmd, **kwargs):
            assert cmd[:2] == ["docker", "inspect"]
            return subprocess.CompletedProcess(cmd, 1)

        monkeypatch.setattr(subprocess, "run", fake_run)
        assert _remove_orphan_container("nope") is False

    def test_orphan_probe_survives_missing_docker(self, monkeypatch):
        import subprocess

        from cua_sandbox.sandbox import _remove_orphan_container

        def fake_run(cmd, **kwargs):
            raise FileNotFoundError("docker")

        monkeypatch.setattr(subprocess, "run", fake_run)
        assert _remove_orphan_container("nope") is False


class TestLocalRuntimeSelection:
    """`Image.linux()` says kind='vm'. Started locally with no explicit runtime it
    used to land on Docker-wrapped QEMU, which `resolve_image` hands the XFCE
    *container* image — so a VM request quietly became a container, identical to
    `kind='container'`, and never touched the pinned containerDisk."""

    def test_a_linux_vm_boots_under_bare_metal_qemu(self, monkeypatch):
        from cua_sandbox.runtime import compat
        from cua_sandbox.runtime.qemu import QEMUBaremetalRuntime
        from cua_sandbox.sandbox import _auto_runtime

        monkeypatch.setattr(compat, "_has_qemu", lambda: True)
        assert isinstance(_auto_runtime(Image.linux(kind="vm")), QEMUBaremetalRuntime)

    def test_a_linux_container_still_goes_to_docker(self):
        from cua_sandbox.runtime.docker import DockerRuntime
        from cua_sandbox.sandbox import _auto_runtime

        assert isinstance(_auto_runtime(Image.linux(kind="container")), DockerRuntime)

    def test_vm_and_container_do_not_resolve_alike(self, monkeypatch):
        """The regression in one assertion. Comparing the two runtime classes is
        not enough: QEMUDockerRuntime *subclasses* DockerRuntime and resolved to
        the same `trycua/cua-xfce` image, so the old behaviour passed a
        type-inequality check while still booting an identical container. The VM
        must not go through Docker at all."""
        from cua_sandbox.runtime import compat
        from cua_sandbox.runtime.docker import DockerRuntime
        from cua_sandbox.sandbox import _auto_runtime

        monkeypatch.setattr(compat, "_has_qemu", lambda: True)
        vm = _auto_runtime(Image.linux(kind="vm"))
        container = _auto_runtime(Image.linux(kind="container"))

        assert isinstance(container, DockerRuntime)
        assert not isinstance(vm, DockerRuntime)

    def test_missing_qemu_fails_loudly_rather_than_silently_downgrading(self, monkeypatch):
        """Falling back to Docker would hand back the XFCE container, which
        cannot become ready — a 120s timeout instead of a named dependency."""
        import pytest as _pytest
        from cua_sandbox.runtime import compat
        from cua_sandbox.sandbox import _auto_runtime

        monkeypatch.setattr(compat, "_has_qemu", lambda: False)
        with _pytest.raises(RuntimeError, match="needs QEMU"):
            _auto_runtime(Image.linux(kind="vm"))

    def test_a_disk_path_still_reaches_bare_metal_qemu(self):
        from cua_sandbox.runtime.qemu import QEMUBaremetalRuntime
        from cua_sandbox.sandbox import _auto_runtime

        with_disk = Image.from_file("/tmp/does-not-need-to-exist.qcow2", os_type="linux")
        assert isinstance(_auto_runtime(with_disk), QEMUBaremetalRuntime)