Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
"""Regression tests for bugs found while verifying the sandbox images guide.
Each test here fails against the code as it was before the accompanying fix.
"""
import inspect
import pytest
from cua_sandbox.image import Image
def _build_helpers():
"""Imported lazily so this module still collects against a tree without the fix."""
from cua_sandbox.builder.build import build_hash, has_build_work
return build_hash, has_build_work
class TestEnvAndFilesAreBuildInputs:
"""`.env()` and `.copy()` were dropped on the QEMU VM path: the builder only
looked at `_layers`, so the variables and files never reached the guest."""
def test_env_alone_counts_as_build_work(self):
_, has_build_work = _build_helpers()
assert has_build_work(Image.linux().env(MY_TOKEN="abc123"))
def test_copy_alone_counts_as_build_work(self):
_, has_build_work = _build_helpers()
assert has_build_work(Image.linux().copy("./config.json", "/app/config.json"))
def test_layers_alone_still_counts(self):
_, has_build_work = _build_helpers()
assert has_build_work(Image.linux().apt_install("curl"))
def test_a_plain_image_has_no_build_work(self):
_, has_build_work = _build_helpers()
assert not has_build_work(Image.linux())
def test_env_participates_in_the_cache_key(self):
"""Two images differing only by an env var must not share a built disk."""
build_hash, _ = _build_helpers()
a = Image.linux().apt_install("curl").env(TOKEN="a")
b = Image.linux().apt_install("curl").env(TOKEN="b")
assert build_hash(a) != build_hash(b)
def test_copied_file_contents_participate_in_the_cache_key(self, tmp_path):
build_hash, _ = _build_helpers()
src = tmp_path / "config.json"
src.write_text('{"v": 1}')
first = build_hash(Image.linux().copy(str(src), "/app/config.json"))
src.write_text('{"v": 2}')
second = build_hash(Image.linux().copy(str(src), "/app/config.json"))
assert first != second, "editing a copied file must rebuild the image"
def test_layers_only_images_keep_their_historical_cache_key(self):
"""Existing cached user images stay valid — the key only changes when
env or files are actually present."""
from cua_sandbox.builder.overlay import layers_hash
build_hash, _ = _build_helpers()
img = Image.linux().apt_install("curl").pip_install("requests")
assert build_hash(img) == layers_hash(list(img._layers))
class TestConcurrentLocalVMsDoNotCollide:
"""A second concurrent bare-metal sandbox died on fixed VNC/QMP ports and
corrupted the first VM's UEFI variables through a shared efivars.fd."""
def test_vnc_display_skips_a_busy_port(self):
import socket
from cua_sandbox.runtime.qemu import _find_free_vnc_display
# Occupy a display the test picks itself instead of assuming host port
# 5900 is free (a real VNC server or VM on this host would break that).
taken = None
for display in range(200, 1200):
candidate = socket.socket()
try:
candidate.bind(("", 5900 + display))
except OSError:
candidate.close()
continue
taken = candidate
break
assert taken is not None, "no free VNC-range port to occupy"
with taken:
taken.listen(1)
found = _find_free_vnc_display(display)
assert found != display
assert found > display
def test_efivars_is_per_vm_not_shared(self):
"""Every session disk lives in one directory; a shared efivars.fd would
be mapped writable into every concurrent UEFI guest at once."""
from pathlib import Path
src = inspect.getsource(
__import__("cua_sandbox.runtime.qemu", fromlist=["x"]).QEMUBaremetalRuntime.start
)
assert 'Path(disk_path).parent / "efivars.fd"' not in src
assert 'with_suffix(".efivars.fd")' in src
a = Path("/s/alpha.qcow2").with_suffix(".efivars.fd")
b = Path("/s/beta.qcow2").with_suffix(".efivars.fd")
assert a != b
def test_expose_forwards_ports_on_the_local_qemu_path(self):
"""Image.expose() recorded ports the local runtime never read, so an exposed
port was silently unreachable: the guest listened, nothing forwarded, and
nothing errored. Verified live against cua-driver's MCP on guest :3000,
which answered /healthz and an MCP initialize once forwarded."""
src = inspect.getsource(
__import__("cua_sandbox.runtime.qemu", fromlist=["x"]).QEMUBaremetalRuntime.start
)
assert "image._ports" in src, "expose() ports are never read"
assert "extra_hostfwd" in src
assert "{extra_hostfwd}" in src, "forwards are built but never passed to qemu"
assert "exposed_ports=exposed_ports" in src, "caller cannot find the host port"
def test_runtime_info_can_report_exposed_ports(self):
from cua_sandbox.runtime.base import RuntimeInfo
info = RuntimeInfo(host="localhost", api_port=1, exposed_ports={3000: 41041})
assert info.exposed_ports == {3000: 41041}
assert RuntimeInfo(host="localhost", api_port=1).exposed_ports is None
def test_qmp_port_is_allocated_not_fixed(self):
src = inspect.getsource(
__import__("cua_sandbox.runtime.qemu", fromlist=["x"]).QEMUBaremetalRuntime.start
)
assert "_find_free_port(self.qmp_port)" in src
assert 'f"tcp:127.0.0.1:{self.qmp_port},server,nowait"' not in src
class TestLayerExecutorKnowsTheGuestOS:
"""The builder constructed LayerExecutor without os_type, so Windows `run`
layers were wrapped in `sudo bash -c`."""
def test_build_user_image_passes_os_type(self):
from cua_sandbox.builder import build as build_mod
src = inspect.getsource(build_mod.build_user_image)
# Assert on the LayerExecutor construction itself — build_user_image also
# calls Image.from_file(..., os_type=image.os_type), which must not count.
start = src.index("LayerExecutor(")
construction = src[start : src.index(")", src.index("api_port", start))]
assert "os_type=image.os_type" in construction
@pytest.mark.parametrize(
"image, expected_kind",
[
# The canonical Linux image: the resolver picks the variant per backend.
(Image.linux(), None),
(Image.linux(kind="vm"), "vm"),
(Image.linux(kind="container"), "container"),
(Image.windows(), "vm"),
],
)
def test_to_dict_reports_the_real_kind(image, expected_kind):
"""to_dict() reports the kind the image was asked for (None: resolved later)."""
assert image.to_dict()["kind"] == expected_kind
class TestDeletingNothingIsNotSuccess:
"""`Sandbox.delete(local=True)` dispatched on runtime_type from a state file
and took no branch when there was none, so deleting a name that never
existed reported success — and a container from a launch that timed out
before writing state could not be deleted at all."""
def test_unknown_name_raises(self, monkeypatch):
import asyncio
import importlib
# cua_sandbox exports a `sandbox()` function that shadows the submodule.
sandbox_mod = importlib.import_module("cua_sandbox.sandbox")
state_mod = importlib.import_module("cua_sandbox.sandbox_state")
monkeypatch.setattr(state_mod, "load", lambda name: None)
monkeypatch.setattr(sandbox_mod, "_remove_orphan_container", lambda name: False)
with pytest.raises(ValueError, match="No local sandbox named"):
asyncio.run(sandbox_mod.Sandbox._delete_local("never-existed"))
def test_orphan_container_is_removed(self, monkeypatch):
import asyncio
import importlib
# cua_sandbox exports a `sandbox()` function that shadows the submodule.
sandbox_mod = importlib.import_module("cua_sandbox.sandbox")
state_mod = importlib.import_module("cua_sandbox.sandbox_state")
deleted_state = []
monkeypatch.setattr(state_mod, "load", lambda name: None)
monkeypatch.setattr(state_mod, "delete", lambda name: deleted_state.append(name))
monkeypatch.setattr(sandbox_mod, "_remove_orphan_container", lambda name: True)
asyncio.run(sandbox_mod.Sandbox._delete_local("timed-out-launch"))
assert deleted_state == ["timed-out-launch"]
def test_orphan_probe_reports_missing_container(self, monkeypatch):
import subprocess
from cua_sandbox.sandbox import _remove_orphan_container
def fake_run(cmd, **kwargs):
assert cmd[:2] == ["docker", "inspect"]
return subprocess.CompletedProcess(cmd, 1)
monkeypatch.setattr(subprocess, "run", fake_run)
assert _remove_orphan_container("nope") is False
def test_orphan_probe_survives_missing_docker(self, monkeypatch):
import subprocess
from cua_sandbox.sandbox import _remove_orphan_container
def fake_run(cmd, **kwargs):
raise FileNotFoundError("docker")
monkeypatch.setattr(subprocess, "run", fake_run)
assert _remove_orphan_container("nope") is False
class TestLocalRuntimeSelection:
"""`Image.linux()` says kind='vm'. Started locally with no explicit runtime it
used to land on Docker-wrapped QEMU, which `resolve_image` hands the XFCE
*container* image — so a VM request quietly became a container, identical to
`kind='container'`, and never touched the pinned containerDisk."""
def test_a_linux_vm_boots_under_bare_metal_qemu(self, monkeypatch):
from cua_sandbox.runtime import compat
from cua_sandbox.runtime.qemu import QEMUBaremetalRuntime
from cua_sandbox.sandbox import _auto_runtime
monkeypatch.setattr(compat, "_has_qemu", lambda: True)
assert isinstance(_auto_runtime(Image.linux(kind="vm")), QEMUBaremetalRuntime)
def test_a_linux_container_still_goes_to_docker(self):
from cua_sandbox.runtime.docker import DockerRuntime
from cua_sandbox.sandbox import _auto_runtime
assert isinstance(_auto_runtime(Image.linux(kind="container")), DockerRuntime)
def test_vm_and_container_do_not_resolve_alike(self, monkeypatch):
"""The regression in one assertion. Comparing the two runtime classes is
not enough: QEMUDockerRuntime *subclasses* DockerRuntime and resolved to
the same `trycua/cua-xfce` image, so the old behaviour passed a
type-inequality check while still booting an identical container. The VM
must not go through Docker at all."""
from cua_sandbox.runtime import compat
from cua_sandbox.runtime.docker import DockerRuntime
from cua_sandbox.sandbox import _auto_runtime
monkeypatch.setattr(compat, "_has_qemu", lambda: True)
vm = _auto_runtime(Image.linux(kind="vm"))
container = _auto_runtime(Image.linux(kind="container"))
assert isinstance(container, DockerRuntime)
assert not isinstance(vm, DockerRuntime)
def test_missing_qemu_fails_loudly_rather_than_silently_downgrading(self, monkeypatch):
"""Falling back to Docker would hand back the XFCE container, which
cannot become ready — a 120s timeout instead of a named dependency."""
import pytest as _pytest
from cua_sandbox.runtime import compat
from cua_sandbox.sandbox import _auto_runtime
monkeypatch.setattr(compat, "_has_qemu", lambda: False)
with _pytest.raises(RuntimeError, match="needs QEMU"):
_auto_runtime(Image.linux(kind="vm"))
def test_a_disk_path_still_reaches_bare_metal_qemu(self):
from cua_sandbox.runtime.qemu import QEMUBaremetalRuntime
from cua_sandbox.sandbox import _auto_runtime
with_disk = Image.from_file("/tmp/does-not-need-to-exist.qcow2", os_type="linux")
assert isinstance(_auto_runtime(with_disk), QEMUBaremetalRuntime)