Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

19.5 KB
Raw Blame
"""Integration tests — runtime scenarios.

    CUA_RUN_RUNTIME_TESTS=1 pytest tests/test_runtime.py -v -s

Opt-in: these start real containers and VMs on this machine, so the whole
module skips unless ``CUA_RUN_RUNTIME_TESTS=1``. Each test also skips when its
runtime isn't available.
"""

from __future__ import annotations

import os
import platform
import subprocess
from pathlib import Path

import pytest
from cua_sandbox import Image, Sandbox

RUNTIME_TESTS_ENABLED = os.environ.get("CUA_RUN_RUNTIME_TESTS") == "1"

pytestmark = [
    pytest.mark.asyncio,
    pytest.mark.skipif(
        not RUNTIME_TESTS_ENABLED,
        reason="starts real VMs/containers; set CUA_RUN_RUNTIME_TESTS=1 to run",
    ),
]

IS_WINDOWS = platform.system() == "Windows"
IS_MACOS = platform.system() == "Darwin"


# ── Helpers ──────────────────────────────────────────────────────────────────


def _has_docker() -> bool:
    try:
        subprocess.run(["docker", "info"], capture_output=True, check=True, timeout=10)
        return True
    except (subprocess.SubprocessError, FileNotFoundError):
        return False


def _has_lume() -> bool:
    try:
        subprocess.run(["lume", "--version"], capture_output=True, check=True)
        return True
    except (subprocess.SubprocessError, FileNotFoundError):
        return False


def _has_hyperv() -> bool:
    if not IS_WINDOWS:
        return False
    try:
        from cua_sandbox.runtime.hyperv import _has_hyperv as check

        return check()
    except Exception:
        return False


def _has_qemu() -> bool:
    """Check if QEMU is available (on PATH or portable)."""
    try:
        from cua_sandbox.runtime.qemu_installer import qemu_bin

        qemu_bin("x86_64")
        return True
    except (RuntimeError, Exception):
        return False


def _has_android_image() -> bool:
    """Check if an Android-x86 disk image is available for bare-metal QEMU."""
    from pathlib import Path

    storage = Path.home() / ".cua" / "cua-sandbox" / "qemu-storage" / "android-x86.qcow2"
    return storage.exists()


# ── 1. Linux container (Docker) ─────────────────────────────────────────────


@pytest.mark.skipif(not _has_docker(), reason="Docker not available")
async def test_linux_container():
    from cua_sandbox.runtime import DockerRuntime

    async with Sandbox.ephemeral(
        Image.linux("ubuntu", "24.04"),
        local=True,
        runtime=DockerRuntime(api_port=18000, vnc_port=16901, ephemeral=True),
        name="cua-test-linux-container",
    ) as sb:
        result = await sb.shell.run("cat /etc/os-release")
        assert result.success
        assert "ubuntu" in result.stdout.lower()

        screenshot = await sb.screenshot()
        assert screenshot[:4] == b"\x89PNG"


# ── 2. Linux VM (QEMU-in-Docker) ────────────────────────────────────────────


@pytest.mark.skipif(not _has_docker(), reason="Docker not available")
async def test_linux_vm():
    from cua_sandbox.runtime import QEMURuntime

    async with Sandbox.ephemeral(
        Image.linux("ubuntu", "24.04"),
        local=True,
        runtime=QEMURuntime(mode="docker", api_port=18001, vnc_port=18006, ephemeral=True),
        name="cua-test-linux-vm",
    ) as sb:
        result = await sb.shell.run("uname -a")
        assert result.success
        assert "Linux" in result.stdout

        screenshot = await sb.screenshot()
        assert screenshot[:4] == b"\x89PNG"


# ── 3. Windows VM (bare-metal QEMU on Windows host) ─────────────────────────


@pytest.mark.skipif(not IS_WINDOWS, reason="Windows host only")
@pytest.mark.skipif(not _has_qemu(), reason="QEMU not available")
async def test_windows_vm():
    from cua_sandbox.runtime import QEMURuntime

    async with Sandbox.ephemeral(
        Image.windows("11"),
        local=True,
        runtime=QEMURuntime(mode="bare-metal", api_port=18002),
        name="cua-test-windows-vm",
    ) as sb:
        result = await sb.shell.run("ver")
        assert result.success or "Windows" in result.stdout

        screenshot = await sb.screenshot()
        assert screenshot[:4] == b"\x89PNG"


# ── 3b. Windows VM (Hyper-V) ────────────────────────────────────────────────


@pytest.mark.skipif(not _has_hyperv(), reason="Hyper-V not available")
async def test_windows_hyperv():
    from cua_sandbox.runtime import HyperVRuntime

    async with Sandbox.ephemeral(
        Image.windows("11"),
        local=True,
        runtime=HyperVRuntime(api_port=18003),
        name="cua-test-hyperv",
    ) as sb:
        result = await sb.shell.run("ver")
        assert result.success or "Windows" in result.stdout

        screenshot = await sb.screenshot()
        assert screenshot[:4] == b"\x89PNG"


# ── 4. macOS VM (Lume) ──────────────────────────────────────────────────────


@pytest.mark.skipif(not IS_MACOS or not _has_lume(), reason="Lume only on macOS")
async def test_macos_vm():
    from cua_sandbox.runtime import LumeRuntime

    async with Sandbox.ephemeral(
        Image.macos("26"),
        local=True,
        runtime=LumeRuntime(),
        name="cua-test-macos-vm",
    ) as sb:
        result = await sb.shell.run("sw_vers")
        assert result.success
        assert "macOS" in result.stdout or "Mac" in result.stdout

        screenshot = await sb.screenshot()
        assert screenshot[:4] == b"\x89PNG"


# ── 5. Android VM (bare-metal QEMU) ──────────────────────────────────────────


def _has_android_iso() -> bool:
    """Check if an Android-x86 ISO is available in ~/Downloads."""
    from pathlib import Path

    return any(Path.home().glob("Downloads/android-x86*.iso"))


def _get_android_iso() -> str:
    from pathlib import Path

    isos = sorted(Path.home().glob("Downloads/android-x86*.iso"))
    return str(isos[0]) if isos else ""


@pytest.mark.skipif(not _has_qemu(), reason="QEMU not available")
@pytest.mark.skipif(
    not _has_android_image(),
    reason="Android-x86 qcow2 not available at ~/.cua/cua-sandbox/qemu-storage/android-x86.qcow2",
)
async def test_android_vm_baremetal():
    """Test Android VM via bare-metal QEMU with a pre-built qcow2."""
    from pathlib import Path

    from cua_sandbox.runtime import QEMURuntime

    android_disk = str(Path.home() / ".cua" / "cua-sandbox" / "qemu-storage" / "android-x86.qcow2")
    async with Sandbox.ephemeral(
        Image.from_file(android_disk, os_type="android", kind="vm"),
        local=True,
        runtime=QEMURuntime(
            mode="bare-metal", api_port=18010, vnc_display=10, memory_mb=4096, cpu_count=4
        ),
        name="cua-test-android-vm",
    ) as sb:
        screenshot = await sb.screenshot()
        assert screenshot[:4] == b"\x89PNG"
        assert len(screenshot) > 1000


@pytest.mark.skipif(not _has_qemu(), reason="QEMU not available")
@pytest.mark.skipif(not _has_android_iso(), reason="No android-x86*.iso in ~/Downloads")
async def test_android_vm_from_iso():
    """Test Android VM booted from an ISO file via bare-metal QEMU.

    The runtime auto-creates a qcow2 disk and attaches the ISO as CD-ROM.
    Uses QMP transport (no computer-server required).
    """
    from cua_sandbox.runtime import QEMURuntime

    iso_path = _get_android_iso()
    async with Sandbox.ephemeral(
        Image.from_file(iso_path, os_type="android", kind="vm"),
        local=True,
        runtime=QEMURuntime(
            mode="bare-metal",
            api_port=18030,
            vnc_display=30,
            qmp_port=4460,
            memory_mb=2048,
            cpu_count=2,
        ),
        name="cua-test-android-iso3",
    ) as sb:
        # Wait for boot to progress past GRUB (Enter keys are sent during start())
        import asyncio

        await asyncio.sleep(15)

        # With QMP transport, we can take screenshots even without computer-server
        screenshot = await sb.screenshot()
        assert screenshot[:4] == b"\x89PNG"
        assert len(screenshot) > 100

        # Save screenshot for inspection
        with open("/tmp/android-boot-screenshot.png", "wb") as f:
            f.write(screenshot)


# ── 5b. Android VM (QEMU-in-Docker) ──────────────────────────────────────────


@pytest.mark.skipif(not _has_docker(), reason="Docker not available")
async def test_android_vm_docker():
    """Test Android VM via QEMU inside Docker."""
    from cua_sandbox.runtime import QEMURuntime

    async with Sandbox.ephemeral(
        Image.android("14"),
        local=True,
        runtime=QEMURuntime(mode="docker", api_port=18011, vnc_port=18080, ephemeral=True),
        name="cua-test-android-docker",
    ) as sb:
        screenshot = await sb.screenshot()
        assert screenshot[:4] == b"\x89PNG"
        assert len(screenshot) > 1000


# ── 6. QEMU bare-metal error on missing binary ───────────────────────────────


def _has_osworld_image() -> bool:
    """Check if the OSWorld Ubuntu qcow2 is available in the image cache."""
    cache = Path.home() / ".cua" / "cua-sandbox" / "image-cache"
    return any(cache.rglob("Ubuntu.qcow2")) if cache.exists() else False


@pytest.mark.skipif(not _has_qemu(), reason="QEMU not available")
@pytest.mark.skipif(not _has_osworld_image(), reason="OSWorld Ubuntu.qcow2 not in image cache")
async def test_osworld_ubuntu_vm():
    """Test OSWorld Ubuntu VM via bare-metal QEMU with OSWorld transport."""
    from cua_sandbox.runtime import QEMURuntime

    async with Sandbox.ephemeral(
        Image.from_file(
            "https://huggingface.co/datasets/xlangai/ubuntu_osworld/resolve/main/Ubuntu.qcow2.zip",
            os_type="linux",
            agent_type="osworld",
        ),
        local=True,
        runtime=QEMURuntime(
            mode="bare-metal", api_port=18020, vnc_display=20, memory_mb=4096, cpu_count=4
        ),
        name="cua-test-osworld",
    ) as sb:
        screenshot = await sb.screenshot()
        assert screenshot[:4] == b"\x89PNG"

        w, h = await sb.get_dimensions()
        assert w == 1920 and h == 1080


# ── 7. Android Emulator (SDK) ────────────────────────────────────────────────


def _has_android_sdk() -> bool:
    """Check if Android SDK emulator is available."""
    try:
        from cua_sandbox.runtime.android_emulator import _find_bin, _sdk_path

        sdk = _sdk_path()
        _find_bin(sdk, "emulator")
        _find_bin(sdk, "adb")
        return True
    except (FileNotFoundError, Exception):
        return False


@pytest.mark.skipif(not _has_android_sdk(), reason="Android SDK not available")
async def test_android_emulator():
    """Test Android emulator boots to homescreen."""
    from cua_sandbox.runtime import AndroidEmulatorRuntime

    async with Sandbox.ephemeral(
        Image.android("14"),
        local=True,
        runtime=AndroidEmulatorRuntime(memory_mb=4096, cpu_count=4, adb_port=5559),
        name="cua-test-android-sdk",
    ) as sb:
        screenshot = await sb.screenshot()
        assert screenshot[:4] == b"\x89PNG"
        assert len(screenshot) > 10000


@pytest.mark.skipif(not _has_android_sdk(), reason="Android SDK not available")
async def test_android_emulator_apk_install():
    """Test Android emulator with F-Droid APK installed from URL."""
    from cua_sandbox.runtime import AndroidEmulatorRuntime

    img = Image.android("14").apk_install("https://f-droid.org/F-Droid.apk")
    async with Sandbox.ephemeral(
        img,
        local=True,
        runtime=AndroidEmulatorRuntime(memory_mb=4096, cpu_count=4, adb_port=5561),
        name="cua-test-android-apk",
    ) as sb:
        screenshot = await sb.screenshot()
        assert screenshot[:4] == b"\x89PNG"

        # Verify F-Droid is installed
        result = await sb._transport.send("shell", command="pm list packages | grep fdroid")
        assert "org.fdroid.fdroid" in result["output"]


# ── 8. Tart (Apple VZ) ────────────────────────────────────────────────────────


def _has_tart() -> bool:
    import shutil

    return shutil.which("tart") is not None


@pytest.mark.skipif(not IS_MACOS, reason="Tart only on macOS")
@pytest.mark.skipif(not _has_tart(), reason="Tart CLI not available")
async def test_tart_ubuntu():
    """Test Ubuntu VM via Tart (Apple VZ)."""
    from cua_sandbox.runtime import TartRuntime

    async with Sandbox.ephemeral(
        Image.from_registry("ghcr.io/cirruslabs/ubuntu:latest"),
        local=True,
        runtime=TartRuntime(ephemeral=True, display="1024x768"),
        name="cua-test-tart-ubuntu",
    ) as sb:
        screenshot = await sb.screenshot()
        assert screenshot[:4] == b"\x89PNG"
        assert len(screenshot) > 1000

        result = await sb.shell.run("uname -a")
        assert result.success
        assert "Linux" in result.stdout


@pytest.mark.skipif(not IS_MACOS, reason="Tart only on macOS")
@pytest.mark.skipif(not _has_tart(), reason="Tart CLI not available")
async def test_tart_macos_tahoe():
    """Test macOS Tahoe VM via Tart (Apple VZ)."""
    from cua_sandbox.runtime import TartRuntime

    async with Sandbox.ephemeral(
        Image.from_registry("ghcr.io/cirruslabs/macos-tahoe-base:latest"),
        local=True,
        runtime=TartRuntime(ephemeral=True),
        name="cua-test-tart-tahoe",
    ) as sb:
        screenshot = await sb.screenshot()
        assert screenshot[:4] == b"\x89PNG"
        assert len(screenshot) > 1000


@pytest.mark.skipif(not IS_MACOS, reason="Tart only on macOS")
@pytest.mark.skipif(not _has_tart(), reason="Tart CLI not available")
async def test_tart_macos_sequoia_cua():
    """Test CUA macOS Sequoia sparse image via Tart."""
    from cua_sandbox.runtime import TartRuntime

    async with Sandbox.ephemeral(
        Image.from_registry("ghcr.io/trycua/macos-sequoia-cua-sparse:latest-oci-layered"),
        local=True,
        runtime=TartRuntime(ephemeral=True),
        name="cua-test-tart-sequoia",
    ) as sb:
        screenshot = await sb.screenshot()
        assert screenshot[:4] == b"\x89PNG"
        assert len(screenshot) > 1000


@pytest.mark.skipif(not IS_MACOS or not _has_lume(), reason="Lume only on macOS")
async def test_lume_macos_tahoe_cua():
    """Test CUA macOS Tahoe image via Lume from ghcr.io/trycua/macos-tahoe-cua:latest."""
    from cua_sandbox.runtime import LumeRuntime

    async with Sandbox.ephemeral(
        Image.from_registry("ghcr.io/trycua/macos-tahoe-cua:latest"),
        local=True,
        runtime=LumeRuntime(),
        name="cua-test-lume-tahoe",
    ) as sb:
        screenshot = await sb.screenshot()
        assert screenshot[:4] == b"\x89PNG"
        assert len(screenshot) > 1000


# ── 9. QEMU bare-metal error on missing binary ───────────────────────────────


def test_qemu_baremetal_missing_binary_error():
    """When QEMU isn't installed, bare-metal runtime should give a helpful error."""
    import shutil

    if shutil.which("qemu-system-x86_64"):
        pytest.skip("QEMU is installed — cannot test missing-binary error")

    with pytest.raises(RuntimeError, match="brew install qemu|not found"):
        from cua_sandbox.runtime.qemu_installer import qemu_bin

        qemu_bin("x86_64")


# ── 10. Auto-runtime (local=True, no explicit runtime) ───────────────────────


@pytest.mark.skipif(not _has_docker(), reason="Docker not available")
async def test_auto_runtime_linux_container():
    """local=True with no runtime auto-selects DockerRuntime for linux container images."""
    async with Sandbox.ephemeral(
        Image.linux("ubuntu", "24.04"),  # kind="container" by default
        local=True,
        name="cua-test-auto-linux-container",
    ) as sb:
        result = await sb.shell.run("uname -s")
        assert result.success
        assert "Linux" in result.stdout

        screenshot = await sb.screenshot()
        assert screenshot[:4] == b"\x89PNG"


@pytest.mark.skipif(not _has_docker(), reason="Docker not available")
async def test_auto_runtime_linux_vm():
    """local=True with no runtime auto-selects QEMURuntime(docker) for linux vm images."""
    async with Sandbox.ephemeral(
        Image.linux("ubuntu", "24.04", kind="vm"),
        local=True,
        name="cua-test-auto-linux-vm",
    ) as sb:
        result = await sb.shell.run("uname -s")
        assert result.success
        assert "Linux" in result.stdout

        screenshot = await sb.screenshot()
        assert screenshot[:4] == b"\x89PNG"


@pytest.mark.skipif(not IS_MACOS or not _has_lume(), reason="Lume only on macOS")
async def test_auto_runtime_macos():
    """local=True with no runtime auto-selects LumeRuntime for macOS images."""
    async with Sandbox.ephemeral(
        Image.macos("26"),
        local=True,
        name="cua-test-auto-macos",
    ) as sb:
        result = await sb.shell.run("sw_vers")
        assert result.success
        assert "macOS" in result.stdout or "Mac" in result.stdout

        screenshot = await sb.screenshot()
        assert screenshot[:4] == b"\x89PNG"


@pytest.mark.skipif(not _has_android_sdk(), reason="Android SDK not available")
async def test_auto_runtime_android():
    """local=True with no runtime auto-selects AndroidEmulatorRuntime for android images."""
    async with Sandbox.ephemeral(
        Image.android("14"),
        local=True,
        name="cua-test-auto-android",
    ) as sb:
        screenshot = await sb.screenshot()
        assert screenshot[:4] == b"\x89PNG"
        assert len(screenshot) > 10000


@pytest.mark.skipif(not IS_WINDOWS, reason="Windows host only")
@pytest.mark.skipif(not _has_qemu(), reason="QEMU not available")
async def test_auto_runtime_windows():
    """local=True with no runtime auto-selects HyperV or QEMU for Windows images."""
    async with Sandbox.ephemeral(
        Image.windows("11"),
        local=True,
        name="cua-test-auto-windows",
    ) as sb:
        result = await sb.shell.run("ver")
        assert result.success or "Windows" in result.stdout

        screenshot = await sb.screenshot()
        assert screenshot[:4] == b"\x89PNG"


# ── 12. Sandbox.create (persistent) ─────────────────────────────────────────


@pytest.mark.skipif(not _has_docker(), reason="Docker not available")
async def test_create_persistent_linux():
    """Sandbox.create provisions a persistent sandbox that survives disconnect."""
    from cua_sandbox.runtime import DockerRuntime

    sb = await Sandbox.create(
        Image.linux("ubuntu", "24.04"),
        local=True,
        runtime=DockerRuntime(api_port=18090, vnc_port=18091, ephemeral=True),
        name="cua-test-create-linux",
    )
    try:
        result = await sb.shell.run("echo persistent")
        assert result.success
        assert "persistent" in result.stdout

        screenshot = await sb.screenshot()
        assert screenshot[:4] == b"\x89PNG"
    finally:
        await sb.disconnect()