Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

14.2 KB
Raw Blame
#!/usr/bin/env python3
"""Smoke one variant of a benchmark image through the cua SDK (the path
``cb run`` uses), on this machine.

    smoke.py ID --variant rootfs|containerdisk [--image REF | --disk PATH]
             [--evidence DIR] [--memory-mb N]

Rootfs defaults to ``cua-e2e-local/bench-ID:docker-local-<host arch>``
(the SDK runs it under gVisor); containerdisk defaults to
``$CUA_BENCH_WORK/ID/out/<arch>/disk.img`` under QEMU (one VM, at most
4 GiB).

Checks, recorded in ``EVIDENCE/<variant>-<where>.json`` with screenshots:
  server       the bench server's health probe (bench.json server.health)
  spacesd       cua-spacesd GetCapabilities
  screen       screenshot size matches the desktop and is not blank
  input        a cua-driver click changes the screen (benchmark hook)
  bench        the benchmark's own checks (libs/images/bench/ID/smoke.py)
  doctor       `cua-spacesd doctor --json` when the image's spacesd has it
Exit 0 only when every check passes (the doctor is skipped, not failed, on
spacesd builds that predate it).
"""

from __future__ import annotations

import argparse
import asyncio
import importlib.util
import io
import json
import os
import platform
import sys
import time
import uuid
from pathlib import Path

HERE = Path(__file__).resolve().parent
sys.path.insert(0, str(HERE))
import bench as benchlib  # noqa: E402


def host_arch() -> str:
    return "arm64" if platform.machine().lower() in ("arm64", "aarch64") else "amd64"


class Report:
    def __init__(self) -> None:
        self.checks: list[dict] = []

    def add(self, name: str, ok: bool | None, detail: object = "", t0: float | None = None) -> None:
        row = {"check": name, "status": "pass" if ok else ("skip" if ok is None else "fail"),
               "detail": detail}
        if t0 is not None:
            row["seconds"] = round(time.monotonic() - t0, 2)
        self.checks.append(row)
        print(f"  {row['status']:4s} {name}: {str(detail)[:300]}", flush=True)

    @property
    def ok(self) -> bool:
        return all(c["status"] != "fail" for c in self.checks)


def png_info(data: bytes) -> tuple[int, int, float]:
    """(width, height, fraction of distinct 8x8-sampled colors) of a PNG."""
    from PIL import Image

    im = Image.open(io.BytesIO(data)).convert("RGB")
    w, h = im.size
    small = im.resize((64, 64))
    colors = len(set(small.get_flattened_data() if hasattr(small, 'get_flattened_data') else small.getdata()))
    return w, h, colors / 4096.0


def screen_diff(a: bytes, b: bytes) -> float:
    """Fraction of pixels that differ between two screenshots (downscaled)."""
    from PIL import Image, ImageChops

    ia = Image.open(io.BytesIO(a)).convert("L").resize((160, 90))
    ib = Image.open(io.BytesIO(b)).convert("L").resize((160, 90))
    diff = ImageChops.difference(ia, ib)
    px = diff.get_flattened_data() if hasattr(diff, "get_flattened_data") else diff.getdata()
    return sum(1 for p in px if p > 16) / (160 * 90)


def load_hooks(bench_id: str):
    path = benchlib.BENCH_ROOT / bench_id / "smoke.py"
    if not path.is_file():
        return None
    spec = importlib.util.spec_from_file_location(f"bench_smoke_{bench_id}", path)
    mod = importlib.util.module_from_spec(spec)
    spec.loader.exec_module(mod)  # type: ignore[union-attr]
    return mod


async def run_checks(sb, b: dict, hooks, report: Report, evidence: Path, tag: str) -> None:
    server = b.get("server") or {}
    method, _, path = (server.get("health") or "GET /").partition(" ")
    t0 = time.monotonic()
    last = None
    for _ in range(90):  # bounded: ~3 minutes for the server to come up after readiness
        try:
            r = await sb.service("server").request(method, path or "/", timeout=20)
            last = r.status_code
            if r.status_code == 200:
                break
        except Exception as e:  # noqa: BLE001 - still booting
            last = repr(e)[:200]
        await asyncio.sleep(2)
    report.add("server", last == 200, f"{method} {path} on :{server.get('port')} -> {last}", t0)

    t0 = time.monotonic()
    try:
        g = await sb.spacesd()
        caps = json.loads(await g.call_json("SystemService/GetCapabilities", "{}"))
        (evidence / f"{tag}-capabilities.json").write_text(json.dumps(caps, indent=2))
        report.add("spacesd", True, f"GetCapabilities: {', '.join(sorted(caps))[:200]}", t0)
    except Exception as e:  # noqa: BLE001
        report.add("spacesd", False, repr(e)[:300], t0)

    t0 = time.monotonic()
    shot = b""
    try:
        # Readiness means spacesd answers; the desktop may still be painting.
        # A VM's spacesd may still await its token (cloud-init) when the
        # bench server already answers, so early calls are retried too.
        err = None
        # bounded: ~6 min (a GNOME session on a 2-vCPU CI runner paints after ~2-3 min)
        for _ in range(180):
            try:
                shot = await sb.screenshot()
            except Exception as e:  # noqa: BLE001
                err = e
                await asyncio.sleep(2)
                continue
            w, h, variety = png_info(shot)
            if variety > 0.002:
                break
            await asyncio.sleep(2)
        if not shot:
            raise RuntimeError(f"no screenshot: {err!r}")
        (evidence / f"{tag}-screen.png").write_bytes(shot)
        want = (b.get("desktop") or {}).get("resolution")
        size_ok = want is None or f"{w}x{h}" == want
        report.add("screen", size_ok and variety > 0.002,
                   f"{w}x{h} (want {want}), color variety {variety:.3f}", t0)
    except Exception as e:  # noqa: BLE001
        report.add("screen", False, repr(e)[:300], t0)

    if hooks is not None and hasattr(hooks, "input_probe"):
        t0 = time.monotonic()
        try:
            detail = await hooks.input_probe(sb, evidence, tag, screen_diff)
            report.add("input", bool(detail.get("ok")), detail, t0)
        except Exception as e:  # noqa: BLE001
            report.add("input", False, repr(e)[:300], t0)
    if hooks is not None and hasattr(hooks, "bench_checks"):
        t0 = time.monotonic()
        try:
            for name, ok, detail in await hooks.bench_checks(sb, evidence, tag):
                report.add(f"bench.{name}", ok, detail)
        except Exception as e:  # noqa: BLE001
            report.add("bench", False, repr(e)[:300], t0)

    t0 = time.monotonic()
    try:
        # The image's own claims (/etc/cua-image/manifest.json) are the bar:
        # --strict fails on warnings too. Publishing is gated on this check.
        # --effects virtual: the doctor may drive the image's own fixture
        # windows (never anything a task left open; the sandbox is fresh).
        # As root when the guest allows it (passwordless sudo), like a pod
        # probe: the claim token directory is root-only (0700).
        r = await sb.shell.run("S=; [ \"$(id -u)\" = 0 ] || { sudo -n true 2>/dev/null && S='sudo -n'; }; "
                               # Images staged before the cua-guestd -> cua-spacesd rename ship the
                               # same daemon (and doctor flags) under the old name.
                               "D=$(command -v cua-spacesd || command -v cua-guestd || echo cua-spacesd); "
                               "$S \"$D\" doctor --strict --effects virtual --json >/tmp/cua-doctor.json 2>/tmp/cua-doctor.err; "
                               "echo \"rc=$?\"; cat /tmp/cua-doctor.json", timeout=600)
        out = getattr(r, "stdout", "") or ""
        rc = out.split("rc=", 1)[1].split()[0] if "rc=" in out else "?"
        report_json = out.split("\n", 1)[1] if "\n" in out else ""
        (evidence / f"{tag}-doctor.json").write_text(report_json)
        detail = f"rc={rc} (report in {tag}-doctor.json)"
        try:
            doc = json.loads(report_json)
            fails = [c.get("id") for c in doc.get("checks", []) if c.get("status") in ("fail", "warn")]
            detail += f"; not passing: {fails[:12]}" if fails else "; all checks pass"
        except ValueError:
            err = await sb.shell.run("tail -c 600 /tmp/cua-doctor.err", timeout=30)
            detail += f"; stderr: {(getattr(err, 'stdout', '') or '').strip()[:300]}"
        report.add("doctor", rc == "0", detail, t0)
    except Exception as e:  # noqa: BLE001
        report.add("doctor", False, f"not run: {e!r}"[:300], t0)


def child_for(index_ref: str, variant: str, arch: str) -> tuple[str, str]:
    """(index that holds the variant, repo@child digest for arch), read with crane."""
    import subprocess

    def manifest(ref: str) -> dict:
        env = dict(os.environ)
        if os.environ.get("CUA_BENCH_REGISTRY_CONFIG"):  # crane only, never the docker CLI
            env["DOCKER_CONFIG"] = os.environ["CUA_BENCH_REGISTRY_CONFIG"]
        return json.loads(subprocess.run(["crane", "manifest", ref], check=True,
                                         capture_output=True, text=True, env=env).stdout)

    repo = index_ref.split("@", 1)[0]
    idx = manifest(index_ref)
    if variant == "containerdisk" and (idx.get("annotations") or {}).get("ai.cua.image.variant") != "containerdisk":
        link = json.loads(idx["annotations"]["ai.cua.image.variants"])["containerdisk"]
        index_ref, idx = link, manifest(link)
    for m in idx["manifests"]:
        if (m.get("platform") or {}).get("architecture") == arch:
            return index_ref, f"{repo}@{m['digest']}"
    raise SystemExit(f"{index_ref} has no {arch} child")


async def main() -> int:
    ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
    ap.add_argument("id")
    ap.add_argument("--variant", required=True, choices=["rootfs", "containerdisk"])
    ap.add_argument("--image", help="registry ref (default: the local build)")
    ap.add_argument("--disk", help="local disk.img for the containerdisk variant")
    ap.add_argument("--memory-mb", type=int, default=None)
    ap.add_argument("--cpu", type=int, default=None,
                    help="vCPUs (default 4, capped at the host's CPU count, e.g. 2 on private-repo CI runners)")
    ap.add_argument("--evidence", default=None)
    ap.add_argument("--arch", choices=["amd64", "arm64"],
                    help="local: run this arch (a foreign one is emulated; rootfs then needs --runc)")
    ap.add_argument("--runc", action="store_true",
                    help="rootfs under runc instead of gVisor (a foreign arch under emulation: "
                         "runsc cannot run an emulated amd64 image on an arm64 host)")
    args = ap.parse_args()
    if args.cpu is None:
        args.cpu = max(1, min(4, os.cpu_count() or 1))

    from cua_sandbox import Image, Sandbox

    b = benchlib.load(args.id)
    hooks = load_hooks(args.id)
    where = "local"
    evidence = Path(args.evidence or (benchlib.BENCH_ROOT / args.id / "evidence"))
    evidence.mkdir(parents=True, exist_ok=True)
    server = b.get("server") or {}
    ports = sorted(set((b.get("extra_ports") or {}).values()) | ({server["port"]} if server else set()))
    kind = "container" if args.variant == "rootfs" else "vm"
    # The host arch when the benchmark has it, else its only arch under
    # emulation (OSWorld is amd64-only).
    arch = args.arch or (host_arch() if host_arch() in b["arch"] else b["arch"][0])
    tag = f"{args.variant}-{where}-{arch}"
    work = Path(os.environ.get("CUA_BENCH_WORK", "~/.cache/cua-bench-images")).expanduser()

    child = None
    index_ref = args.image
    if args.image and args.arch and "@sha256:" in args.image:
        # The SDK runs the host arch of an index; for another arch, run that
        # arch's child by digest (for the disk: in the index its
        # ai.cua.image.variants names) and record which index it belongs to.
        index_ref, child = child_for(args.image, args.variant, args.arch)
        print(f"   {args.arch} {args.variant} child of {index_ref}: {child}", flush=True)
    if args.variant == "rootfs" or args.image:
        ref = child or args.image or f"cua-e2e-local/bench-{args.id}:docker-local-{arch}"
        # A private package: CUA_REGISTRY_USERNAME/PASSWORD
        # (RegistrySecret.from_env) go to the pull only.
        secret = None
        if os.environ.get("CUA_REGISTRY_PASSWORD"):
            from cua_sandbox import RegistrySecret
            secret = RegistrySecret.from_env()
        image = Image.from_registry(ref, os_type=b["os"], kind=kind, secret=secret)
    else:
        disk = Path(args.disk or work / args.id / "out" / arch / "disk.img").expanduser()
        ref = str(disk)
        image = Image.from_file(str(disk), os_type=b["os"], kind="vm")
    for p in ports:
        image = image.expose(p)
    mem = args.memory_mb or 4096
    if mem > 4096:
        ap.error("local smokes stay at or under 4096 MiB (AGENT_BRIEF memory rule)")
    report = Report()
    pinned = None
    print(f"== smoke {args.id} {args.variant} {where}: {ref}", flush=True)
    started = time.monotonic()
    extra = {"runtime": "runc"} if args.runc else {}
    async with Sandbox.ephemeral(image, local=True, name=f"cua-e2e-bench-{args.id}-{uuid.uuid4().hex[:6]}", **extra,
                                 server_port=server.get("port"), cpu=args.cpu, memory_mb=mem,
                                 time_to_start=1500, telemetry_enabled=False) as sb:
        report.add("ready", True, f"after {time.monotonic() - started:.0f}s")
        info = getattr(sb, "image_info", None)
        pinned = getattr(info, "pinned_ref", None)
        await run_checks(sb, b, hooks, report, evidence, tag)
    if child:
        # Evidence is keyed by the index the child was read from.
        pinned = index_ref
    out = {"id": args.id, "variant": args.variant, "where": where, "arch": arch, "image": ref,
           "child": child,
           "container_runtime": ("runc" if args.runc else "auto (gVisor when available)") if args.variant == "rootfs" else None,
           "pinned_ref": pinned, "doctor": next((c["status"] for c in report.checks if c["check"] == "doctor"), None),
           "seconds": round(time.monotonic() - started, 1), "ok": report.ok, "checks": report.checks,
           "date": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())}
    (evidence / f"{tag}.json").write_text(json.dumps(out, indent=2) + "\n")
    print(f"== {'PASS' if report.ok else 'FAIL'} {args.id} {tag} ({out['seconds']}s) -> {evidence}/{tag}.json")
    return 0 if report.ok else 1


if __name__ == "__main__":
    sys.exit(asyncio.run(main()))