Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
"""The unmodified `modal` client against `cua compat serve --provider modal`
on real Spaces (a Linux guest running as root): SDK-visible behaviour
recorded as observations and compared with the derived golden
(fixtures/modal-<version>-derived.json).
conformance.py > run.json
compare.py fixtures/modal-1.6.2-derived.json run.json
Each case returns a dict of observations (facts the client can see: output,
exit codes, exception classes, ordering); volatile values (ids, timings) are
reduced to booleans before they are recorded. Cases are tagged with the lanes
they run in: every case runs on real Spaces; `hermetic` ones also run over
mock Spaces (see hermetic_sdk.py for those).
"""
from __future__ import annotations
import hashlib
import json
import os
import socket
import ssl
import sys
import tempfile
import threading
import time
import traceback
import urllib.request
import modal
from modal.exception import (
AlreadyExistsError,
NotFoundError,
SandboxFilesystemNotFoundError,
SandboxTerminatedError,
SandboxTimeoutError,
)
from modal.types import FileType, FileWatchEventType
RESULTS: dict[str, dict] = {}
LANE = os.environ.get("CONFORMANCE_LANE", "space")
ONLY = set(filter(None, os.environ.get("ONLY", "").split(",")))
app = modal.App.lookup("conformance", create_if_missing=True)
BASE = os.environ.get("CONFORMANCE_IMAGE", "ghcr.io/trycua/linux:24.04")
IMG = modal.Image.from_registry(BASE)
CREATED: list = []
def new(*args, **kw):
kw.setdefault("image", IMG)
kw.setdefault("timeout", 600)
sb = modal.Sandbox.create(*args, app=app, **kw)
CREATED.append(sb)
return sb
class Skip(Exception):
"""The Space cannot show this case (the reason is recorded)."""
def case(name):
def deco(fn):
if ONLY and name not in ONLY:
return fn
t0 = time.monotonic()
try:
RESULTS[name] = {"ok": True, "observed": fn()}
print(f"PASS {name} ({time.monotonic() - t0:.1f}s)", file=sys.stderr, flush=True)
except Skip as e:
RESULTS[name] = {"ok": True, "skipped": str(e)}
print(f"SKIP {name}: {e}", file=sys.stderr, flush=True)
except BaseException as e: # noqa: BLE001
traceback.print_exc(file=sys.stderr)
RESULTS[name] = {"ok": False, "error": f"{type(e).__name__}: {e}"}
print(f"FAIL {name}: {type(e).__name__}: {e}", file=sys.stderr, flush=True)
return fn
return deco
def exc(fn):
"""The exception class name `fn` raises, or None."""
try:
fn()
except BaseException as e: # noqa: BLE001
return type(e).__name__
return None
def sha(b: bytes) -> str:
return hashlib.sha256(b).hexdigest()
sb = new("sleep", "100000")
@case("exec_streams_and_exit_code")
def _():
p = sb.exec("sh", "-c", "echo out; echo err >&2; exit 3")
return {"stdout": p.stdout.read(), "stderr": p.stderr.read(), "returncode": p.wait()}
@case("exec_default_user_cwd_env")
def _():
p = sb.exec(
"sh",
"-c",
"id -u; pwd; echo $HOME; echo $MODAL_SANDBOX_ID | cut -c1-3; echo ${MODAL_TASK_ID:+task}",
)
return {"stdout": p.stdout.read(), "returncode": p.wait()}
@case("sandbox_env_secrets_and_exec_env")
def _():
s = new("sleep", "100000", env={"A": "1"}, secrets=[modal.Secret.from_dict({"B": "2"})])
a = s.exec("sh", "-c", "echo $A $B").stdout.read()
b = s.exec("sh", "-c", "echo $A $B $C", env={"A": "x", "C": "3"}).stdout.read()
s.terminate()
return {"default": a, "override": b}
@case("workdir")
def _():
s = new("sleep", "100000", workdir="/tmp/wd")
a = s.exec("pwd").stdout.read()
b = s.exec("pwd", workdir="/etc").stdout.read()
s.terminate()
return {"sandbox_workdir": a, "exec_workdir": b}
@case("stdin_binary_roundtrip")
def _():
data = os.urandom(2 * 1024 * 1024)
p = sb.exec("sha256sum", text=False)
p.stdin.write(data)
p.stdin.write_eof()
p.stdin.drain()
out = p.stdout.read().decode().split()[0]
return {"sha_matches": out == sha(data), "returncode": p.wait()}
@case("stdout_binary_large")
def _():
p = sb.exec(
"sh",
"-c",
"head -c 3000000 /dev/urandom > /tmp/r.bin; sha256sum /tmp/r.bin | cut -d' ' -f1 >&2; cat /tmp/r.bin",
text=False,
)
data = p.stdout.read()
want = p.stderr.read().decode().strip()
return {"length": len(data), "sha_matches": sha(data) == want}
@case("invalid_utf8_in_text_mode")
def _():
p = sb.exec("sh", "-c", "printf '\\377\\n'")
return {"exception": exc(lambda: p.stdout.read())}
@case("killed_by_signal_is_128_plus")
def _():
p = sb.exec("sh", "-c", "kill -9 $$")
return {"returncode": p.wait()}
@case("exec_timeout_kills_the_process")
def _():
p = sb.exec("sleep", "31", timeout=2)
t = time.monotonic()
rc = p.wait()
took = time.monotonic() - t
time.sleep(6)
alive = sb.exec(
"sh", "-c", "pgrep -x sleep -a | grep -q ' 31$' && echo alive || echo gone"
).stdout.read()
return {"returncode": rc, "within_5s": took < 5, "process": alive}
@case("pty_exec")
def _():
p = sb.exec("sh", "-c", "tty; echo hi", pty=True)
out = p.stdout.read()
return {
"has_pts": "/dev/pts/" in out,
"has_hi": "hi" in out,
"stderr": p.stderr.read(),
"returncode": p.wait(),
}
@case("parallel_execs")
def _():
t = time.monotonic()
ps = [sb.exec("sh", "-c", "sleep 0.5; echo $$") for _ in range(6)]
outs = [p.stdout.read() for p in ps]
return {"distinct": len(set(outs)) == 6, "parallel": time.monotonic() - t < 3.0}
@case("filesystem_stat_symlink_list")
def _():
sb.exec(
"sh",
"-c",
"mkdir -p /tmp/fs && install -m 640 /dev/null /tmp/fs/f && echo -n abc > /tmp/fs/f && ln -sf /tmp/fs/f /tmp/fs/l && mkdir -p /tmp/fs/d",
).wait()
f = sb.filesystem.stat("/tmp/fs/f")
lnk = sb.filesystem.stat("/tmp/fs/l")
d = sb.filesystem.stat("/tmp/fs/d")
names = sorted(e.name for e in sb.filesystem.list_files("/tmp/fs"))
return {
"file": [f.type.value, f.size, f.permissions, f.owner, f.group, f.mode],
"link": [lnk.type.value, lnk.symlink_target],
"dir": [d.type.value, d.permissions],
"list": names,
}
@case("filesystem_large_file")
def _():
data = os.urandom(25 * 1024 * 1024)
sb.filesystem.write_bytes(data, "/tmp/fs/big.bin")
back = sb.filesystem.read_bytes("/tmp/fs/big.bin")
with tempfile.TemporaryDirectory() as d:
out = os.path.join(d, "x", "big.bin")
sb.filesystem.copy_to_local("/tmp/fs/big.bin", out)
same = open(out, "rb").read() == data
return {"roundtrip": back == data, "copy_to_local": same}
@case("filesystem_watch")
def _():
sb.exec("mkdir", "-p", "/tmp/w").wait()
events: list = []
def watch():
for ev in sb.filesystem.watch("/tmp/w", timeout=15):
events.append((ev.type.value, ev.paths))
th = threading.Thread(target=watch)
th.start()
time.sleep(2)
sb.exec(
"sh",
"-c",
"echo x > /tmp/w/new.txt; sleep 0.5; echo y >> /tmp/w/new.txt; sleep 0.5; rm /tmp/w/new.txt",
).wait()
th.join(timeout=30)
kinds = sorted({k for k, p in events if p == ["/tmp/w/new.txt"]})
return {"kinds": kinds}
@case("tunnels_encrypted_and_raw")
def _():
s = new(
"python3",
"-m",
"http.server",
"8080",
"--directory",
"/etc",
encrypted_ports=[8080],
unencrypted_ports=[8080],
readiness_probe=modal.Probe.with_tcp(8080),
)
s.wait_until_ready()
t = s.tunnels()[8080]
ctx = ssl.create_default_context(cafile=os.environ.get("SSL_CERT_FILE"))
body = urllib.request.urlopen(t.url, timeout=20, context=ctx).read().decode()
host, port = t.tcp_socket
c = socket.create_connection((host, port), timeout=10)
c.sendall(b"GET / HTTP/1.0\r\n\r\n")
raw = b""
while chunk := c.recv(65536):
raw += chunk
s.terminate()
return {"tls": "Directory listing" in body, "raw": b"Directory listing" in raw}
@case("snapshot_filesystem_boots_fresh")
def _():
s = new("sleep", "100000")
s.exec("sh", "-c", "echo v1 > /root/snap.txt; sync").wait()
img = s.snapshot_filesystem()
s2 = new("sleep", "100000", image=img)
content = s2.filesystem.read_text("/root/snap.txt")
s.terminate()
s2.terminate()
return {"content": content}
@case("images_python_and_pip")
def _():
img = (
modal.Image.debian_slim(python_version="3.11")
.pip_install("six==1.16.0")
.env({"IMG_ENV": "set"})
.workdir("/work")
)
s = new("sleep", "100000", image=img)
out = s.exec(
"python",
"-c",
"import six, sys, os; print(six.__version__, sys.version_info[:2], os.environ['IMG_ENV'], os.getcwd())",
).stdout.read()
s.terminate()
return {"out": out}
@case("image_console_scripts_are_on_path")
def _():
# pip's console scripts land next to the interpreter, on PATH (a python:* image's /usr/local/bin).
img = modal.Image.debian_slim(python_version="3.12").uv_pip_install("pyflakes")
s = new("sleep", "100000", image=img)
out = s.exec("sh", "-c", "command -v pyflakes && pyflakes --version").stdout.read()
s.terminate()
return {"on_path": "/usr/local/bin/pyflakes" in out}
@case("volume_mount_roundtrip")
def _():
vol = modal.Volume.from_name("conformance-vol", create_if_missing=True)
with vol.batch_upload(force=True) as b:
b.put_file(__file__, "/up/self.py")
s = new("sleep", "100000", volumes={"/data": vol})
listing = s.exec("ls", "/data/up").stdout.read()
s.exec("sh", "-c", "echo from-sandbox > /data/up/out.txt").wait()
s.terminate()
names = sorted(e.path for e in vol.listdir("up"))
got = b"".join(vol.read_file("up/out.txt")).decode()
vol.remove_file("up", recursive=True)
modal.Volume.objects.delete("conformance-vol")
return {"mounted_listing": listing, "listdir": names, "read": got}
@case("dict_and_queue")
def _():
d = modal.Dict.from_name("conformance-dict", create_if_missing=True)
d["a"] = 1
d.put("b", {"x": [1, 2]})
q = modal.Queue.from_name("conformance-queue", create_if_missing=True)
q.put_many([1, 2, 3])
r = {
"dict": [d["a"], d["b"], d.len(), d.get("zz", "none")],
"queue": [q.get(), q.get_many(5), q.len()],
}
modal.Dict.objects.delete("conformance-dict")
modal.Queue.objects.delete("conformance-queue")
return r
@case("entrypoint_stdout_streams_in_order")
def _():
s = new("sh", "-c", "for i in 1 2 3; do echo $i; sleep 0.5; done")
t = time.monotonic()
lines = []
first_at = None
for line in s.stdout:
if first_at is None:
first_at = time.monotonic() - t
lines.append(line)
s.wait()
return {
"lines": lines,
"streamed": first_at is not None and first_at < 1.4,
"returncode": s.returncode,
}
@case("sandbox_timeout_is_124_and_space_ends")
def _():
s = new("sleep", "100000", timeout=4)
cls = exc(s.wait)
return {"exception": cls, "returncode": s.returncode}
@case("exec_after_terminate")
def _():
s = new("sleep", "100000")
s.terminate()
return {"exec": exc(lambda: s.exec("echo", "x")), "poll": s.poll()}
@case("snapshot_directory_and_mount_image")
def _():
s = new("sleep", "100000")
s.exec("sh", "-c", "mkdir -p /work/data && echo snap > /work/data/a.txt").wait()
img = s.snapshot_directory("/work/data")
s2 = new("sleep", "100000")
s2.mount_image("/mnt/snap", img)
mounted = s2.exec("cat", "/mnt/snap/a.txt").stdout.read()
s2.unmount_image("/mnt/snap")
after = s2.exec("sh", "-c", "ls /mnt/snap 2>/dev/null | wc -l").stdout.read().strip()
s.terminate()
s2.terminate()
return {"mounted": mounted, "after_unmount": after}
@case("memory_snapshot_keeps_processes")
def _():
s = new("sleep", "100000", _experimental_enable_snapshot=True)
s.exec("sh", "-c", "echo mem > /tmp/mem.txt; (sleep 888 &) ; sync").wait()
snap = s._experimental_snapshot()
s2 = modal.Sandbox._experimental_from_snapshot(snap)
content = s2.exec("cat", "/tmp/mem.txt").stdout.read()
procs = s2.exec(
"sh", "-c", "pgrep -f 'sleep 888' >/dev/null && echo carried || echo fresh"
).stdout.read()
s.terminate()
s2.terminate()
return {"content": content, "processes": procs}
@case("volume_read_only_and_sub_path")
def _():
vol = modal.Volume.from_name("conformance-vol2", create_if_missing=True)
vol_b = modal.Volume.from_name("conformance-vol3", create_if_missing=True)
with vol.batch_upload(force=True) as b:
b.put_file(__file__, "/sub/self.py")
b.put_file(__file__, "/other.py")
with vol_b.batch_upload(force=True) as b:
b.put_file(__file__, "/ro.py")
s = new(
"sleep",
"100000",
volumes={"/ro": vol_b.read_only(), "/sub": vol.with_mount_options(sub_path="sub")},
)
sub = s.exec("ls", "/sub").stdout.read()
write = s.exec("sh", "-c", "echo x > /ro/new.txt; echo rc=$?").stdout.read()
s.exec("sh", "-c", "echo from-sandbox > /sub/out.txt").wait()
s.terminate() # commits what the Sandbox wrote
committed = b"".join(vol.read_file("sub/out.txt")).decode()
vol.remove_file("sub", recursive=True)
vol.remove_file("other.py")
modal.Volume.objects.delete("conformance-vol2")
modal.Volume.objects.delete("conformance-vol3")
return {"sub": sub, "read_only_write": "rc=0" not in write, "committed_at_exit": committed}
@case("cloud_bucket_mount_public_bucket")
def _():
# AWS Open Data: a public bucket read without credentials.
mount = modal.CloudBucketMount("noaa-ghcn-pds", read_only=True)
try:
s = new("sleep", "100000", volumes={"/bucket": mount})
except modal.exception.ConflictError as e:
if "fuse" in str(e).lower() or "bucket mount needs" in str(e):
raise Skip(
"this Space has no /dev/fuse (a container Space); bucket mounts need a VM Space"
)
raise
listing = s.exec("sh", "-c", "ls /bucket | head -3").stdout.read()
s.terminate()
return {"listed": bool(listing.strip())}
@case("readiness_probe_and_connect_token")
def _():
s = new(
"python3",
"-m",
"http.server",
"8080",
"--directory",
"/etc",
readiness_probe=modal.Probe.with_tcp(8080),
)
s.wait_until_ready(timeout=60)
t = s.create_connect_token(user_metadata={"u": 1}, port=8080)
ctx = ssl.create_default_context(cafile=os.environ.get("SSL_CERT_FILE"))
req = urllib.request.Request(t.url, headers={"Authorization": f"Bearer {t.token}"})
body = urllib.request.urlopen(req, timeout=20, context=ctx).read().decode()
try:
urllib.request.urlopen(t.url, timeout=20, context=ctx)
unauth = "reachable"
except Exception as e: # noqa: BLE001
unauth = getattr(e, "code", type(e).__name__)
s.terminate()
return {"with_token": "Directory listing" in body, "without_token": unauth}
@case("image_run_function")
def _():
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
import fn_app
img = modal.Image.debian_slim(python_version="3.12").run_function(
fn_app.write_marker, args=("/opt/built.txt", "built"), kwargs={"suffix": "!"}
)
with fn_app.app.run():
s = new("sleep", "100000", image=img)
content = s.exec("cat", "/opt/built.txt").stdout.read()
s.terminate()
return {"content": content}
@case("idle_timeout")
def _():
s = new("sleep", "100000", idle_timeout=3)
time.sleep(8)
return {"poll_is_set": s.poll() is not None}
@case("function_remote_and_map_in_a_space")
def _():
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
import fn_app
with fn_app.app.run():
out = {
"remote": fn_app.square.remote(7),
"map": list(fn_app.square.map([1, 2, 3])),
"secret": fn_app.secret_env.remote(),
"gen": list(fn_app.gen.remote_gen(3)),
}
vol = fn_app.writes_volume.remote("from-function")
out["volume"] = vol
return out
def main():
for s in CREATED:
try:
s.terminate()
except Exception: # noqa: BLE001
pass
print(
json.dumps(
{"lane": LANE, "modal": modal.__version__, "cases": RESULTS}, indent=1, sort_keys=True
)
)
sys.exit(0 if all(r["ok"] for r in RESULTS.values()) else 1)
main()