Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
# /// script
# requires-python = ">=3.11,<3.14"
# dependencies = ["cua-sandbox", "httpx>=0.27,<1"]
# ///
"""
Demo 3: OSWorld desktop on local QEMU with the Cua Driver MCP.
Boots an OSWorld disk, screenshots it through the OSWorld control server, then
drives it over the cua-driver MCP endpoint baked into the image.
Usage:
OSWORLD_QCOW2=/path/to/overlay.qcow2 uv run examples/python/3_osworld_local.py
See docs/content/docs/cua-bench/guides/benchmarks.mdx (OSWorld-Verified).
"""
import asyncio
import json
import os
import re
import sys
import time
from pathlib import Path
import httpx
from cua_sandbox import Image, Sandbox
sys.stdout.reconfigure(encoding="utf-8") # tool results contain emoji; keep Windows consoles happy
QCOW2 = os.environ.get("OSWORLD_QCOW2", str(Path.home() / "osworld-cua" / "osworld-cua.qcow2"))
MCP_PORT = 3000
image = (
Image.from_file(QCOW2, os_type="linux", agent_type="osworld")
.expose(5000)
.expose(MCP_PORT)
.expose(8080)
.expose(9222) # OSWorld server, MCP, VLC, Chrome CDP
)
class Mcp:
"""Minimal MCP Streamable-HTTP client over the sandbox's port-3000 service."""
def __init__(self, sb: Sandbox):
self.sb, self.session, self.next_id = sb, None, 1
async def _post(self, payload: dict) -> httpx.Response:
headers = {
"content-type": "application/json",
"accept": "application/json, text/event-stream",
}
if self.session:
headers["mcp-session-id"] = self.session
# Bare-metal QEMU forwards exposed guest ports to localhost.
host_port = self.sb.exposed_ports[MCP_PORT]
async with httpx.AsyncClient(timeout=120) as c:
return await c.post(f"http://127.0.0.1:{host_port}/mcp", headers=headers, json=payload)
async def call(self, method: str, params: dict | None = None) -> dict:
payload = {"jsonrpc": "2.0", "id": self.next_id, "method": method, "params": params or {}}
self.next_id += 1
r = await self._post(payload)
r.raise_for_status()
self.session = r.headers.get("mcp-session-id", self.session)
body = r.text
if r.headers.get("content-type", "").startswith("text/event-stream"):
body = [ln[5:] for ln in body.splitlines() if ln.startswith("data:")][-1]
msg = json.loads(body)
if "error" in msg:
raise RuntimeError(msg["error"])
return msg["result"]
async def notify(self, method: str) -> None:
await self._post({"jsonrpc": "2.0", "method": method})
async def tool(self, tool_name: str, **arguments) -> str:
result = await self.call("tools/call", {"name": tool_name, "arguments": arguments})
return "\n".join(c.get("text", "") for c in result.get("content", []))
async def main() -> None:
t0 = time.time()
async with Sandbox.ephemeral(
image, local=True, cpu=4, memory_mb=8192, time_to_start=1500
) as sb:
print(f"[{time.time() - t0:.0f}s] sandbox {sb.name} ready")
# 1. OSWorld control server (:5000) through the normal Sandbox surface.
print("screen:", await sb.get_dimensions())
r = await sb.shell.run("whoami; hostname; cua-driver --version", timeout=30)
print("shell:", r.stdout.strip().replace("\n", " | "), "rc", r.returncode)
Path("osworld.png").write_bytes(await sb.screenshot())
print("saved osworld.png")
# 2. cua-driver MCP (:3000/mcp) baked into the image.
mcp = Mcp(sb)
info = await mcp.call(
"initialize",
{
"protocolVersion": "2025-03-26",
"capabilities": {},
"clientInfo": {"name": "osworld-tutorial", "version": "0.1"},
},
)
await mcp.notify("notifications/initialized")
print("mcp server:", info["serverInfo"])
tools = [t["name"] for t in (await mcp.call("tools/list"))["tools"]]
print(f"mcp tools: {len(tools)} e.g. {tools[:8]}")
await mcp.tool("start_session", session="osworld-tutorial")
print("screen size via MCP:", await mcp.tool("get_screen_size"))
print("apps via MCP:", (await mcp.tool("list_apps"))[:300])
launched = await mcp.tool("launch_app", name="gnome-calculator", session="osworld-tutorial")
print("launch_app:", launched[:300])
await asyncio.sleep(3)
if pid := re.search(r"pid (\d+)", launched):
print(
"bring_to_front:",
await mcp.tool("bring_to_front", pid=int(pid.group(1)), session="osworld-tutorial"),
)
await asyncio.sleep(1)
print("windows via MCP:", (await mcp.tool("list_windows"))[:400])
await mcp.tool("end_session", session="osworld-tutorial")
Path("osworld-after.png").write_bytes(await sb.screenshot())
print("saved osworld-after.png")
print(f"[{time.time() - t0:.0f}s] VM stopped")
if __name__ == "__main__":
sys.exit(asyncio.run(main()))