Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
#!/usr/bin/env python3
"""Benchmark script for CUA-Bench worker throughput.
This script measures the performance of parallel workers with CBEnvWorkerClient,
tracking reset times, step times, and finish times.
Usage:
uv run python scripts/benchmark_workers.py --num_workers 16 --num_steps 10
"""
import argparse
import asyncio
import multiprocessing
import sys
import tempfile
import time
from multiprocessing import Manager
from pathlib import Path
# ANSI colors
BLUE = "\033[94m"
GREEN = "\033[92m"
YELLOW = "\033[93m"
RED = "\033[91m"
RESET = "\033[0m"
# Simple HTML for test tasks
SIMPLE_BUTTON_HTML = """
<div style="display:flex;flex-direction:column;align-items:center;justify-content:center;height:100vh;background:#f0f0f0;">
<h1 style="margin-bottom:20px;">Benchmark Task</h1>
<button
id="test-btn"
style="background:#3b82f6;color:white;padding:12px 24px;border:none;border-radius:8px;cursor:pointer;"
onclick="window.__clicked = true; window.__score = 1.0;"
>
Click Me
</button>
</div>
<script>
window.__clicked = false;
window.__score = 0.0;
</script>
"""
def create_temp_task(tmp_dir: Path) -> Path:
"""Create a temporary test task directory."""
task_dir = tmp_dir / "benchmark-task"
task_dir.mkdir(exist_ok=True)
(task_dir / "main.py").write_text(f'''
import cua_bench as cb
HTML = """{SIMPLE_BUTTON_HTML}"""
pid = None
@cb.tasks_config(split="train")
def get_tasks():
return [cb.Task(
description="Click the button",
computer={{"provider": "simulated", "setup_config": {{"width": 800, "height": 600}}}}
) for _ in range(20)]
@cb.setup_task(split="train")
async def setup(task, session):
global pid
pid = await session.launch_window(html=HTML, title="Benchmark", width=400, height=300)
@cb.evaluate_task(split="train")
async def evaluate(task, session):
global pid
clicked = await session.execute_javascript(pid, "window.__clicked")
return [1.0 if clicked else 0.0]
''')
return task_dir
def run_worker(
worker_id: int,
server_url: str,
task_path: str,
num_steps: int,
results_dict: dict,
):
"""Run a single worker process."""
from cua_bench_rl.workers import CBEnvWorkerClient
print(f"{BLUE}Worker {worker_id} starting...{RESET}")
# Create task configs for this worker
task_configs = [
{"env_path": task_path, "task_index": (worker_id + i) % 10, "split": "train"}
for i in range(10)
]
# Create client with env_config dict
env_config = {
"server_url": server_url,
"task_configs": task_configs,
"max_step": 100,
"max_hist": 10,
"timeout": 300,
}
client = CBEnvWorkerClient(env_config)
# Initialize local counters
worker_reset_time = 0.0
worker_reset_count = 0
worker_step_time = 0.0
worker_step_count = 0
worker_finish_time = 0.0
worker_finish_count = 0
try:
# Reset environment (picks random task from task_configs)
start_time = time.time()
env_ret, meta_info = client.reset()
reset_time = time.time() - start_time
worker_reset_time += reset_time
worker_reset_count += 1
assert env_ret is not None
assert meta_info is not None
print(f"{YELLOW}Worker {worker_id} reset in {reset_time:.2f}s{RESET}")
# Run steps
for i in range(num_steps):
start_time = time.time()
# Action is now a string with action tags
action = "<|action_start|>click(500,500)<|action_end|>"
env_ret, meta_info = client.step(action)
step_time = time.time() - start_time
worker_step_time += step_time
worker_step_count += 1
assert env_ret is not None
assert meta_info is not None
if env_ret.get("done"):
print(f"{YELLOW}Worker {worker_id} episode done at step {i+1}{RESET}")
# Reset for next episode
start_time = time.time()
env_ret, meta_info = client.reset()
reset_time = time.time() - start_time
worker_reset_time += reset_time
worker_reset_count += 1
if (i + 1) % 5 == 0:
print(
f"{YELLOW}Worker {worker_id} step {i+1}/{num_steps} "
f"in {step_time:.2f}s{RESET}"
)
# Finish with done action
start_time = time.time()
action = "<|action_start|>done()<|action_end|>"
env_ret, meta_info = client.step(action)
finish_time = time.time() - start_time
worker_finish_time += finish_time
worker_finish_count += 1
assert env_ret is not None
assert env_ret.get("done")
print(f"{GREEN}Worker {worker_id} finish in {finish_time:.2f}s{RESET}")
# Store results
results_dict[worker_id] = {
"reset_time": worker_reset_time,
"reset_count": worker_reset_count,
"step_time": worker_step_time,
"step_count": worker_step_count,
"finish_time": worker_finish_time,
"finish_count": worker_finish_count,
}
except Exception as e:
print(f"{RED}Worker {worker_id} error: {e}{RESET}")
results_dict[worker_id] = {
"error": str(e),
"reset_time": worker_reset_time,
"reset_count": worker_reset_count,
"step_time": worker_step_time,
"step_count": worker_step_count,
"finish_time": worker_finish_time,
"finish_count": worker_finish_count,
}
def run_benchmark(
num_workers: int = 16,
num_steps: int = 10,
task_path: str | None = None,
):
"""Run the worker benchmark."""
from cua_bench_rl.workers import cleanup_workers, create_workers
async def _run():
print(f"\n{GREEN}=== CUA-Bench Worker Benchmark ==={RESET}")
print(f"{BLUE}Workers: {num_workers}{RESET}")
print(f"{BLUE}Steps per worker: {num_steps}{RESET}")
# Create temp task if no task_path provided
tmp_dir = None
if task_path is None:
tmp_dir = tempfile.mkdtemp(prefix="cua-bench-benchmark-")
actual_task_path = str(create_temp_task(Path(tmp_dir)))
print(f"{BLUE}Using temp task: {actual_task_path}{RESET}")
else:
actual_task_path = task_path
print(f"{BLUE}Task path: {actual_task_path}{RESET}")
try:
# Start worker servers
print(f"\n{BLUE}Starting {num_workers} worker servers...{RESET}")
start_time = time.time()
workers = await create_workers(
n_workers=num_workers,
allowed_ips=["127.0.0.1"],
startup_timeout=120.0,
)
worker_startup_time = time.time() - start_time
print(f"{GREEN}Worker servers started in {worker_startup_time:.2f}s{RESET}")
try:
# Create manager for shared results
manager = Manager()
results_dict = manager.dict()
# Start worker processes
print(f"\n{BLUE}Starting {num_workers} worker clients...{RESET}")
processes = []
for i, worker in enumerate(workers):
p = multiprocessing.Process(
target=run_worker,
args=(i, worker.api_url, actual_task_path, num_steps, results_dict),
)
processes.append(p)
p.start()
time.sleep(0.1) # Small delay between starts
# Wait for all processes
for p in processes:
p.join()
# Aggregate results
reset_time_total = 0.0
reset_count = 0
step_time_total = 0.0
step_count = 0
finish_time_total = 0.0
finish_count = 0
errors = []
for worker_id, result in results_dict.items():
if "error" in result:
errors.append(f"Worker {worker_id}: {result['error']}")
reset_time_total += result["reset_time"]
reset_count += result["reset_count"]
step_time_total += result["step_time"]
step_count += result["step_count"]
finish_time_total += result["finish_time"]
finish_count += result["finish_count"]
# Print summary
print(f"\n{GREEN}=== Benchmark Results ==={RESET}")
print(f"{BLUE}Workers: {num_workers}{RESET}")
print(f"{BLUE}Steps per worker: {num_steps}{RESET}")
print(f"{BLUE}Total resets: {reset_count}{RESET}")
print(f"{BLUE}Total steps: {step_count}{RESET}")
print(f"{BLUE}Total finishes: {finish_count}{RESET}")
if reset_count > 0:
print(f"{BLUE}Average reset time: {reset_time_total / reset_count:.2f}s{RESET}")
if step_count > 0:
print(f"{BLUE}Average step time: {step_time_total / step_count:.2f}s{RESET}")
per_worker_throughput = step_count / step_time_total
total_throughput = per_worker_throughput * num_workers
print(
f"{BLUE}Step throughput (per worker): {per_worker_throughput:.2f} steps/sec{RESET}"
)
print(f"{BLUE}Step throughput (total): {total_throughput:.2f} steps/sec{RESET}")
if finish_count > 0:
print(
f"{BLUE}Average finish time: {finish_time_total / finish_count:.2f}s{RESET}"
)
if errors:
print(f"\n{RED}Errors ({len(errors)}):{RESET}")
for err in errors:
print(f"{RED} {err}{RESET}")
finally:
print(f"\n{BLUE}Cleaning up workers...{RESET}")
await cleanup_workers(workers)
finally:
# Clean up temp directory
if tmp_dir is not None:
import shutil
shutil.rmtree(tmp_dir, ignore_errors=True)
asyncio.run(_run())
def main():
parser = argparse.ArgumentParser(description="Benchmark CUA-Bench workers")
parser.add_argument(
"--num_workers",
type=int,
default=16,
help="Number of parallel workers (default: 16)",
)
parser.add_argument(
"--num_steps",
type=int,
default=10,
help="Number of steps per worker (default: 10)",
)
parser.add_argument(
"--task_path",
type=str,
default=None,
help="Path to task directory (default: creates temp task)",
)
args = parser.parse_args()
run_benchmark(
num_workers=args.num_workers,
num_steps=args.num_steps,
task_path=args.task_path,
)
return 0
if __name__ == "__main__":
sys.exit(main())