Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
updated metrics
Dillon DuPont8aef7b8b1a99
diff --git a/libs/python/agent/benchmarks/README.md b/libs/python/agent/benchmarks/README.mdindex 033f0157f7..225fc30b74 100644--- a/libs/python/agent/benchmarks/README.md+++ b/libs/python/agent/benchmarks/README.md@@ -98,19 +98,13 @@ def get_available_models() -> List[Union[str, ModelProtocol]]:### 1. Configure ModelsEdit `utils.py` to specify which models you want to test in `get_available_models()`.-### 2. Set Sample Count-Edit the benchmark script to change the number of samples:-```python-max_samples = 50 # Set to None to evaluate on full dataset-```--### 3. Run Benchmark+### 2. Run Benchmark```bash# ScreenSpot-v2 benchmark-python ss-v2.py+python ss-v2.py --samples 50# ScreenSpot-Pro benchmark-python ss-pro.py+python ss-pro.py --samples 50# Interactive testingpython interactive.py@@ -121,14 +115,9 @@ python interactive.py### Console Output```Model Results:- Accuracy: 85.50%- Correct: 171/200- Errors: 5- Error Rate: 2.50%- Avg Time: 1.23s- Time Range: 0.89s - 2.45s- VRAM Max: 4.5GB- VRAM Avg: 3.4GB+ Accuracy: 85.50% (171/200)+ Avg Time: 1.23s (0.89s - 2.45s)+ VRAM Usage: 4.5GB (max) / 3.4GB (avg)```### Generated Files@@ -139,20 +128,10 @@ Model Results:## Metrics Tracked- **Accuracy**: Percentage of clicks within bounding boxes-- **Error Rate**: Percentage of failed predictions- **Timing**: Average, min, max prediction times- **VRAM Usage**: Maximum and average GPU memory usage- **Per-sample Results**: Detailed breakdown for debugging-## Requirements--- Python 3.8+-- PyTorch (for VRAM tracking)-- PIL/Pillow (for image processing)-- datasets (for HuggingFace datasets)-- tqdm (for progress bars)-- Computer Agent SDK-## ArchitectureThe benchmark system is designed for:@@ -160,13 +139,6 @@ The benchmark system is designed for:- **Flexibility**: Works with any iterator of dicts with `image`, `bbox`, `instruction` keys- **Performance**: VRAM tracking and timing analysis- **Visualization**: Automatic generation of prediction visualizations-- **No Exception Handling**: Fails fast to surface real issues--## Results Table--| Model | Dataset | Accuracy | Error Rate | Avg Time | VRAM Max | VRAM Avg |-|-------|---------|----------|------------|----------|----------|----------|-| (coming soon) | | | | | | |## Contributingdiff --git a/libs/python/agent/benchmarks/ss-pro.py b/libs/python/agent/benchmarks/ss-pro.pyindex e1e54a1dce..80e5e72f8e 100644--- a/libs/python/agent/benchmarks/ss-pro.py+++ b/libs/python/agent/benchmarks/ss-pro.py@@ -6,7 +6,10 @@ Evaluates models on the ScreenSpot-Pro dataset for click prediction accuracy.Supports both ComputerAgent model strings and custom model classes."""+import argparseimport asyncio+import random+import statisticsimport timefrom typing import Optional@@ -18,7 +21,8 @@ from utils import (is_click_in_bbox,save_results_to_markdown,save_visualizations,- get_available_models+ get_available_models,+ get_gpu_memory)@@ -87,6 +91,7 @@ async def evaluate_model(model_wrapper: ModelWrapper, dataset, max_samples: Opti# Calculate timing statisticssuccessful_times = [r['prediction_time'] for r in results if not r['failed']]avg_prediction_time = sum(successful_times) / len(successful_times) if successful_times else 0.0+ median_prediction_time = statistics.median(successful_times) if successful_times else 0.0min_prediction_time = min(successful_times) if successful_times else 0.0max_prediction_time = max(successful_times) if successful_times else 0.0@@ -101,6 +106,7 @@ async def evaluate_model(model_wrapper: ModelWrapper, dataset, max_samples: Opti'accuracy': accuracy,'failure_rate': error_rate,'avg_prediction_time': avg_prediction_time,+ 'median_prediction_time': median_prediction_time,'min_prediction_time': min_prediction_time,'max_prediction_time': max_prediction_time,'vram_max_mb': vram_stats['max_mb'],@@ -113,6 +119,17 @@ async def main():"""Main function to run the benchmark."""+ # Parse command line arguments+ parser = argparse.ArgumentParser(description='ScreenSpot-Pro Benchmark Script')+ parser.add_argument('--samples', type=int, default=300,+ help='Number of samples to evaluate (default: 300)')+ parser.add_argument('--seed', type=int, default=42,+ help='Random seed for shuffling (default: 42)')+ args = parser.parse_args()++ # Set random seed+ random.seed(args.seed)+# Load datasetprint("Loading ScreenSpot-Pro dataset...")ds = load_dataset("lmms-lab/ScreenSpot-Pro")@@ -121,11 +138,15 @@ async def main():dataset_list = list(dataset)print(f"Dataset loaded: {len(dataset_list)} samples")+ # Shuffle dataset with seed+ random.shuffle(dataset_list)+ print(f"Dataset shuffled with seed {args.seed}")+# Get available modelsmodels = get_available_models()# Evaluation settings- max_samples = 300 # Set to None to evaluate on full dataset+ max_samples = args.samples # Use command line argument# Run evaluationsall_results = []@@ -142,9 +163,15 @@ async def main():print(f" Errors: {result['failed_predictions']}")print(f" Error Rate: {result['failure_rate']*100:.2f}%")print(f" Avg Time: {result['avg_prediction_time']:.2f}s")+ print(f" Median Time: {result['median_prediction_time']:.2f}s")print(f" Time Range: {result['min_prediction_time']:.2f}s - {result['max_prediction_time']:.2f}s")print(f" VRAM Max: {result['vram_max_mb']:.1f}MB")print(f" VRAM Avg: {result['vram_avg_mb']:.1f}MB")++ # Print GPU memory info+ gpu_memory = get_gpu_memory()+ if gpu_memory and gpu_memory[0] > 0:+ print(f" GPU Free Memory: {gpu_memory[0]:.1f}MB")# Save resultsif all_results:diff --git a/libs/python/agent/benchmarks/ss-v2.py b/libs/python/agent/benchmarks/ss-v2.pyindex 919a10013f..dab1d4b1aa 100644--- a/libs/python/agent/benchmarks/ss-v2.py+++ b/libs/python/agent/benchmarks/ss-v2.py@@ -1,12 +1,15 @@#!/usr/bin/env python3"""-ScreenSpot-Pro Benchmark Script+ScreenSpot-v2 Benchmark Script-Evaluates models on the ScreenSpot-Pro dataset for click prediction accuracy.+Evaluates models on the ScreenSpot-v2 dataset for click prediction accuracy.Supports both ComputerAgent model strings and custom model classes."""+import argparseimport asyncio+import random+import statisticsimport timefrom typing import Optional@@ -18,7 +21,8 @@ from utils import (is_click_in_bbox,save_results_to_markdown,save_visualizations,- get_available_models+ get_available_models,+ get_gpu_memory)@@ -93,6 +97,7 @@ async def evaluate_model(model_wrapper: ModelWrapper, samples, max_samples: Opti# Calculate timing statisticssuccessful_times = [r['prediction_time'] for r in results if not r['failed']]avg_prediction_time = sum(successful_times) / len(successful_times) if successful_times else 0.0+ median_prediction_time = statistics.median(successful_times) if successful_times else 0.0min_prediction_time = min(successful_times) if successful_times else 0.0max_prediction_time = max(successful_times) if successful_times else 0.0@@ -107,6 +112,7 @@ async def evaluate_model(model_wrapper: ModelWrapper, samples, max_samples: Opti'accuracy': accuracy,'failure_rate': error_rate,'avg_prediction_time': avg_prediction_time,+ 'median_prediction_time': median_prediction_time,'min_prediction_time': min_prediction_time,'max_prediction_time': max_prediction_time,'vram_max_mb': vram_stats['max_mb'],@@ -119,6 +125,17 @@ async def main():"""Main function to run the benchmark."""+ # Parse command line arguments+ parser = argparse.ArgumentParser(description='ScreenSpot-v2 Benchmark Script')+ parser.add_argument('--samples', type=int, default=500,+ help='Number of samples to evaluate (default: 500)')+ parser.add_argument('--seed', type=int, default=42,+ help='Random seed for shuffling (default: 42)')+ args = parser.parse_args()++ # Set random seed+ random.seed(args.seed)+# Load datasetprint("Loading ScreenSpot-v2 dataset...")ds = load_dataset("lmms-lab/ScreenSpot-v2")@@ -141,11 +158,15 @@ async def main():})print(f"Dataset loaded: {len(samples)} samples")+ # Shuffle samples with seed+ random.shuffle(samples)+ print(f"Samples shuffled with seed {args.seed}")+# Get available modelsmodels = get_available_models()# Evaluation settings- max_samples = 500 # Set to None to evaluate on full dataset+ max_samples = args.samples # Use command line argument# Run evaluationsall_results = []@@ -162,9 +183,15 @@ async def main():print(f" Errors: {result['failed_predictions']}")print(f" Error Rate: {result['failure_rate']*100:.2f}%")print(f" Avg Time: {result['avg_prediction_time']:.2f}s")+ print(f" Median Time: {result['median_prediction_time']:.2f}s")print(f" Time Range: {result['min_prediction_time']:.2f}s - {result['max_prediction_time']:.2f}s")print(f" VRAM Max: {result['vram_max_mb']:.1f}MB")print(f" VRAM Avg: {result['vram_avg_mb']:.1f}MB")++ # Print GPU memory info+ gpu_memory = get_gpu_memory()+ if gpu_memory and gpu_memory[0] > 0:+ print(f" GPU Free Memory: {gpu_memory[0]:.1f}MB")# Save resultsif all_results:diff --git a/libs/python/agent/benchmarks/utils.py b/libs/python/agent/benchmarks/utils.pyindex 099499a54a..7a3b70a3cc 100644--- a/libs/python/agent/benchmarks/utils.py+++ b/libs/python/agent/benchmarks/utils.py@@ -7,6 +7,8 @@ import asyncioimport base64import osimport sys+import subprocess as sp+import statisticsfrom datetime import datetimefrom io import BytesIOfrom typing import List, Union, Tuple, Optional@@ -22,6 +24,28 @@ from agent.agent import ComputerAgentfrom models import GTA1Modelfrom models.base import ModelProtocol+def get_gpu_memory() -> List[int]:+ """+ Get GPU memory usage using nvidia-smi.++ Returns:+ List of free memory values in MB for each GPU+ """+ try:+ command = "nvidia-smi --query-gpu=memory.free --format=csv"+ memory_free_info = sp.check_output(command.split()).decode('ascii').split('\n')[:-1][1:]+ memory_free_values = [int(x.split()[0]) for i, x in enumerate(memory_free_info)]+ return memory_free_values+ except (sp.CalledProcessError, FileNotFoundError, IndexError):+ # Fallback to torch if nvidia-smi is not available+ if torch.cuda.is_available():+ device = torch.cuda.current_device()+ total = torch.cuda.get_device_properties(device).total_memory / 1024 / 1024+ reserved = torch.cuda.memory_reserved(device) / 1024 / 1024+ return [int(total - reserved)]+ return [0]++def get_vram_usage() -> dict:"""Get current VRAM usage statistics.@@ -61,11 +85,11 @@ def get_available_models() -> List[Union[str, ModelProtocol]]:models = [# === ComputerAgent model strings ===f"{local_provider}HelloKKMe/GTA1-7B",- f"{local_provider}HelloKKMe/GTA1-32B",+ # f"{local_provider}HelloKKMe/GTA1-32B",# === Reference model classes ===GTA1Model("HelloKKMe/GTA1-7B"),- GTA1Model("HelloKKMe/GTA1-32B"),+ # GTA1Model("HelloKKMe/GTA1-32B"),]return models@@ -203,8 +227,8 @@ def save_results_to_markdown(all_results: List[dict],output_file: str = "screens# Summary tablef.write("## Summary\n\n")- f.write("| Model | Total Samples | Correct | Errors | Accuracy | Error Rate | Avg Time (s) | Time Range (s) | VRAM Max (GB) | VRAM Avg (GB) |\n")- f.write("|-------|---------------|---------|--------|----------|------------|--------------|----------------|---------------|---------------|\n")+ f.write("| Model | Total Samples | Correct | Errors | Accuracy | Error Rate | Avg Time (s) | Median Time (s) | Time Range (s) | VRAM Max (GB) | VRAM Avg (GB) |\n")+ f.write("|-------|---------------|---------|--------|----------|------------|--------------|-----------------|----------------|---------------|---------------|\n")for result in all_results:model_name = result['model_name']@@ -214,13 +238,14 @@ def save_results_to_markdown(all_results: List[dict],output_file: str = "screensaccuracy = result['accuracy'] * 100error_rate = result['failure_rate'] * 100avg_time = result.get('avg_prediction_time', 0.0)+ median_time = result.get('median_prediction_time', 0.0)min_time = result.get('min_prediction_time', 0.0)max_time = result.get('max_prediction_time', 0.0)time_range = f"{min_time:.2f} - {max_time:.2f}"vram_max = result.get('vram_max_mb', 0.0) / 1024vram_avg = result.get('vram_avg_mb', 0.0) / 1024- f.write(f"| {model_name} | {total} | {correct} | {errors} | {accuracy:.2f}% | {error_rate:.2f}% | {avg_time:.2f} | {time_range} | {vram_max:.1f} | {vram_avg:.1f} |\n")+ f.write(f"| {model_name} | {total} | {correct} | {errors} | {accuracy:.2f}% | {error_rate:.2f}% | {avg_time:.2f} | {median_time:.2f} | {time_range} | {vram_max:.1f} | {vram_avg:.1f} |\n")# Detailed results for each modelfor result in all_results: