Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
added sandbox provider
Dillon DuPontf12be458e2f3
diff --git a/libs/computer/computer/computer.py b/libs/computer/computer/computer.pyindex 1d5d261e2c..8596bc8bf4 100644--- a/libs/computer/computer/computer.py+++ b/libs/computer/computer/computer.py@@ -109,7 +109,7 @@ class Computer:# Windows Sandbox always uses ephemeral storageif self.provider_type == VMProviderType.WINSANDBOX:- if not ephemeral:+ if not ephemeral and storage != None and storage != "ephemeral":self.logger.warning("Windows Sandbox storage is always ephemeral. Setting ephemeral=True.")self.ephemeral = Trueself.storage = "ephemeral"@@ -400,7 +400,6 @@ class Computer:# Wait for VM to be ready with a valid IP addressself.logger.info("Waiting for VM to be ready with a valid IP address...")try:- # Increased values for Lumier provider which needs more time for initial setupif self.provider_type == VMProviderType.LUMIER:max_retries = 60 # Increased for Lumier VM startup which takes longerretry_delay = 3 # 3 seconds between retries for Lumier@@ -530,7 +529,7 @@ class Computer:return# @property- async def get_ip(self, max_retries: int = 15, retry_delay: int = 2) -> str:+ async def get_ip(self, max_retries: int = 15, retry_delay: int = 3) -> str:"""Get the IP address of the VM or localhost if using host computer server.This method delegates to the provider's get_ip method, which waits indefinitelydiff --git a/libs/computer/computer/providers/winsandbox/logon_script.bat b/libs/computer/computer/providers/winsandbox/logon_script.batdeleted file mode 100644index f5bd58c1bd..0000000000--- a/libs/computer/computer/providers/winsandbox/logon_script.bat+++ /dev/null@@ -1,10 +0,0 @@-@echo off-REM Logon script for Windows Sandbox CUA Computer provider-REM This script runs when the sandbox starts--REM Open explorer to show the desktop-explorer .--REM TODO: Install CUA computer server-REM pip install cua-computer-server-REM python -m computer_server.main --wsdiff --git a/libs/computer/computer/providers/winsandbox/provider.py b/libs/computer/computer/providers/winsandbox/provider.pyindex 29c52330c8..53d0148e95 100644--- a/libs/computer/computer/providers/winsandbox/provider.py+++ b/libs/computer/computer/providers/winsandbox/provider.py@@ -121,18 +121,73 @@ class WinSandboxProvider(BaseVMProvider):# Check if sandbox is still runningtry:- # For Windows Sandbox, we assume it's running if it's in our active list- # and hasn't been terminated# Try to ping the sandbox to see if it's responsivetry:- # Simple test to see if RPyC connection is alivesandbox.rpyc.modules.os.getcwd()- status = "running"- # Windows Sandbox typically uses localhost for RPyC connections- ip_address = "127.0.0.1"+ sandbox_responsive = Trueexcept Exception:+ sandbox_responsive = False++ if not sandbox_responsive:+ return {+ "name": name,+ "status": "starting",+ "ip_address": None,+ "storage": "ephemeral",+ "memory_mb": self.memory_mb,+ "networking": self.networking+ }++ # Check for computer server address file+ server_address_file = r"C:\Users\WDAGUtilityAccount\Desktop\shared_windows_sandbox_dir\server_address"++ try:+ # Check if the server address file exists+ file_exists = sandbox.rpyc.modules.os.path.exists(server_address_file)++ if file_exists:+ # Read the server address file+ with sandbox.rpyc.builtin.open(server_address_file, 'r') as f:+ server_address = f.read().strip()++ if server_address and ':' in server_address:+ # Parse IP:port from the file+ ip_address, port = server_address.split(':', 1)++ # Verify the server is actually responding+ try:+ import socket+ sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)+ sock.settimeout(3)+ result = sock.connect_ex((ip_address, int(port)))+ sock.close()++ if result == 0:+ # Server is responding+ status = "running"+ self.logger.debug(f"Computer server found at {ip_address}:{port}")+ else:+ # Server file exists but not responding+ status = "starting"+ ip_address = None+ except Exception as e:+ self.logger.debug(f"Error checking server connectivity: {e}")+ status = "starting"+ ip_address = None+ else:+ # File exists but doesn't contain valid address+ status = "starting"+ ip_address = None+ else:+ # Server address file doesn't exist yet+ status = "starting"+ ip_address = None++ except Exception as e:+ self.logger.debug(f"Error checking server address file: {e}")status = "starting"ip_address = None+except Exception as e:self.logger.error(f"Error checking sandbox status: {e}")status = "error"@@ -187,9 +242,6 @@ class WinSandboxProvider(BaseVMProvider):networking = run_opts.get("networking", self.networking)- # Get the logon script path- script_path = os.path.join(os.path.dirname(__file__), "logon_script.bat")-# Create folder mappers if shared directories are specifiedfolder_mappers = []shared_directories = run_opts.get("shared_directories", [])@@ -209,11 +261,10 @@ class WinSandboxProvider(BaseVMProvider):if folder_mappers:self.logger.info(f"Shared directories: {len(folder_mappers)}")- # Create the sandbox+ # Create the sandbox without logon scriptsandbox = winsandbox.new_sandbox(memory_mb=str(memory_mb),networking=networking,- logon_script=f'cmd /c "{script_path}"',folder_mappers=folder_mappers)@@ -222,6 +273,9 @@ class WinSandboxProvider(BaseVMProvider):self.logger.info(f"Windows Sandbox {name} created successfully")+ # Setup the computer server in the sandbox+ await self._setup_computer_server(sandbox, name)+return {"success": True,"name": name,@@ -233,6 +287,9 @@ class WinSandboxProvider(BaseVMProvider):except Exception as e:self.logger.error(f"Failed to create Windows Sandbox {name}: {e}")+ # stack trace+ import traceback+ self.logger.error(f"Stack trace: {traceback.format_exc()}")return {"success": False,"error": f"Failed to create sandbox: {str(e)}"@@ -348,3 +405,66 @@ class WinSandboxProvider(BaseVMProvider):# Add progress log every 10 attemptsif total_attempts % 10 == 0:self.logger.info(f"Still waiting for Windows Sandbox {name} IP after {total_attempts} attempts...")++ async def _setup_computer_server(self, sandbox, name: str, visible: bool = False):+ """Setup the computer server in the Windows Sandbox using RPyC.++ Args:+ sandbox: The Windows Sandbox instance+ name: Name of the sandbox+ visible: Whether the opened process should be visible (default: False)+ """+ try:+ self.logger.info(f"Setting up computer server in sandbox {name}...")+ print(f"Setting up computer server in sandbox {name}...")++ # Read the PowerShell setup script+ script_path = os.path.join(os.path.dirname(__file__), "setup_script.ps1")+ with open(script_path, 'r', encoding='utf-8') as f:+ setup_script_content = f.read()++ # Write the setup script to the sandbox using RPyC+ script_dest_path = r"C:\Users\WDAGUtilityAccount\setup_cua.ps1"++ print(f"Writing setup script to {script_dest_path}")+ with sandbox.rpyc.builtin.open(script_dest_path, 'w') as f:+ f.write(setup_script_content)++ # Execute the PowerShell script in the background+ print("Executing setup script in sandbox...")++ # Use subprocess to run PowerShell script+ import subprocess+ powershell_cmd = [+ "powershell.exe",+ "-ExecutionPolicy", "Bypass",+ "-NoExit", # Keep window open after script completes+ "-File", script_dest_path+ ]++ # Set creation flags based on visibility preference+ if visible:+ # CREATE_NEW_CONSOLE - creates a new console window (visible)+ creation_flags = 0x00000010+ else:+ # DETACHED_PROCESS - runs in background (not visible)+ creation_flags = 0x00000008++ # Start the process using RPyC+ process = sandbox.rpyc.modules.subprocess.Popen(+ powershell_cmd,+ creationflags=creation_flags,+ shell=False+ )++ # Sleep for 30 seconds+ await asyncio.sleep(30)++ ip = await self.get_ip(name)+ print(f"Sandbox IP: {ip}")+ print(f"Setup script started in background in sandbox {name} with PID: {process.pid}")++ except Exception as e:+ self.logger.error(f"Failed to setup computer server in sandbox {name}: {e}")+ import traceback+ self.logger.error(f"Stack trace: {traceback.format_exc()}")diff --git a/libs/computer/computer/providers/winsandbox/setup_script.ps1 b/libs/computer/computer/providers/winsandbox/setup_script.ps1new file mode 100644index 0000000000..73074764f0--- /dev/null+++ b/libs/computer/computer/providers/winsandbox/setup_script.ps1@@ -0,0 +1,124 @@+# Setup script for Windows Sandbox CUA Computer provider+# This script runs when the sandbox starts++Write-Host "Starting CUA Computer setup in Windows Sandbox..."++# Function to find the mapped Python installation from pywinsandbox+function Find-MappedPython {+ Write-Host "Looking for mapped Python installation from pywinsandbox..."++ # pywinsandbox maps the host Python installation to the sandbox+ # Look for mapped shared folders on the desktop (common pywinsandbox pattern)+ $desktopPath = "C:\Users\WDAGUtilityAccount\Desktop"+ $sharedFolders = Get-ChildItem -Path $desktopPath -Directory -ErrorAction SilentlyContinue++ foreach ($folder in $sharedFolders) {+ # Look for Python executables in shared folders+ $pythonPaths = @(+ "$($folder.FullName)\python.exe",+ "$($folder.FullName)\Scripts\python.exe",+ "$($folder.FullName)\bin\python.exe"+ )++ foreach ($pythonPath in $pythonPaths) {+ if (Test-Path $pythonPath) {+ try {+ $version = & $pythonPath --version 2>&1+ if ($version -match "Python") {+ Write-Host "Found mapped Python: $pythonPath - $version"+ return $pythonPath+ }+ } catch {+ continue+ }+ }+ }++ # Also check subdirectories that might contain Python+ $subDirs = Get-ChildItem -Path $folder.FullName -Directory -ErrorAction SilentlyContinue+ foreach ($subDir in $subDirs) {+ $pythonPath = "$($subDir.FullName)\python.exe"+ if (Test-Path $pythonPath) {+ try {+ $version = & $pythonPath --version 2>&1+ if ($version -match "Python") {+ Write-Host "Found mapped Python in subdirectory: $pythonPath - $version"+ return $pythonPath+ }+ } catch {+ continue+ }+ }+ }+ }++ # Fallback: try common Python commands that might be available+ $pythonCommands = @("python", "py", "python3")+ foreach ($cmd in $pythonCommands) {+ try {+ $version = & $cmd --version 2>&1+ if ($version -match "Python") {+ Write-Host "Found Python via command '$cmd': $version"+ return $cmd+ }+ } catch {+ continue+ }+ }++ throw "Could not find any Python installation (mapped or otherwise)"+}++try {+ # Step 1: Find the mapped Python installation+ Write-Host "Step 1: Finding mapped Python installation..."+ $pythonExe = Find-MappedPython+ Write-Host "Using Python: $pythonExe"++ # Verify Python works and show version+ $pythonVersion = & $pythonExe --version 2>&1+ Write-Host "Python version: $pythonVersion"++ # Step 2: Install cua-computer-server directly+ Write-Host "Step 2: Installing cua-computer-server..."++ Write-Host "Upgrading pip..."+ & $pythonExe -m pip install --upgrade pip --quiet++ Write-Host "Installing cua-computer-server..."+ & $pythonExe -m pip install cua-computer-server --quiet++ Write-Host "cua-computer-server installation completed."++ # Step 3: Start computer server in background+ Write-Host "Step 3: Starting computer server in background..."+ Write-Host "Starting computer server with: $pythonExe"++ # Start the computer server in the background+ $serverProcess = Start-Process -FilePath $pythonExe -ArgumentList "-m", "computer_server.main" -WindowStyle Hidden -PassThru+ Write-Host "Computer server started in background with PID: $($serverProcess.Id)"++ # Give it a moment to start+ Start-Sleep -Seconds 3++ # Check if the process is still running+ if (Get-Process -Id $serverProcess.Id -ErrorAction SilentlyContinue) {+ Write-Host "Computer server is running successfully in background"+ } else {+ throw "Computer server failed to start or exited immediately"+ }++} catch {+ Write-Error "Setup failed: $_"+ Write-Host "Error details: $($_.Exception.Message)"+ Write-Host "Stack trace: $($_.ScriptStackTrace)"+ Write-Host ""+ Write-Host "Press any key to close this window..."+ $null = $Host.UI.RawUI.ReadKey("NoEcho,IncludeKeyDown")+ exit 1+}++Write-Host ""+Write-Host "Setup completed successfully!"+Write-Host "Press any key to close this window..."+$null = $Host.UI.RawUI.ReadKey("NoEcho,IncludeKeyDown")