Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
Remove watchdog that crash-loops in containers (#1323)
r33drichards8f2ae300c80c
diff --git a/libs/python/computer-server/computer_server/cli.py b/libs/python/computer-server/computer_server/cli.pyindex ca3ef52551..1d2218cd2f 100644--- a/libs/python/computer-server/computer_server/cli.py+++ b/libs/python/computer-server/computer_server/cli.py@@ -3,11 +3,9 @@ Command-line interface for the Computer API server."""import argparse-import asyncioimport loggingimport osimport sys-import threadingfrom typing import List, Optionallogger = logging.getLogger(__name__)@@ -53,23 +51,6 @@ def parse_args(args: Optional[List[str]] = None) -> argparse.Namespace:type=str,help="Path to SSL certificate file (enables HTTPS)",)- parser.add_argument(- "--watchdog",- action="store_true",- help="Enable watchdog monitoring (automatically enabled if CONTAINER_NAME env var is set)",- )- parser.add_argument(- "--watchdog-interval",- type=int,- default=30,- help="Watchdog ping interval in seconds (default: 30)",- )- parser.add_argument(- "--no-restart",- action="store_true",- help="Disable automatic server restart in watchdog",- )-# Backend selectionparser.add_argument("--backend",@@ -128,51 +109,6 @@ def main() -> None:vnc_host = args.vnc_host or os.environ.get("CUA_VNC_HOST")logger.info(f"VNC backend enabled → {vnc_host}:{args.vnc_port}")- # Check if watchdog should be enabled- container_name = os.environ.get("CONTAINER_NAME")- enable_watchdog = (args.watchdog or bool(container_name)) and not sys.platform.startswith("win")-- if container_name:- logger.info(- f"Container environment detected (CONTAINER_NAME={container_name}), enabling watchdog"- )- elif args.watchdog:- logger.info("Watchdog explicitly enabled via --watchdog flag")-- # Start watchdog if enabled- if enable_watchdog:- logger.info(f"Starting watchdog monitoring with {args.watchdog_interval}s interval")-- def run_watchdog_thread():- """Run watchdog in a separate thread."""- loop = asyncio.new_event_loop()- asyncio.set_event_loop(loop)- try:- # Create CLI args dict for watchdog- cli_args = {- "host": args.host,- "port": args.port,- "log_level": args.log_level,- "ssl_keyfile": args.ssl_keyfile,- "ssl_certfile": args.ssl_certfile,- }-- # Create watchdog with restart settings- from .watchdog import Watchdog-- watchdog = Watchdog(cli_args=cli_args, ping_interval=args.watchdog_interval)- watchdog.restart_enabled = not args.no_restart-- loop.run_until_complete(watchdog.start_monitoring())- except Exception as e:- logger.error(f"Watchdog error: {e}")- finally:- loop.close()-- # Start watchdog in background thread- watchdog_thread = threading.Thread(target=run_watchdog_thread, daemon=True, name="watchdog")- watchdog_thread.start()-# Create and start the serverlogger.info(f"Starting Cua Computer API server on {args.host}:{args.port}...")logger.info("HTTP API available at /ws, /cmd, /status endpoints")diff --git a/libs/python/computer-server/computer_server/watchdog.py b/libs/python/computer-server/computer_server/watchdog.pydeleted file mode 100644index 50ade79689..0000000000--- a/libs/python/computer-server/computer_server/watchdog.py+++ /dev/null@@ -1,366 +0,0 @@-"""-Watchdog module for monitoring the Computer API server health.-Unix/Linux only - provides process management and restart capabilities.-"""--import asyncio-import fcntl-import json-import logging-import os-import platform-import subprocess-import sys-import time-from typing import Optional--import websockets--logger = logging.getLogger(__name__)---def instance_already_running(label="watchdog"):- """- Detect if an an instance with the label is already running, globally- at the operating system level.-- Using `os.open` ensures that the file pointer won't be closed- by Python's garbage collector after the function's scope is exited.-- The lock will be released when the program exits, or could be- released if the file pointer were closed.- """-- lock_file_pointer = os.open(f"/tmp/instance_{label}.lock", os.O_WRONLY | os.O_CREAT)-- try:- fcntl.lockf(lock_file_pointer, fcntl.LOCK_EX | fcntl.LOCK_NB)- already_running = False- except IOError:- already_running = True-- return already_running---class Watchdog:- """Watchdog class to monitor server health via WebSocket connection.- Unix/Linux only - provides restart capabilities.- """-- def __init__(self, cli_args: Optional[dict] = None, ping_interval: int = 30):- """- Initialize the watchdog.-- Args:- cli_args: Dictionary of CLI arguments to replicate when restarting- ping_interval: Interval between ping checks in seconds- """- # Check if running on Unix/Linux- if platform.system() not in ["Linux", "Darwin"]:- raise RuntimeError("Watchdog is only supported on Unix/Linux systems")-- # Store CLI arguments for restart- self.cli_args = cli_args or {}- self.host = self.cli_args.get("host", "localhost")- self.port = self.cli_args.get("port", 8000)- self.ping_interval = ping_interval- self.container_name = os.environ.get("CONTAINER_NAME")- self.running = False- self.restart_enabled = True-- @property- def ws_uri(self) -> str:- """Get the WebSocket URI using the current IP address.-- Returns:- WebSocket URI for the Computer API Server- """- if not self.container_name:- return "ws://localhost:8000/ws"-- # Try .sandbox.cua.ai first, fallback to .containers.cloud.trycua.com- return f"wss://{self.container_name}.sandbox.cua.ai:8443/ws"-- @property- def ws_uri_fallback(self) -> str:- """Get the fallback WebSocket URI using legacy hostname.-- Returns:- Fallback WebSocket URI for the Computer API Server- """- if not self.container_name:- return "ws://localhost:8000/ws"-- return f"wss://{self.container_name}.containers.cloud.trycua.com:8443/ws"-- async def ping(self) -> bool:- """- Test connection to the WebSocket endpoint.-- Returns:- True if connection successful, False otherwise- """- # Create a simple ping message- ping_message = {"command": "get_screen_size", "params": {}}-- # Try primary URI first (.sandbox.cua.ai)- try:- async with websockets.connect(- self.ws_uri, max_size=1024 * 1024 * 10 # 10MB limit to match server- ) as websocket:- # Send ping message- await websocket.send(json.dumps(ping_message))-- # Wait for any response or just close- try:- response = await asyncio.wait_for(websocket.recv(), timeout=5)- logger.debug(f"Ping response received from primary URI: {response[:100]}...")- return True- except asyncio.TimeoutError:- return False- except Exception as e:- logger.debug(f"Primary URI ping failed: {e}")-- # Try fallback URI (.containers.cloud.trycua.com)- if self.container_name:- try:- async with websockets.connect(- self.ws_uri_fallback,- max_size=1024 * 1024 * 10, # 10MB limit to match server- ) as websocket:- # Send ping message- await websocket.send(json.dumps(ping_message))-- # Wait for any response or just close- try:- response = await asyncio.wait_for(websocket.recv(), timeout=5)- logger.debug(- f"Ping response received from fallback URI: {response[:100]}..."- )- return True- except asyncio.TimeoutError:- return False- except Exception as fallback_e:- logger.warning(- f"Both primary and fallback ping failed. Primary: {e}, Fallback: {fallback_e}"- )- return False- else:- logger.warning(f"Ping failed: {e}")- return False-- def kill_processes_on_port(self, port: int) -> bool:- """- Kill any processes using the specified port.-- Args:- port: Port number to check and kill processes on-- Returns:- True if processes were killed or none found, False on error- """- try:- # Find processes using the port- result = subprocess.run(- ["lsof", "-ti", f":{port}"], capture_output=True, text=True, timeout=10- )-- if result.returncode == 0 and result.stdout.strip():- pids = result.stdout.strip().split("\n")- logger.info(f"Found {len(pids)} processes using port {port}: {pids}")-- # Kill each process- for pid in pids:- if pid.strip():- try:- subprocess.run(["kill", "-9", pid.strip()], timeout=5)- logger.info(f"Killed process {pid}")- except subprocess.TimeoutExpired:- logger.warning(f"Timeout killing process {pid}")- except Exception as e:- logger.warning(f"Error killing process {pid}: {e}")-- return True- else:- logger.debug(f"No processes found using port {port}")- return True-- except subprocess.TimeoutExpired:- logger.error(f"Timeout finding processes on port {port}")- return False- except Exception as e:- logger.error(f"Error finding processes on port {port}: {e}")- return False-- def restart_server(self) -> bool:- """- Attempt to restart the server by killing existing processes and starting new one.-- Returns:- True if restart was attempted, False on error- """- if not self.restart_enabled:- logger.info("Server restart is disabled")- return False-- try:- logger.info("Attempting to restart server...")-- # Kill processes on the port- port_to_kill = 8443 if self.container_name else self.port- if not self.kill_processes_on_port(port_to_kill):- logger.error("Failed to kill processes on port, restart aborted")- return False-- # Wait a moment for processes to die- time.sleep(2)-- # Try to restart the server- # In container mode, we can't easily restart, so just log- if self.container_name:- logger.warning("Container mode detected - cannot restart server automatically")- logger.warning("Container orchestrator should handle restart")- return False- else:- # For local mode, try to restart the CLI- logger.info("Attempting to restart local server...")-- # Get the current Python executable and script- python_exe = sys.executable-- # Try to find the CLI module- try:- # Build command with all original CLI arguments- cmd = [python_exe, "-m", "computer_server.cli"]-- # Add all CLI arguments except watchdog-related ones- for key, value in self.cli_args.items():- if key in ["watchdog", "watchdog_interval", "no_restart"]:- continue # Skip watchdog args to avoid recursive watchdog-- # Convert underscores to hyphens for CLI args- arg_name = f"--{key.replace('_', '-')}"-- if isinstance(value, bool):- if value: # Only add flag if True- cmd.append(arg_name)- else:- cmd.extend([arg_name, str(value)])-- logger.info(f"Starting server with command: {' '.join(cmd)}")-- # Start process in background- subprocess.Popen(- cmd,- stdout=subprocess.DEVNULL,- stderr=subprocess.DEVNULL,- start_new_session=True,- )-- logger.info("Server restart initiated")- return True-- except Exception as e:- logger.error(f"Failed to restart server: {e}")- return False-- except Exception as e:- logger.error(f"Error during server restart: {e}")- return False-- async def start_monitoring(self) -> None:- """Start the watchdog monitoring loop."""- self.running = True- logger.info(f"Starting watchdog monitoring for {self.ws_uri}")- logger.info(f"Ping interval: {self.ping_interval} seconds")- if self.container_name:- logger.info(f"Container mode detected: {self.container_name}")-- consecutive_failures = 0- max_failures = 3-- while self.running:- try:- success = await self.ping()-- if success:- if consecutive_failures > 0:- logger.info("Server connection restored")- consecutive_failures = 0- logger.debug("Ping successful")- else:- consecutive_failures += 1- logger.warning(f"Ping failed ({consecutive_failures}/{max_failures})")-- if consecutive_failures >= max_failures:- logger.error(- f"Server appears to be down after {max_failures} consecutive failures"- )-- # Attempt to restart the server- if self.restart_enabled:- logger.info("Attempting automatic server restart...")- restart_success = self.restart_server()-- if restart_success:- logger.info("Server restart initiated, waiting before next ping...")- # Wait longer after restart attempt- await asyncio.sleep(self.ping_interval * 2)- consecutive_failures = 0 # Reset counter after restart attempt- else:- logger.error("Server restart failed")- else:- logger.warning("Automatic restart is disabled")-- # Wait for next ping interval- await asyncio.sleep(self.ping_interval)-- except asyncio.CancelledError:- logger.info("Watchdog monitoring cancelled")- break- except Exception as e:- logger.error(f"Unexpected error in watchdog loop: {e}")- await asyncio.sleep(self.ping_interval)-- def stop_monitoring(self) -> None:- """Stop the watchdog monitoring."""- self.running = False- logger.info("Stopping watchdog monitoring")---async def run_watchdog(cli_args: Optional[dict] = None, ping_interval: int = 30) -> None:- """- Run the watchdog monitoring.-- Args:- cli_args: Dictionary of CLI arguments to replicate when restarting- ping_interval: Interval between ping checks in seconds- """- watchdog = Watchdog(cli_args=cli_args, ping_interval=ping_interval)-- try:- await watchdog.start_monitoring()- except KeyboardInterrupt:- logger.info("Watchdog stopped by user")- finally:- watchdog.stop_monitoring()---if __name__ == "__main__":- # For testing the watchdog standalone- import argparse-- parser = argparse.ArgumentParser(description="Run Computer API server watchdog")- parser.add_argument("--host", default="localhost", help="Server host to monitor")- parser.add_argument("--port", type=int, default=8000, help="Server port to monitor")- parser.add_argument("--ping-interval", type=int, default=30, help="Ping interval in seconds")-- args = parser.parse_args()-- logging.basicConfig(- level=logging.INFO,- format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",- )-- cli_args = {"host": args.host, "port": args.port}- asyncio.run(run_watchdog(cli_args, args.ping_interval))diff --git a/tests/test_watchdog.py b/tests/test_watchdog.pydeleted file mode 100644index c499c17009..0000000000--- a/tests/test_watchdog.py+++ /dev/null@@ -1,232 +0,0 @@-"""-Watchdog Recovery Tests-Tests for the watchdog functionality to ensure server recovery after hanging commands.-Required environment variables:-- CUA_API_KEY: API key for Cua cloud provider-- CUA_CONTAINER_NAME: Name of the container to use-"""--import asyncio-import os-import sys-import time-import traceback-from pathlib import Path--import pytest--# Load environment variables from .env file-project_root = Path(__file__).parent.parent-env_file = project_root / ".env"-print(f"Loading environment from: {env_file}")-from dotenv import load_dotenv--load_dotenv(env_file)--# Add paths to sys.path if needed-pythonpath = os.environ.get("PYTHONPATH", "")-for path in pythonpath.split(":"):- if path and path not in sys.path:- sys.path.insert(0, path) # Insert at beginning to prioritize- print(f"Added to sys.path: {path}")--from computer import Computer, VMProviderType--[email protected](scope="session")-async def computer():- """Shared Computer instance for all test cases."""- # Create a remote Linux computer with Cua- computer = Computer(- os_type="linux",- api_key=os.getenv("CUA_API_KEY"),- name=str(os.getenv("CUA_CONTAINER_NAME")),- provider_type=VMProviderType.CLOUD,- )-- try:- await computer.run()- yield computer- finally:- await computer.disconnect()--[email protected](loop_scope="session")-async def test_simple_server_ping(computer):- """- Simple test to verify server connectivity before running watchdog tests.- """- print("Testing basic server connectivity...")-- try:- result = await computer.interface.run_command("echo 'Server ping test'")- print(f"Ping successful: {result}")- assert result is not None, "Server ping returned None"- print("✅ Server connectivity test passed")- except Exception as e:- print(f"❌ Server ping failed: {e}")- pytest.fail(f"Basic server connectivity test failed: {e}")--[email protected](loop_scope="session")-async def test_watchdog_recovery_after_hanging_command(computer):- """- Test that the watchdog can recover the server after a hanging command.-- This test runs two concurrent tasks:- 1. A long-running command that hangs the server (sleep 300 = 5 minutes)- 2. Periodic ping commands every 30 seconds to test server responsiveness-- The watchdog should detect the unresponsive server and restart it.- """- print("Starting watchdog recovery test...")-- async def hanging_command():- """Execute a command that sleeps forever to hang the server."""- try:- print("Starting hanging command (sleep infinity)...")- # Use a very long sleep that should never complete naturally- result = await computer.interface.run_command("sleep 999999")- print(f"Hanging command completed unexpectedly: {result}")- return True # Should never reach here if watchdog works- except Exception as e:- print(f"Hanging command interrupted (expected if watchdog restarts): {e}")- return None # Expected result when watchdog kills the process-- async def ping_server():- """Ping the server every 30 seconds with echo commands."""- ping_count = 0- successful_pings = 0- failed_pings = 0-- try:- # Run pings for up to 4 minutes (8 pings at 30-second intervals)- for i in range(8):- try:- ping_count += 1- print(f"Ping #{ping_count}: Sending echo command...")-- start_time = time.time()- result = await asyncio.wait_for(- computer.interface.run_command(- f"echo 'Ping {ping_count} at {int(start_time)}'"- ),- timeout=10.0, # 10 second timeout for each ping- )- end_time = time.time()-- print(- f"Ping #{ping_count} successful in {end_time - start_time:.2f}s: {result}"- )- successful_pings += 1-- except asyncio.TimeoutError:- print(f"Ping #{ping_count} timed out (server may be unresponsive)")- failed_pings += 1- except Exception as e:- print(f"Ping #{ping_count} failed with exception: {e}")- failed_pings += 1-- # Wait 30 seconds before next ping- if i < 7: # Don't wait after the last ping- print("Waiting 30 seconds before next ping...")- await asyncio.sleep(30)-- print(f"Ping summary: {successful_pings} successful, {failed_pings} failed")- return successful_pings, failed_pings-- except Exception as e:- print(f"Ping server function failed with critical error: {e}")- traceback.print_exc()- return successful_pings, failed_pings-- # Run both tasks concurrently- print("Starting concurrent tasks: hanging command and ping monitoring...")-- try:- # Use asyncio.gather to run both tasks concurrently- hanging_task = asyncio.create_task(hanging_command())- ping_task = asyncio.create_task(ping_server())-- # Wait for both tasks to complete or timeout after 5 minutes- done, pending = await asyncio.wait(- [hanging_task, ping_task],- timeout=300, # 5 minute timeout- return_when=asyncio.ALL_COMPLETED,- )-- # Cancel any pending tasks- for task in pending:- task.cancel()- try:- await task- except asyncio.CancelledError:- pass-- # Get results from completed tasks- ping_result = None- hanging_result = None-- if ping_task in done:- try:- ping_result = await ping_task- print(f"Ping task completed with result: {ping_result}")- except Exception as e:- print(f"Error getting ping task result: {e}")- traceback.print_exc()-- if hanging_task in done:- try:- hanging_result = await hanging_task- print(f"Hanging task completed with result: {hanging_result}")- except Exception as e:- print(f"Error getting hanging task result: {e}")- traceback.print_exc()-- # Analyze results- if ping_result:- successful_pings, failed_pings = ping_result-- # Test passes if we had some successful pings, indicating recovery- assert (- successful_pings > 0- ), "No successful pings detected. Server may not have recovered."-- # Check if hanging command was killed (indicating watchdog restart)- if hanging_result is None:- print("✅ SUCCESS: Hanging command was killed - watchdog restart detected")- elif hanging_result is True:- print(- "⚠️ WARNING: Hanging command completed naturally - watchdog may not have restarted"- )-- # If we had failures followed by successes, that indicates watchdog recovery- if failed_pings > 0 and successful_pings > 0:- print(- "✅ SUCCESS: Watchdog recovery detected - server became unresponsive then recovered"- )- # Additional check: hanging command should be None if watchdog worked- assert (- hanging_result is None- ), "Expected hanging command to be killed by watchdog restart"- elif successful_pings > 0 and failed_pings == 0:- print("✅ SUCCESS: Server remained responsive throughout test")-- print(- f"Test completed: {successful_pings} successful pings, {failed_pings} failed pings"- )- print(- f"Hanging command result: {hanging_result} (None = killed by watchdog, True = completed naturally)"- )- else:- pytest.fail("Ping task did not complete - unable to assess server recovery")-- except Exception as e:- print(f"Test failed with exception: {e}")- traceback.print_exc()- pytest.fail(f"Watchdog recovery test failed: {e}")---if __name__ == "__main__":- # Run tests directly- pytest.main([__file__, "-v"])