Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

Remove watchdog that crash-loops in containers (#1323)

r33drichards8f2ae300c80c
diff --git a/libs/python/computer-server/computer_server/cli.py b/libs/python/computer-server/computer_server/cli.py
index ca3ef52551..1d2218cd2f 100644
--- a/libs/python/computer-server/computer_server/cli.py
+++ b/libs/python/computer-server/computer_server/cli.py
@@ -3,11 +3,9 @@ Command-line interface for the Computer API server.
"""
import argparse
-import asyncio
import logging
import os
import sys
-import threading
from typing import List, Optional
logger = logging.getLogger(__name__)
@@ -53,23 +51,6 @@ def parse_args(args: Optional[List[str]] = None) -> argparse.Namespace:
type=str,
help="Path to SSL certificate file (enables HTTPS)",
)
- parser.add_argument(
- "--watchdog",
- action="store_true",
- help="Enable watchdog monitoring (automatically enabled if CONTAINER_NAME env var is set)",
- )
- parser.add_argument(
- "--watchdog-interval",
- type=int,
- default=30,
- help="Watchdog ping interval in seconds (default: 30)",
- )
- parser.add_argument(
- "--no-restart",
- action="store_true",
- help="Disable automatic server restart in watchdog",
- )
-
# Backend selection
parser.add_argument(
"--backend",
@@ -128,51 +109,6 @@ def main() -> None:
vnc_host = args.vnc_host or os.environ.get("CUA_VNC_HOST")
logger.info(f"VNC backend enabled → {vnc_host}:{args.vnc_port}")
- # Check if watchdog should be enabled
- container_name = os.environ.get("CONTAINER_NAME")
- enable_watchdog = (args.watchdog or bool(container_name)) and not sys.platform.startswith("win")
-
- if container_name:
- logger.info(
- f"Container environment detected (CONTAINER_NAME={container_name}), enabling watchdog"
- )
- elif args.watchdog:
- logger.info("Watchdog explicitly enabled via --watchdog flag")
-
- # Start watchdog if enabled
- if enable_watchdog:
- logger.info(f"Starting watchdog monitoring with {args.watchdog_interval}s interval")
-
- def run_watchdog_thread():
- """Run watchdog in a separate thread."""
- loop = asyncio.new_event_loop()
- asyncio.set_event_loop(loop)
- try:
- # Create CLI args dict for watchdog
- cli_args = {
- "host": args.host,
- "port": args.port,
- "log_level": args.log_level,
- "ssl_keyfile": args.ssl_keyfile,
- "ssl_certfile": args.ssl_certfile,
- }
-
- # Create watchdog with restart settings
- from .watchdog import Watchdog
-
- watchdog = Watchdog(cli_args=cli_args, ping_interval=args.watchdog_interval)
- watchdog.restart_enabled = not args.no_restart
-
- loop.run_until_complete(watchdog.start_monitoring())
- except Exception as e:
- logger.error(f"Watchdog error: {e}")
- finally:
- loop.close()
-
- # Start watchdog in background thread
- watchdog_thread = threading.Thread(target=run_watchdog_thread, daemon=True, name="watchdog")
- watchdog_thread.start()
-
# Create and start the server
logger.info(f"Starting Cua Computer API server on {args.host}:{args.port}...")
logger.info("HTTP API available at /ws, /cmd, /status endpoints")
diff --git a/libs/python/computer-server/computer_server/watchdog.py b/libs/python/computer-server/computer_server/watchdog.py
deleted file mode 100644
index 50ade79689..0000000000
--- a/libs/python/computer-server/computer_server/watchdog.py
+++ /dev/null
@@ -1,366 +0,0 @@
-"""
-Watchdog module for monitoring the Computer API server health.
-Unix/Linux only - provides process management and restart capabilities.
-"""
-
-import asyncio
-import fcntl
-import json
-import logging
-import os
-import platform
-import subprocess
-import sys
-import time
-from typing import Optional
-
-import websockets
-
-logger = logging.getLogger(__name__)
-
-
-def instance_already_running(label="watchdog"):
- """
- Detect if an an instance with the label is already running, globally
- at the operating system level.
-
- Using `os.open` ensures that the file pointer won't be closed
- by Python's garbage collector after the function's scope is exited.
-
- The lock will be released when the program exits, or could be
- released if the file pointer were closed.
- """
-
- lock_file_pointer = os.open(f"/tmp/instance_{label}.lock", os.O_WRONLY | os.O_CREAT)
-
- try:
- fcntl.lockf(lock_file_pointer, fcntl.LOCK_EX | fcntl.LOCK_NB)
- already_running = False
- except IOError:
- already_running = True
-
- return already_running
-
-
-class Watchdog:
- """Watchdog class to monitor server health via WebSocket connection.
- Unix/Linux only - provides restart capabilities.
- """
-
- def __init__(self, cli_args: Optional[dict] = None, ping_interval: int = 30):
- """
- Initialize the watchdog.
-
- Args:
- cli_args: Dictionary of CLI arguments to replicate when restarting
- ping_interval: Interval between ping checks in seconds
- """
- # Check if running on Unix/Linux
- if platform.system() not in ["Linux", "Darwin"]:
- raise RuntimeError("Watchdog is only supported on Unix/Linux systems")
-
- # Store CLI arguments for restart
- self.cli_args = cli_args or {}
- self.host = self.cli_args.get("host", "localhost")
- self.port = self.cli_args.get("port", 8000)
- self.ping_interval = ping_interval
- self.container_name = os.environ.get("CONTAINER_NAME")
- self.running = False
- self.restart_enabled = True
-
- @property
- def ws_uri(self) -> str:
- """Get the WebSocket URI using the current IP address.
-
- Returns:
- WebSocket URI for the Computer API Server
- """
- if not self.container_name:
- return "ws://localhost:8000/ws"
-
- # Try .sandbox.cua.ai first, fallback to .containers.cloud.trycua.com
- return f"wss://{self.container_name}.sandbox.cua.ai:8443/ws"
-
- @property
- def ws_uri_fallback(self) -> str:
- """Get the fallback WebSocket URI using legacy hostname.
-
- Returns:
- Fallback WebSocket URI for the Computer API Server
- """
- if not self.container_name:
- return "ws://localhost:8000/ws"
-
- return f"wss://{self.container_name}.containers.cloud.trycua.com:8443/ws"
-
- async def ping(self) -> bool:
- """
- Test connection to the WebSocket endpoint.
-
- Returns:
- True if connection successful, False otherwise
- """
- # Create a simple ping message
- ping_message = {"command": "get_screen_size", "params": {}}
-
- # Try primary URI first (.sandbox.cua.ai)
- try:
- async with websockets.connect(
- self.ws_uri, max_size=1024 * 1024 * 10 # 10MB limit to match server
- ) as websocket:
- # Send ping message
- await websocket.send(json.dumps(ping_message))
-
- # Wait for any response or just close
- try:
- response = await asyncio.wait_for(websocket.recv(), timeout=5)
- logger.debug(f"Ping response received from primary URI: {response[:100]}...")
- return True
- except asyncio.TimeoutError:
- return False
- except Exception as e:
- logger.debug(f"Primary URI ping failed: {e}")
-
- # Try fallback URI (.containers.cloud.trycua.com)
- if self.container_name:
- try:
- async with websockets.connect(
- self.ws_uri_fallback,
- max_size=1024 * 1024 * 10, # 10MB limit to match server
- ) as websocket:
- # Send ping message
- await websocket.send(json.dumps(ping_message))
-
- # Wait for any response or just close
- try:
- response = await asyncio.wait_for(websocket.recv(), timeout=5)
- logger.debug(
- f"Ping response received from fallback URI: {response[:100]}..."
- )
- return True
- except asyncio.TimeoutError:
- return False
- except Exception as fallback_e:
- logger.warning(
- f"Both primary and fallback ping failed. Primary: {e}, Fallback: {fallback_e}"
- )
- return False
- else:
- logger.warning(f"Ping failed: {e}")
- return False
-
- def kill_processes_on_port(self, port: int) -> bool:
- """
- Kill any processes using the specified port.
-
- Args:
- port: Port number to check and kill processes on
-
- Returns:
- True if processes were killed or none found, False on error
- """
- try:
- # Find processes using the port
- result = subprocess.run(
- ["lsof", "-ti", f":{port}"], capture_output=True, text=True, timeout=10
- )
-
- if result.returncode == 0 and result.stdout.strip():
- pids = result.stdout.strip().split("\n")
- logger.info(f"Found {len(pids)} processes using port {port}: {pids}")
-
- # Kill each process
- for pid in pids:
- if pid.strip():
- try:
- subprocess.run(["kill", "-9", pid.strip()], timeout=5)
- logger.info(f"Killed process {pid}")
- except subprocess.TimeoutExpired:
- logger.warning(f"Timeout killing process {pid}")
- except Exception as e:
- logger.warning(f"Error killing process {pid}: {e}")
-
- return True
- else:
- logger.debug(f"No processes found using port {port}")
- return True
-
- except subprocess.TimeoutExpired:
- logger.error(f"Timeout finding processes on port {port}")
- return False
- except Exception as e:
- logger.error(f"Error finding processes on port {port}: {e}")
- return False
-
- def restart_server(self) -> bool:
- """
- Attempt to restart the server by killing existing processes and starting new one.
-
- Returns:
- True if restart was attempted, False on error
- """
- if not self.restart_enabled:
- logger.info("Server restart is disabled")
- return False
-
- try:
- logger.info("Attempting to restart server...")
-
- # Kill processes on the port
- port_to_kill = 8443 if self.container_name else self.port
- if not self.kill_processes_on_port(port_to_kill):
- logger.error("Failed to kill processes on port, restart aborted")
- return False
-
- # Wait a moment for processes to die
- time.sleep(2)
-
- # Try to restart the server
- # In container mode, we can't easily restart, so just log
- if self.container_name:
- logger.warning("Container mode detected - cannot restart server automatically")
- logger.warning("Container orchestrator should handle restart")
- return False
- else:
- # For local mode, try to restart the CLI
- logger.info("Attempting to restart local server...")
-
- # Get the current Python executable and script
- python_exe = sys.executable
-
- # Try to find the CLI module
- try:
- # Build command with all original CLI arguments
- cmd = [python_exe, "-m", "computer_server.cli"]
-
- # Add all CLI arguments except watchdog-related ones
- for key, value in self.cli_args.items():
- if key in ["watchdog", "watchdog_interval", "no_restart"]:
- continue # Skip watchdog args to avoid recursive watchdog
-
- # Convert underscores to hyphens for CLI args
- arg_name = f"--{key.replace('_', '-')}"
-
- if isinstance(value, bool):
- if value: # Only add flag if True
- cmd.append(arg_name)
- else:
- cmd.extend([arg_name, str(value)])
-
- logger.info(f"Starting server with command: {' '.join(cmd)}")
-
- # Start process in background
- subprocess.Popen(
- cmd,
- stdout=subprocess.DEVNULL,
- stderr=subprocess.DEVNULL,
- start_new_session=True,
- )
-
- logger.info("Server restart initiated")
- return True
-
- except Exception as e:
- logger.error(f"Failed to restart server: {e}")
- return False
-
- except Exception as e:
- logger.error(f"Error during server restart: {e}")
- return False
-
- async def start_monitoring(self) -> None:
- """Start the watchdog monitoring loop."""
- self.running = True
- logger.info(f"Starting watchdog monitoring for {self.ws_uri}")
- logger.info(f"Ping interval: {self.ping_interval} seconds")
- if self.container_name:
- logger.info(f"Container mode detected: {self.container_name}")
-
- consecutive_failures = 0
- max_failures = 3
-
- while self.running:
- try:
- success = await self.ping()
-
- if success:
- if consecutive_failures > 0:
- logger.info("Server connection restored")
- consecutive_failures = 0
- logger.debug("Ping successful")
- else:
- consecutive_failures += 1
- logger.warning(f"Ping failed ({consecutive_failures}/{max_failures})")
-
- if consecutive_failures >= max_failures:
- logger.error(
- f"Server appears to be down after {max_failures} consecutive failures"
- )
-
- # Attempt to restart the server
- if self.restart_enabled:
- logger.info("Attempting automatic server restart...")
- restart_success = self.restart_server()
-
- if restart_success:
- logger.info("Server restart initiated, waiting before next ping...")
- # Wait longer after restart attempt
- await asyncio.sleep(self.ping_interval * 2)
- consecutive_failures = 0 # Reset counter after restart attempt
- else:
- logger.error("Server restart failed")
- else:
- logger.warning("Automatic restart is disabled")
-
- # Wait for next ping interval
- await asyncio.sleep(self.ping_interval)
-
- except asyncio.CancelledError:
- logger.info("Watchdog monitoring cancelled")
- break
- except Exception as e:
- logger.error(f"Unexpected error in watchdog loop: {e}")
- await asyncio.sleep(self.ping_interval)
-
- def stop_monitoring(self) -> None:
- """Stop the watchdog monitoring."""
- self.running = False
- logger.info("Stopping watchdog monitoring")
-
-
-async def run_watchdog(cli_args: Optional[dict] = None, ping_interval: int = 30) -> None:
- """
- Run the watchdog monitoring.
-
- Args:
- cli_args: Dictionary of CLI arguments to replicate when restarting
- ping_interval: Interval between ping checks in seconds
- """
- watchdog = Watchdog(cli_args=cli_args, ping_interval=ping_interval)
-
- try:
- await watchdog.start_monitoring()
- except KeyboardInterrupt:
- logger.info("Watchdog stopped by user")
- finally:
- watchdog.stop_monitoring()
-
-
-if __name__ == "__main__":
- # For testing the watchdog standalone
- import argparse
-
- parser = argparse.ArgumentParser(description="Run Computer API server watchdog")
- parser.add_argument("--host", default="localhost", help="Server host to monitor")
- parser.add_argument("--port", type=int, default=8000, help="Server port to monitor")
- parser.add_argument("--ping-interval", type=int, default=30, help="Ping interval in seconds")
-
- args = parser.parse_args()
-
- logging.basicConfig(
- level=logging.INFO,
- format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
- )
-
- cli_args = {"host": args.host, "port": args.port}
- asyncio.run(run_watchdog(cli_args, args.ping_interval))
diff --git a/tests/test_watchdog.py b/tests/test_watchdog.py
deleted file mode 100644
index c499c17009..0000000000
--- a/tests/test_watchdog.py
+++ /dev/null
@@ -1,232 +0,0 @@
-"""
-Watchdog Recovery Tests
-Tests for the watchdog functionality to ensure server recovery after hanging commands.
-Required environment variables:
-- CUA_API_KEY: API key for Cua cloud provider
-- CUA_CONTAINER_NAME: Name of the container to use
-"""
-
-import asyncio
-import os
-import sys
-import time
-import traceback
-from pathlib import Path
-
-import pytest
-
-# Load environment variables from .env file
-project_root = Path(__file__).parent.parent
-env_file = project_root / ".env"
-print(f"Loading environment from: {env_file}")
-from dotenv import load_dotenv
-
-load_dotenv(env_file)
-
-# Add paths to sys.path if needed
-pythonpath = os.environ.get("PYTHONPATH", "")
-for path in pythonpath.split(":"):
- if path and path not in sys.path:
- sys.path.insert(0, path) # Insert at beginning to prioritize
- print(f"Added to sys.path: {path}")
-
-from computer import Computer, VMProviderType
-
-
[email protected](scope="session")
-async def computer():
- """Shared Computer instance for all test cases."""
- # Create a remote Linux computer with Cua
- computer = Computer(
- os_type="linux",
- api_key=os.getenv("CUA_API_KEY"),
- name=str(os.getenv("CUA_CONTAINER_NAME")),
- provider_type=VMProviderType.CLOUD,
- )
-
- try:
- await computer.run()
- yield computer
- finally:
- await computer.disconnect()
-
-
[email protected](loop_scope="session")
-async def test_simple_server_ping(computer):
- """
- Simple test to verify server connectivity before running watchdog tests.
- """
- print("Testing basic server connectivity...")
-
- try:
- result = await computer.interface.run_command("echo 'Server ping test'")
- print(f"Ping successful: {result}")
- assert result is not None, "Server ping returned None"
- print("✅ Server connectivity test passed")
- except Exception as e:
- print(f"❌ Server ping failed: {e}")
- pytest.fail(f"Basic server connectivity test failed: {e}")
-
-
[email protected](loop_scope="session")
-async def test_watchdog_recovery_after_hanging_command(computer):
- """
- Test that the watchdog can recover the server after a hanging command.
-
- This test runs two concurrent tasks:
- 1. A long-running command that hangs the server (sleep 300 = 5 minutes)
- 2. Periodic ping commands every 30 seconds to test server responsiveness
-
- The watchdog should detect the unresponsive server and restart it.
- """
- print("Starting watchdog recovery test...")
-
- async def hanging_command():
- """Execute a command that sleeps forever to hang the server."""
- try:
- print("Starting hanging command (sleep infinity)...")
- # Use a very long sleep that should never complete naturally
- result = await computer.interface.run_command("sleep 999999")
- print(f"Hanging command completed unexpectedly: {result}")
- return True # Should never reach here if watchdog works
- except Exception as e:
- print(f"Hanging command interrupted (expected if watchdog restarts): {e}")
- return None # Expected result when watchdog kills the process
-
- async def ping_server():
- """Ping the server every 30 seconds with echo commands."""
- ping_count = 0
- successful_pings = 0
- failed_pings = 0
-
- try:
- # Run pings for up to 4 minutes (8 pings at 30-second intervals)
- for i in range(8):
- try:
- ping_count += 1
- print(f"Ping #{ping_count}: Sending echo command...")
-
- start_time = time.time()
- result = await asyncio.wait_for(
- computer.interface.run_command(
- f"echo 'Ping {ping_count} at {int(start_time)}'"
- ),
- timeout=10.0, # 10 second timeout for each ping
- )
- end_time = time.time()
-
- print(
- f"Ping #{ping_count} successful in {end_time - start_time:.2f}s: {result}"
- )
- successful_pings += 1
-
- except asyncio.TimeoutError:
- print(f"Ping #{ping_count} timed out (server may be unresponsive)")
- failed_pings += 1
- except Exception as e:
- print(f"Ping #{ping_count} failed with exception: {e}")
- failed_pings += 1
-
- # Wait 30 seconds before next ping
- if i < 7: # Don't wait after the last ping
- print("Waiting 30 seconds before next ping...")
- await asyncio.sleep(30)
-
- print(f"Ping summary: {successful_pings} successful, {failed_pings} failed")
- return successful_pings, failed_pings
-
- except Exception as e:
- print(f"Ping server function failed with critical error: {e}")
- traceback.print_exc()
- return successful_pings, failed_pings
-
- # Run both tasks concurrently
- print("Starting concurrent tasks: hanging command and ping monitoring...")
-
- try:
- # Use asyncio.gather to run both tasks concurrently
- hanging_task = asyncio.create_task(hanging_command())
- ping_task = asyncio.create_task(ping_server())
-
- # Wait for both tasks to complete or timeout after 5 minutes
- done, pending = await asyncio.wait(
- [hanging_task, ping_task],
- timeout=300, # 5 minute timeout
- return_when=asyncio.ALL_COMPLETED,
- )
-
- # Cancel any pending tasks
- for task in pending:
- task.cancel()
- try:
- await task
- except asyncio.CancelledError:
- pass
-
- # Get results from completed tasks
- ping_result = None
- hanging_result = None
-
- if ping_task in done:
- try:
- ping_result = await ping_task
- print(f"Ping task completed with result: {ping_result}")
- except Exception as e:
- print(f"Error getting ping task result: {e}")
- traceback.print_exc()
-
- if hanging_task in done:
- try:
- hanging_result = await hanging_task
- print(f"Hanging task completed with result: {hanging_result}")
- except Exception as e:
- print(f"Error getting hanging task result: {e}")
- traceback.print_exc()
-
- # Analyze results
- if ping_result:
- successful_pings, failed_pings = ping_result
-
- # Test passes if we had some successful pings, indicating recovery
- assert (
- successful_pings > 0
- ), "No successful pings detected. Server may not have recovered."
-
- # Check if hanging command was killed (indicating watchdog restart)
- if hanging_result is None:
- print("✅ SUCCESS: Hanging command was killed - watchdog restart detected")
- elif hanging_result is True:
- print(
- "⚠️ WARNING: Hanging command completed naturally - watchdog may not have restarted"
- )
-
- # If we had failures followed by successes, that indicates watchdog recovery
- if failed_pings > 0 and successful_pings > 0:
- print(
- "✅ SUCCESS: Watchdog recovery detected - server became unresponsive then recovered"
- )
- # Additional check: hanging command should be None if watchdog worked
- assert (
- hanging_result is None
- ), "Expected hanging command to be killed by watchdog restart"
- elif successful_pings > 0 and failed_pings == 0:
- print("✅ SUCCESS: Server remained responsive throughout test")
-
- print(
- f"Test completed: {successful_pings} successful pings, {failed_pings} failed pings"
- )
- print(
- f"Hanging command result: {hanging_result} (None = killed by watchdog, True = completed naturally)"
- )
- else:
- pytest.fail("Ping task did not complete - unable to assess server recovery")
-
- except Exception as e:
- print(f"Test failed with exception: {e}")
- traceback.print_exc()
- pytest.fail(f"Watchdog recovery test failed: {e}")
-
-
-if __name__ == "__main__":
- # Run tests directly
- pytest.main([__file__, "-v"])