Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
feat(computer-server): add VNC backend with --backend flag (#1196)
Francesco Bonacci5d7f13c3feb4
diff --git a/docs/content/docs/cua/reference/agent-sdk/index.mdx b/docs/content/docs/cua/reference/agent-sdk/index.mdxindex 16ef2f1808..ece0060541 100644--- a/docs/content/docs/cua/reference/agent-sdk/index.mdx+++ b/docs/content/docs/cua/reference/agent-sdk/index.mdx@@ -7,7 +7,7 @@ description: Python API reference for building computer-use agentsAUTO-GENERATED FILE - DO NOT EDIT DIRECTLYGenerated by: npx tsx scripts/docs-generators/python-sdk.tsSource: libs/python/agent/agent- Version: 0.7.30+ Version: 0.7.36*/}import { Callout } from 'fumadocs-ui/components/callout';@@ -17,7 +17,7 @@ import { VersionHeader } from '@/components/version-selector';<VersionHeaderversions={[{"version":"0.7","href":"/cua/reference/agent-sdk","isCurrent":true},{"version":"0.6","href":"/cua/reference/agent-sdk/v0.6/api","isCurrent":false},{"version":"0.5","href":"/cua/reference/agent-sdk/v0.5/api","isCurrent":false},{"version":"0.4","href":"/cua/reference/agent-sdk/v0.4/api","isCurrent":false},{"version":"0.3","href":"/cua/reference/agent-sdk/v0.3/api","isCurrent":false}]}currentVersion="0.7"- fullVersion="0.7.30"+ fullVersion="0.7.36"packageName="cua-agent"/>@@ -1346,3 +1346,12 @@ async def on_computer_call_end(self, item: Dict[str, Any], result: List[Dict[strCalled when a computer call has completed.Saves screenshots and computer call output.++#### TrajectorySaverCallback.on_function_call_end++```python+async def on_function_call_end(self, item: Dict[str, Any], result: List[Dict[str, Any]]) -> None+```++Called when a function call has completed.+Saves screenshots and function call output for GPT 5.4 / BrowserTool.diff --git a/docs/content/docs/cua/reference/cli/index.mdx b/docs/content/docs/cua/reference/cli/index.mdxindex 165ac3ee38..f7fa87bf0b 100644--- a/docs/content/docs/cua/reference/cli/index.mdx+++ b/docs/content/docs/cua/reference/cli/index.mdx@@ -7,7 +7,7 @@ description: Python API reference for the Cua command-line interfaceAUTO-GENERATED FILE - DO NOT EDIT DIRECTLYGenerated by: npx tsx scripts/docs-generators/python-sdk.tsSource: libs/python/cua-cli/cua_cli- Version: 0.1.7+ Version: 0.1.8*/}import { Callout } from 'fumadocs-ui/components/callout';@@ -17,7 +17,7 @@ import { VersionHeader } from '@/components/version-selector';<VersionHeaderversions={[{"version":"0.1","href":"/cua/reference/cli","isCurrent":true}]}currentVersion="0.1"- fullVersion="0.1.7"+ fullVersion="0.1.8"packageName="cua-cli"/>diff --git a/docs/content/docs/cua/reference/computer-sdk/index.mdx b/docs/content/docs/cua/reference/computer-sdk/index.mdxindex 89d2d7e013..40d15551f4 100644--- a/docs/content/docs/cua/reference/computer-sdk/index.mdx+++ b/docs/content/docs/cua/reference/computer-sdk/index.mdx@@ -7,7 +7,7 @@ description: Python API reference for controlling virtual machines and computerAUTO-GENERATED FILE - DO NOT EDIT DIRECTLYGenerated by: npx tsx scripts/docs-generators/python-sdk.tsSource: libs/python/computer/computer- Version: 0.5.16+ Version: 0.5.17*/}import { Callout } from 'fumadocs-ui/components/callout';@@ -17,7 +17,7 @@ import { VersionHeader } from '@/components/version-selector';<VersionHeaderversions={[{"version":"0.5","href":"/cua/reference/computer-sdk","isCurrent":true},{"version":"0.4","href":"/cua/reference/computer-sdk/v0.4/api","isCurrent":false},{"version":"0.3","href":"/cua/reference/computer-sdk/v0.3/api","isCurrent":false}]}currentVersion="0.5"- fullVersion="0.5.16"+ fullVersion="0.5.17"packageName="cua-computer"/>@@ -41,7 +41,7 @@ Computer is the main class for interacting with the computer.### Constructor```python-Computer(self, display: Union[Display, Dict[str, int], str] = '1024x768', memory: str = '8GB', cpu: str = '4', os_type: OSType = 'macos', name: str = '', image: Optional[str] = None, shared_directories: Optional[List[str]] = None, use_host_computer_server: bool = False, verbosity: Union[int, LogLevel] = logging.INFO, telemetry_enabled: bool = True, provider_type: Union[str, VMProviderType] = VMProviderType.LUME, provider_port: Optional[int] = 7777, noVNC_port: Optional[int] = 8006, api_port: Optional[int] = None, host: str = 'localhost', api_host: Optional[str] = None, storage: Optional[str] = None, ephemeral: bool = False, api_key: Optional[str] = None, experiments: Optional[List[str]] = None, timeout: int = 100, run_opts: Optional[Dict[str, Any]] = None)+Computer(self, display: Union[Display, Dict[str, int], str] = '1024x768', memory: str = '8GB', cpu: str = '4', os_type: OSType = 'macos', name: str = '', image: Optional[str] = None, shared_directories: Optional[List[str]] = None, use_host_computer_server: bool = False, verbosity: Union[int, LogLevel] = logging.INFO, telemetry_enabled: bool = True, provider_type: Union[str, VMProviderType] = VMProviderType.LUME, provider_port: Optional[int] = 7777, noVNC_port: Optional[int] = 8006, api_port: Optional[int] = None, host: str = 'localhost', api_host: Optional[str] = None, storage: Optional[str] = None, ephemeral: bool = False, api_key: Optional[str] = None, experiments: Optional[List[str]] = None, timeout: int = 100, backend: Literal['native', 'vnc'] = 'native', vnc_host: Optional[str] = None, vnc_port: int = 5900, vnc_password: str = '', run_opts: Optional[Dict[str, Any]] = None)```### Attributes@@ -61,6 +61,10 @@ Computer(self, display: Union[Display, Dict[str, int], str] = '1024x768', memory| `api_key` | `Any` | || `timeout` | `Any` | || `experiments` | `Any` | |+| `backend` | `Any` | |+| `vnc_host` | `Any` | |+| `vnc_port` | `Any` | |+| `vnc_password` | `Any` | || `custom_run_opts` | `Any` | || `storage` | `Any` | || `shared_path` | `Any` | |diff --git a/docs/content/docs/cuabench/reference/api.mdx b/docs/content/docs/cuabench/reference/api.mdxindex a6ebc9abf3..4258fd13bd 100644--- a/docs/content/docs/cuabench/reference/api.mdx+++ b/docs/content/docs/cuabench/reference/api.mdx@@ -7,7 +7,7 @@ description: Python API reference for the desktop automation benchmarking framewAUTO-GENERATED FILE - DO NOT EDIT DIRECTLYGenerated by: npx tsx scripts/docs-generators/python-sdk.tsSource: libs/cua-bench/cua_bench- Version: 0.2.4+ Version: 0.2.6*/}import { Callout } from 'fumadocs-ui/components/callout';@@ -17,7 +17,7 @@ import { VersionHeader } from '@/components/version-selector';<VersionHeaderversions={[{"version":"0.2","href":"/cuabench/reference","isCurrent":true}]}currentVersion="0.2"- fullVersion="0.2.4"+ fullVersion="0.2.6"packageName="cua-bench"/>@@ -3595,7 +3595,7 @@ TaskRunner(self, agent_image: str = DEFAULT_AGENT_IMAGE, env_hostname: str = 'cu#### TaskRunner.run_task```python-async def run_task(self, env_path: Path, task_index: int, env_type: str, golden_name: Optional[str] = None, agent: Optional[str] = None, agent_image: Optional[str] = None, agent_command: Optional[List[str]] = None, agent_import_path: Optional[str] = None, model: Optional[str] = None, max_steps: int = 100, oracle: bool = False, memory: str = '8G', cpus: str = '8', vnc_port: Optional[int] = None, api_port: Optional[int] = None, output_dir: Optional[str] = None, stream_agent_logs: bool = False, timeout: Optional[int] = None, cleanup_before: bool = True, remove_images_after: bool = False, provider_type: Optional[str] = None) -> TaskResult+async def run_task(self, env_path: Path, task_index: int, env_type: str, golden_name: Optional[str] = None, agent: Optional[str] = None, agent_image: Optional[str] = None, agent_command: Optional[List[str]] = None, agent_import_path: Optional[str] = None, model: Optional[str] = None, max_steps: int = 100, oracle: bool = False, memory: str = '8G', cpus: str = '8', vnc_port: Optional[int] = None, api_port: Optional[int] = None, output_dir: Optional[str] = None, stream_agent_logs: bool = False, timeout: Optional[int] = None, cleanup_before: bool = True, remove_images_after: bool = False, provider_type: Optional[str] = None, dev_paths: Optional[List[str]] = None) -> TaskResult```Run a task with 2-container architecture.diff --git a/libs/lume/scripts/setup-cua-computer.sh b/libs/lume/scripts/setup-cua-computer.shnew file mode 100755index 0000000000..c5d7f5aaa8--- /dev/null+++ b/libs/lume/scripts/setup-cua-computer.sh@@ -0,0 +1,476 @@+#!/bin/bash+#+# setup-cua-computer.sh+# Makes a Lume macOS VM image cua-computer compatible+#+# This script runs ON the VM itself (not the host).+# It installs cua-computer-server, cua-agent, Playwright, and configures+# the VM for headless CUA operation (auto-login, disable sleep, LaunchAgent).+#+# Usage:+# lume ssh <vm-name> "curl -fsSL <url> | bash"+# lume ssh <vm-name> < setup-cua-computer.sh+# # Or copy and run directly inside the VM:+# ./setup-cua-computer.sh [--yes] [--port PORT]+#+# Prerequisites:+# - macOS VM created via lume (with SSH enabled, user lume/lume)+#++set -e++# ============================================+# Configuration+# ============================================++RED='\033[0;31m'+GREEN='\033[0;32m'+YELLOW='\033[1;33m'+BLUE='\033[0;34m'+NC='\033[0m'++AUTO_YES=false+CUA_SERVER_PORT=8443+VM_PASSWORD="${VM_PASSWORD:-lume}"+CUA_DIR="$HOME/.cua-server"++# Parse arguments+while [ "$#" -gt 0 ]; do+ case "$1" in+ --yes|-y)+ AUTO_YES=true+ ;;+ --port)+ CUA_SERVER_PORT="$2"+ shift+ ;;+ --help)+ echo "Usage: $0 [OPTIONS]"+ echo ""+ echo "Options:"+ echo " --yes, -y Non-interactive mode (accept all prompts)"+ echo " --port PORT CUA computer server port (default: 8443)"+ echo " --help Show this help message"+ exit 0+ ;;+ *)+ echo -e "${RED}Unknown option: $1${NC}"+ exit 1+ ;;+ esac+ shift+done++CURRENT_USER="$(whoami)"+USER_HOME="$HOME"++# ============================================+# Helpers+# ============================================++print_phase() {+ echo ""+ echo -e "${BLUE}========================================${NC}"+ echo -e "${BLUE} $1${NC}"+ echo -e "${BLUE}========================================${NC}"+ echo ""+}++confirm() {+ local prompt="$1"+ if [ "$AUTO_YES" = true ]; then+ echo -e "${YELLOW}${prompt} [Y/n] ${NC}y (auto)"+ return 0+ fi+ read -r -p "$(echo -e "${YELLOW}${prompt} [Y/n] ${NC}")" response+ case "$response" in+ [nN][oO]|[nN]) return 1 ;;+ *) return 0 ;;+ esac+}++ensure_brew() {+ if [ -f /opt/homebrew/bin/brew ]; then+ eval "$(/opt/homebrew/bin/brew shellenv)"+ fi+}++# Create an askpass helper for non-interactive sudo+setup_sudo() {+ if [ "$AUTO_YES" = true ]; then+ ASKPASS_SCRIPT=$(mktemp)+ printf '#!/bin/bash\necho "%s"\n' "$VM_PASSWORD" > "$ASKPASS_SCRIPT"+ chmod +x "$ASKPASS_SCRIPT"+ export SUDO_ASKPASS="$ASKPASS_SCRIPT"+ sudo -A -v 2>/dev/null || true+ fi+}++cleanup_sudo() {+ if [ -n "${ASKPASS_SCRIPT:-}" ] && [ -f "$ASKPASS_SCRIPT" ]; then+ rm -f "$ASKPASS_SCRIPT"+ fi+}+trap cleanup_sudo EXIT++# ============================================+# Phase 1: Prerequisites (Homebrew + Python)+# ============================================++install_homebrew() {+ print_phase "Phase 1: Homebrew"++ if command -v brew &>/dev/null; then+ echo -e "${GREEN}Homebrew already installed$(brew --version | head -1 | sed 's/Homebrew //' | sed 's/^/ (v/')${NC})"+ ensure_brew+ return 0+ fi++ if ! confirm "Install Homebrew?"; then+ echo -e "${YELLOW}Skipping Homebrew. Later steps will fail.${NC}"+ return 1+ fi++ setup_sudo++ if [ "$AUTO_YES" = true ]; then+ NONINTERACTIVE=1 SUDO_ASKPASS="$ASKPASS_SCRIPT" /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)"+ else+ echo -e "${YELLOW}Homebrew requires sudo access. Enter your password if prompted.${NC}"+ /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)"+ fi++ # Add brew to shell profile+ if ! grep -q 'brew shellenv' "$HOME/.zprofile" 2>/dev/null; then+ echo 'eval "$(/opt/homebrew/bin/brew shellenv)"' >> "$HOME/.zprofile"+ fi++ ensure_brew+ echo -e "${GREEN}Homebrew installed${NC}"+}++install_python() {+ print_phase "Phase 2: Python 3.13"++ ensure_brew++ if brew list [email protected] &>/dev/null; then+ echo -e "${GREEN}Python 3.13 already installed${NC}"+ else+ if ! confirm "Install Python 3.13 via Homebrew?"; then+ echo -e "${YELLOW}Skipping Python 3.13.${NC}"+ return 1+ fi+ brew install [email protected]+ echo -e "${GREEN}Python 3.13 installed${NC}"+ fi++ export PATH="/opt/homebrew/opt/[email protected]/bin:$PATH"+}++install_system_deps() {+ print_phase "Phase 3: System Dependencies"++ ensure_brew++ local deps_needed=()+ for pkg in ffmpeg; do+ if ! brew list "$pkg" &>/dev/null; then+ deps_needed+=("$pkg")+ fi+ done++ if [ ${#deps_needed[@]} -eq 0 ]; then+ echo -e "${GREEN}All system dependencies already installed${NC}"+ return 0+ fi++ if confirm "Install system dependencies (${deps_needed[*]})?"; then+ brew install "${deps_needed[@]}"+ echo -e "${GREEN}System dependencies installed${NC}"+ fi+}++# ============================================+# Phase 4: CUA Server + Agent+# ============================================++setup_cua_server() {+ print_phase "Phase 4: CUA Computer Server & Agent"++ ensure_brew+ export PATH="/opt/homebrew/opt/[email protected]/bin:$PATH"++ if ! confirm "Install cua-computer-server and cua-agent?"; then+ echo -e "${YELLOW}Skipping CUA server setup.${NC}"+ return 1+ fi++ mkdir -p "$CUA_DIR"++ # Create venv if needed+ if [ ! -d "$CUA_DIR/venv" ]; then+ echo "Creating Python 3.13 virtual environment..."+ /opt/homebrew/opt/[email protected]/bin/python3.13 -m venv "$CUA_DIR/venv"+ fi++ echo "Installing cua-computer-server..."+ source "$CUA_DIR/venv/bin/activate"+ pip install --upgrade pip+ pip install cua-computer-server++ echo "Installing cua-agent..."+ pip install 'cua-agent[all]'++ # Playwright + Firefox (non-blocking)+ echo "Installing Playwright and Firefox browser..."+ pip install playwright && playwright install firefox || \+ echo -e "${YELLOW}Playwright install failed (non-blocking, BrowserTool may not work)${NC}"++ deactivate++ echo -e "${GREEN}CUA server and agent installed${NC}"+}++# ============================================+# Phase 5: Startup Script + LaunchAgent+# ============================================++create_startup_script() {+ print_phase "Phase 5: Startup Script & LaunchAgent"++ if ! confirm "Create CUA server startup script and LaunchAgent?"; then+ echo -e "${YELLOW}Skipping LaunchAgent setup.${NC}"+ return 1+ fi++ local STARTUP_SCRIPT="$CUA_DIR/start_server.sh"++ cat > "$STARTUP_SCRIPT" <<'SCRIPT_EOF'+#!/bin/bash++LOG_FILE="$HOME/.cua-server/server.log"+echo "=== CUA Server startup at $(date) ===" >> "$LOG_FILE"++# Wait for window server+sleep 10++cd "$HOME/.cua-server"+source venv/bin/activate++export DISPLAY=:0++# Update packages+echo "Updating cua-agent..." >> "$LOG_FILE"+pip install --upgrade --no-input "cua-agent[all]" >> "$LOG_FILE" 2>&1 || true++echo "Updating cua-computer-server..." >> "$LOG_FILE"+pip install --upgrade --no-input cua-computer-server >> "$LOG_FILE" 2>&1 || true++# Ensure Playwright Firefox+echo "Ensuring Playwright Firefox..." >> "$LOG_FILE"+pip install --upgrade --no-input playwright >> "$LOG_FILE" 2>&1 || true+playwright install firefox >> "$LOG_FILE" 2>&1 || true++# Start server+echo "Starting CUA computer server on port __PORT__..." >> "$LOG_FILE"+python -m computer_server --port __PORT__ >> "$LOG_FILE" 2>&1+SCRIPT_EOF++ # Stamp the port+ sed -i '' "s/__PORT__/$CUA_SERVER_PORT/g" "$STARTUP_SCRIPT"+ chmod +x "$STARTUP_SCRIPT"++ # Create LaunchAgent+ local SERVICE_NAME="com.trycua.computer_server"+ local PLIST_PATH="$USER_HOME/Library/LaunchAgents/$SERVICE_NAME.plist"++ mkdir -p "$USER_HOME/Library/LaunchAgents"++ # Unload existing+ launchctl unload "$PLIST_PATH" 2>/dev/null || true++ cat > "$PLIST_PATH" <<EOF+<?xml version="1.0" encoding="UTF-8"?>+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">+<plist version="1.0">+<dict>+ <key>Label</key>+ <string>$SERVICE_NAME</string>++ <key>WorkingDirectory</key>+ <string>$CUA_DIR</string>++ <key>EnvironmentVariables</key>+ <dict>+ <key>PATH</key>+ <string>/opt/homebrew/bin:/opt/homebrew/opt/[email protected]/bin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin</string>+ <key>HOME</key>+ <string>$USER_HOME</string>+ <key>DISPLAY</key>+ <string>:0</string>+ </dict>++ <key>ProgramArguments</key>+ <array>+ <string>/bin/bash</string>+ <string>$CUA_DIR/start_server.sh</string>+ </array>++ <key>RunAtLoad</key>+ <true/>++ <key>KeepAlive</key>+ <true/>++ <key>StandardOutPath</key>+ <string>/tmp/computer_server.log</string>++ <key>StandardErrorPath</key>+ <string>/tmp/computer_server.error.log</string>++ <key>ProcessType</key>+ <string>Interactive</string>++ <key>SessionType</key>+ <string>Aqua</string>+</dict>+</plist>+EOF++ chmod 644 "$PLIST_PATH"+ touch /tmp/computer_server.log /tmp/computer_server.error.log++ echo -e "${GREEN}Startup script and LaunchAgent created${NC}"+ echo "LaunchAgent will start CUA server on port $CUA_SERVER_PORT at login"+}++# ============================================+# Phase 6: System Configuration+# ============================================++configure_system() {+ print_phase "Phase 6: System Configuration"++ if ! confirm "Disable screen lock, screensaver, and sleep?"; then+ echo -e "${YELLOW}Skipping system configuration.${NC}"+ return 0+ fi++ setup_sudo++ # Disable screensaver+ defaults -currentHost write com.apple.screensaver idleTime 0+ defaults write com.apple.screensaver askForPassword -int 0+ defaults write com.apple.screensaver askForPasswordDelay -int 0++ # Disable sleep+ if [ "$AUTO_YES" = true ]; then+ sudo -A pmset -a displaysleep 0 sleep 0 disksleep 0+ else+ sudo pmset -a displaysleep 0 sleep 0 disksleep 0+ fi++ # Enable full keyboard navigation (needed for accessibility automation)+ defaults write NSGlobalDomain AppleKeyboardUIMode -int 3++ echo -e "${GREEN}System configured (no sleep, no screensaver, full keyboard nav)${NC}"+}++# ============================================+# Phase 7: Verification+# ============================================++verify() {+ print_phase "Verification"++ echo "System:"+ echo " User: $CURRENT_USER"+ sw_vers | sed 's/^/ /'++ echo ""+ echo "Installed:"++ # Homebrew+ if command -v brew &>/dev/null; then+ echo -e " ${GREEN}✓${NC} Homebrew $(brew --version | head -1 | grep -oE '[0-9]+\.[0-9]+\.[0-9]+')"+ else+ echo -e " ${RED}✗${NC} Homebrew"+ fi++ # Python+ ensure_brew+ if brew list [email protected] &>/dev/null; then+ echo -e " ${GREEN}✓${NC} Python 3.13"+ else+ echo -e " ${RED}✗${NC} Python 3.13"+ fi++ # CUA server+ if [ -f "$CUA_DIR/venv/bin/python" ]; then+ local server_ver+ server_ver=$("$CUA_DIR/venv/bin/pip" show cua-computer-server 2>/dev/null | grep Version | awk '{print $2}')+ echo -e " ${GREEN}✓${NC} cua-computer-server ${server_ver:-unknown}"+ else+ echo -e " ${RED}✗${NC} cua-computer-server"+ fi++ # Agent+ if [ -f "$CUA_DIR/venv/bin/python" ]; then+ local agent_ver+ agent_ver=$("$CUA_DIR/venv/bin/pip" show cua-agent 2>/dev/null | grep Version | awk '{print $2}')+ echo -e " ${GREEN}✓${NC} cua-agent ${agent_ver:-unknown}"+ else+ echo -e " ${RED}✗${NC} cua-agent"+ fi++ # Playwright+ if "$CUA_DIR/venv/bin/python" -c "import playwright" 2>/dev/null; then+ echo -e " ${GREEN}✓${NC} Playwright + Firefox"+ else+ echo -e " ${YELLOW}~${NC} Playwright (may not be installed)"+ fi++ # LaunchAgent+ local PLIST="$USER_HOME/Library/LaunchAgents/com.trycua.computer_server.plist"+ if [ -f "$PLIST" ]; then+ echo -e " ${GREEN}✓${NC} LaunchAgent (port $CUA_SERVER_PORT)"+ else+ echo -e " ${RED}✗${NC} LaunchAgent"+ fi++ echo ""+ echo -e "${GREEN}========================================${NC}"+ echo -e "${GREEN} Setup Complete!${NC}"+ echo -e "${GREEN}========================================${NC}"+ echo ""+ echo "CUA computer server will start on port $CUA_SERVER_PORT at next login."+ echo ""+ echo "To start it now (from the host):"+ echo " lume ssh <vm-name> 'launchctl load ~/Library/LaunchAgents/com.trycua.computer_server.plist'"+ echo ""+ echo "To test from the host:"+ echo " curl http://<vm-ip>:$CUA_SERVER_PORT/health"+ echo ""+}++# ============================================+# Main+# ============================================++echo -e "${BLUE}"+echo " ⠀⣀⣀⡀⠀⠀⠀⠀⢀⣀⣀⣀⡀⠘⠋⢉⠙⣷⠀⠀ ⠀"+echo " ⠀⠀⢀⣴⣿⡿⠋⣉⠁⣠⣾⣿⣿⣿⣿⡿⠿⣦⡈⠀⣿⡇⠃⠀"+echo " ⠀⠀⠀⣽⣿⣧⠀⠃⢰⣿⣿⡏⠙⣿⠿⢧⣀⣼⣷⠀⡿⠃⠀⠀"+echo " ⠀⠀⠀⠉⣿⣿⣦⠀⢿⣿⣿⣷⣾⡏⠀⠀⢹⣿⣿⠀⠀⠀⠀⠀⠀"+echo -e " ⠀⠀⠀⠀⠀⠉⠛⠁⠈⠿⣿⣿⣿⣷⣄⣠⡼⠟⠁${NC} ${BLUE}CUA Computer Setup${NC}"+echo -e "${BLUE} Make a Lume VM cua-computer compatible${NC}"+echo ""++install_homebrew+install_python+install_system_deps+setup_cua_server+create_startup_script+configure_system+verifydiff --git a/libs/python/agent/agent/adapters/yutori_adapter.py b/libs/python/agent/agent/adapters/yutori_adapter.pyindex 1bf6eb4a2a..47b821f7c1 100644--- a/libs/python/agent/agent/adapters/yutori_adapter.py+++ b/libs/python/agent/agent/adapters/yutori_adapter.py@@ -21,7 +21,7 @@ class YutoriAdapter(CustomLLM):def _normalize_model(self, model: str) -> str:"""Strip the yutori/ prefix to get the bare model name."""if model.startswith("yutori/"):- return model[len("yutori/"):]+ return model[len("yutori/") :]return modeldef _resolve_api_key(self, kwargs: dict | None = None) -> str:@@ -61,7 +61,13 @@ class YutoriAdapter(CustomLLM):params["tool_choice"] = kwargs["tool_choice"]# Forward optional generation params- for key in ("temperature", "top_p", "max_completion_tokens", "max_tokens", "response_format"):+ for key in (+ "temperature",+ "top_p",+ "max_completion_tokens",+ "max_tokens",+ "response_format",+ ):if key in kwargs:params[key] = kwargs[key]diff --git a/libs/python/agent/agent/agent.py b/libs/python/agent/agent/agent.pyindex 388a514e7b..70e93293b5 100644--- a/libs/python/agent/agent/agent.py+++ b/libs/python/agent/agent/agent.py@@ -91,6 +91,7 @@ def get_json(obj: Any, max_depth: int = 10) -> Any:# Handle enums — just use their valueimport enum+if isinstance(o, enum.Enum):return o.value@@ -856,7 +857,9 @@ class ComputerAgent:# Note: function_call_output only supports text, so we include the# screenshot as a separate user message with image contentif is_terminate:- output_content = json.dumps(action_result if action_result else {"terminated": True})+ output_content = json.dumps(+ action_result if action_result else {"terminated": True}+ )call_output = {"type": "function_call_output","call_id": item.get("call_id"),@@ -870,7 +873,9 @@ class ComputerAgent:if action_result is not None:output_content = json.dumps(action_result)else:- output_content = json.dumps({"success": True, "screenshot_captured": True})+ output_content = json.dumps(+ {"success": True, "screenshot_captured": True}+ )call_output = {"type": "function_call_output","call_id": item.get("call_id"),diff --git a/libs/python/agent/agent/callbacks/trajectory_saver.py b/libs/python/agent/agent/callbacks/trajectory_saver.pyindex 81627882f7..5d091e3641 100644--- a/libs/python/agent/agent/callbacks/trajectory_saver.py+++ b/libs/python/agent/agent/callbacks/trajectory_saver.py@@ -156,7 +156,10 @@ def extract_computer_call_outputs(new_content = []content_modified = Falsefor content_item in content:- if isinstance(content_item, dict) and content_item.get("type") == "input_image":+ if (+ isinstance(content_item, dict)+ and content_item.get("type") == "input_image"+ ):image_url = content_item.get("image_url", "")if isinstance(image_url, str) and image_url.startswith("data:"):# Generate a unique ID for this screenshot@@ -559,6 +562,7 @@ class TrajectorySaverCallback(AsyncCallbackHandler):# Try to evaluate as Python literal (for stringified dicts)try:import ast+output_dict = ast.literal_eval(output)except (ValueError, SyntaxError):continue@@ -589,7 +593,12 @@ class TrajectorySaverCallback(AsyncCallbackHandler):image_bytes = base64.b64decode(image_data)# If we have coordinates, draw crosshair annotation- if x_coord is not None and y_coord is not None and x_coord != 0 and y_coord != 0:+ if (+ x_coord is not None+ and y_coord is not None+ and x_coord != 0+ and y_coord != 0+ ):annotated_image = self._draw_crosshair_on_image(image_bytes, int(x_coord), int(y_coord))@@ -610,7 +619,10 @@ class TrajectorySaverCallback(AsyncCallbackHandler):content = result_item.get("content", [])if isinstance(content, list):for content_item in content:- if isinstance(content_item, dict) and content_item.get("type") == "input_image":+ if (+ isinstance(content_item, dict)+ and content_item.get("type") == "input_image"+ ):image_url = content_item.get("image_url", "")if isinstance(image_url, str) and image_url.startswith("data:"):try:@@ -618,7 +630,12 @@ class TrajectorySaverCallback(AsyncCallbackHandler):image_bytes = base64.b64decode(b64_payload)# If we have coordinates, draw crosshair annotation- if x_coord is not None and y_coord is not None and x_coord != 0 and y_coord != 0:+ if (+ x_coord is not None+ and y_coord is not None+ and x_coord != 0+ and y_coord != 0+ ):annotated_image = self._draw_crosshair_on_image(image_bytes, int(x_coord), int(y_coord))diff --git a/libs/python/agent/agent/loops/gemini.py b/libs/python/agent/agent/loops/gemini.pyindex 9df2127b2d..37c45dcad8 100644--- a/libs/python/agent/agent/loops/gemini.py+++ b/libs/python/agent/agent/loops/gemini.py@@ -89,11 +89,7 @@ def _sanitize_for_json(obj: Any) -> Any:return _sanitize_for_json(obj.model_dump())# Handle objects with __dict__ (like Gemini SDK response objects)if hasattr(obj, "__dict__"):- return {- k: _sanitize_for_json(v)- for k, v in obj.__dict__.items()- if not k.startswith("__")- }+ return {k: _sanitize_for_json(v) for k, v in obj.__dict__.items() if not k.startswith("__")}# Fallback to string representationreturn str(obj)@@ -828,9 +824,7 @@ class GeminiComputerUseConfig(AsyncAgentConfig):}if _on_api_start:- await _on_api_start(- _sanitize_for_json(api_kwargs)- )+ await _on_api_start(_sanitize_for_json(api_kwargs))response = client.models.generate_content(**api_kwargs)diff --git a/libs/python/agent/agent/loops/openai.py b/libs/python/agent/agent/loops/openai.pyindex e013e746e8..5648e67084 100644--- a/libs/python/agent/agent/loops/openai.py+++ b/libs/python/agent/agent/loops/openai.py@@ -15,7 +15,9 @@ from ..decorators import register_agentfrom ..types import AgentCapability, AgentResponse, Messages, Tools-async def _map_computer_tool_to_openai(computer_handler: Any, use_native_tool: bool = True) -> Dict[str, Any]:+async def _map_computer_tool_to_openai(+ computer_handler: Any, use_native_tool: bool = True+) -> Dict[str, Any]:"""Map a computer tool to OpenAI's tool schema.Args:@@ -136,6 +138,7 @@ async def _map_computer_tool_to_openai(computer_handler: Any, use_native_tool: bdef _is_native_computer_use_model(model: str) -> bool:"""Check if the model supports native computer_use_preview tool format."""import re+# Only computer-use-preview models support native computer_use_preview tool# GPT 5.4 does NOT support computer_use_preview - it uses function callingreturn bool(re.search(r"computer-use-preview", model, re.IGNORECASE))@@ -154,18 +157,22 @@ async def _prepare_tools_for_openai(tool_schemas: List[Dict[str, Any]], model: sfor schema in tool_schemas:if schema["type"] == "computer":# Map computer tool to OpenAI format (native or function based on model)- computer_tool = await _map_computer_tool_to_openai(schema["computer"], use_native_tool=use_native)+ computer_tool = await _map_computer_tool_to_openai(+ schema["computer"], use_native_tool=use_native+ )openai_tools.append(computer_tool)elif schema["type"] == "function":# Function tools for Responses API need: {type, name, description, parameters}# Note: parameters are at the root level, NOT nested under 'function'func = schema["function"]- openai_tools.append({- "type": "function",- "name": func["name"],- "description": func.get("description", ""),- "parameters": func.get("parameters", {}),- })+ openai_tools.append(+ {+ "type": "function",+ "name": func["name"],+ "description": func.get("description", ""),+ "parameters": func.get("parameters", {}),+ }+ )return openai_tools@@ -387,10 +394,7 @@ Task: Click {instruction}. Output ONLY a click action on the target element.""",continue# Native format: computer_call with action dict- if (- item.get("type") == "computer_call"- and isinstance(item.get("action"), dict)- ):+ if item.get("type") == "computer_call" and isinstance(item.get("action"), dict):action = item["action"]if action.get("x") is not None and action.get("y") is not None:return (int(action.get("x")), int(action.get("y")))diff --git a/libs/python/agent/agent/loops/yutori.py b/libs/python/agent/agent/loops/yutori.pyindex 861fcdf453..0b38e0671e 100644--- a/libs/python/agent/agent/loops/yutori.py+++ b/libs/python/agent/agent/loops/yutori.py@@ -370,16 +370,12 @@ class YutoriN1Config(AsyncAgentConfig):}],}- output_items.extend(- convert_completion_messages_to_responses_items([fake_cm])- )+ output_items.extend(convert_completion_messages_to_responses_items([fake_cm]))# Only use content_text oncecontent_text = ""else:# Custom tool — emit as function_call- output_items.append(- make_function_call_item(fn_name, args, call_id=tc_id)- )+ output_items.append(make_function_call_item(fn_name, args, call_id=tc_id))else:# No tool calls — task is completeif content_text:diff --git a/libs/python/agent/agent/tools/browser_tool.py b/libs/python/agent/agent/tools/browser_tool.pyindex 95ff43fcb7..08b04f4424 100644--- a/libs/python/agent/agent/tools/browser_tool.py+++ b/libs/python/agent/agent/tools/browser_tool.py@@ -188,7 +188,11 @@ class BrowserTool(BaseComputerTool):],"type": "string",},- "keys": {"description": "Required only by action=key.", "type": "array", "items": {"type": "string"}},+ "keys": {+ "description": "Required only by action=key.",+ "type": "array",+ "items": {"type": "string"},+ },"text": {"description": "Required only by action=type.", "type": "string"},"coordinate": {"description": "(x, y) coordinates for mouse actions. Required only by action=left_click, action=mouse_move, and action=type.",diff --git a/libs/python/computer-server/computer_server/cli.py b/libs/python/computer-server/computer_server/cli.pyindex aa68351321..3ea0c76f0d 100644--- a/libs/python/computer-server/computer_server/cli.py+++ b/libs/python/computer-server/computer_server/cli.py@@ -10,8 +10,6 @@ import sysimport threadingfrom typing import List, Optional-from .server import Server-logger = logging.getLogger(__name__)@@ -72,6 +70,33 @@ def parse_args(args: Optional[List[str]] = None) -> argparse.Namespace:help="Disable automatic server restart in watchdog",)+ # Backend selection+ parser.add_argument(+ "--backend",+ choices=["native", "vnc"],+ default="native",+ help="Handler backend: 'native' uses OS-specific handlers, 'vnc' uses VNC (default: native)",+ )++ # VNC backend options+ parser.add_argument(+ "--vnc-host",+ type=str,+ help="VNC server host (required when --backend=vnc)",+ )+ parser.add_argument(+ "--vnc-port",+ type=int,+ default=5900,+ help="VNC server port (default: 5900)",+ )+ parser.add_argument(+ "--vnc-password",+ type=str,+ default="",+ help="VNC server password",+ )+return parser.parse_args(args)@@ -86,6 +111,21 @@ def main() -> None:stream=sys.stderr, # Use stderr for MCP compatibility)+ # Set backend env vars from CLI args before Server import triggers handler creation+ if args.backend == "vnc" or args.vnc_host:+ if not args.vnc_host and not os.environ.get("CUA_VNC_HOST"):+ parser_err = "--vnc-host is required when using --backend=vnc"+ logger.error(parser_err)+ sys.exit(1)+ os.environ["CUA_BACKEND"] = "vnc"+ if args.vnc_host:+ os.environ["CUA_VNC_HOST"] = args.vnc_host+ os.environ["CUA_VNC_PORT"] = str(args.vnc_port)+ if args.vnc_password:+ os.environ["CUA_VNC_PASSWORD"] = args.vnc_password+ vnc_host = args.vnc_host or os.environ.get("CUA_VNC_HOST")+ logger.info(f"VNC backend enabled → {vnc_host}:{args.vnc_port}")+# Check if watchdog should be enabledcontainer_name = os.environ.get("CONTAINER_NAME")enable_watchdog = (args.watchdog or bool(container_name)) and not sys.platform.startswith("win")@@ -151,6 +191,10 @@ def main() -> None:else:logger.info("HTTP mode (no SSL certificates provided)")+ # Import Server lazily so env vars (e.g. CUA_VNC_HOST) are set before+ # the module-level handler factory runs in main.py.+ from .server import Server+server = Server(host=args.host, port=args.port, log_level=args.log_level, **ssl_args)try:diff --git a/libs/python/computer-server/computer_server/handlers/base.py b/libs/python/computer-server/computer_server/handlers/base.pyindex d38ef553a0..833b21b416 100644--- a/libs/python/computer-server/computer_server/handlers/base.py+++ b/libs/python/computer-server/computer_server/handlers/base.py@@ -1,6 +1,11 @@+import asyncio+import logging+import sysfrom abc import ABC, abstractmethodfrom typing import Any, Dict, List, Optional, Tuple+logger = logging.getLogger(__name__)+class BaseAccessibilityHandler(ABC):"""Abstract base class for OS-specific accessibility handlers."""@@ -322,17 +327,54 @@ class BaseAutomationHandler(ABC):pass# Clipboard Actions- @abstractmethodasync def copy_to_clipboard(self) -> Dict[str, Any]:- """Get the current clipboard content."""- pass+ """Get the current clipboard content using pyperclip."""+ try:+ import pyperclip++ content = pyperclip.paste()+ return {"success": True, "content": content}+ except Exception as e:+ return {"success": False, "error": str(e)}- @abstractmethodasync def set_clipboard(self, text: str) -> Dict[str, Any]:- """Set the clipboard content."""- pass+ """Set the clipboard content using pyperclip."""+ try:+ import pyperclip- @abstractmethod+ pyperclip.copy(text)+ return {"success": True}+ except Exception as e:+ return {"success": False, "error": str(e)}++ # Command Executionasync def run_command(self, command: str) -> Dict[str, Any]:- """Run a command and return the output."""- pass+ """Run a shell command locally and return its output.++ When IS_CUA_ANDROID env var is set, routes the command through+ ``adb shell`` so execution runs inside the Android emulator.+ """+ import os++ try:+ if os.environ.get("IS_CUA_ANDROID") == "true":+ process = await asyncio.create_subprocess_exec(+ "adb",+ "shell",+ command,+ stdout=asyncio.subprocess.PIPE,+ stderr=asyncio.subprocess.PIPE,+ )+ else:+ process = await asyncio.create_subprocess_shell(+ command, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE+ )+ stdout, stderr = await process.communicate()+ return {+ "success": True,+ "stdout": stdout.decode() if stdout else "",+ "stderr": stderr.decode() if stderr else "",+ "return_code": process.returncode,+ }+ except Exception as e:+ return {"success": False, "error": str(e)}diff --git a/libs/python/computer-server/computer_server/handlers/factory.py b/libs/python/computer-server/computer_server/handlers/factory.pyindex 26a0792d5f..00029fa10c 100644--- a/libs/python/computer-server/computer_server/handlers/factory.py+++ b/libs/python/computer-server/computer_server/handlers/factory.py@@ -1,3 +1,5 @@+import logging+import osfrom typing import Tuplefrom computer_server.diorama.base import BaseDioramaHandler@@ -11,6 +13,8 @@ from .base import (BaseWindowHandler,)+logger = logging.getLogger(__name__)+OS_TYPE = get_current_os()if OS_TYPE == "android":@@ -55,7 +59,28 @@ class HandlerFactory:NotImplementedError: If the current OS is not supportedRuntimeError: If unable to determine the current OS"""- if OS_TYPE == "android":+ backend = os.environ.get("CUA_BACKEND", "native")+ vnc_host = os.environ.get("CUA_VNC_HOST")+ if backend == "vnc" or vnc_host:+ if not vnc_host:+ raise RuntimeError(+ "CUA_VNC_HOST must be set when using VNC backend "+ "(--backend=vnc requires --vnc-host)"+ )+ from .vnc import VNCAccessibilityHandler, VNCAutomationHandler++ vnc_port = int(os.environ.get("CUA_VNC_PORT", "5900"))+ vnc_password = os.environ.get("CUA_VNC_PASSWORD", "")+ logger.info(f"Using VNC backend → {vnc_host}:{vnc_port}")+ return (+ VNCAccessibilityHandler(),+ VNCAutomationHandler(host=vnc_host, port=vnc_port, password=vnc_password),+ BaseDioramaHandler(),+ GenericFileHandler(),+ GenericDesktopHandler(),+ GenericWindowHandler(),+ )+ elif OS_TYPE == "android":return (AndroidAccessibilityHandler(),AndroidAutomationHandler(),diff --git a/libs/python/computer-server/computer_server/handlers/linux.py b/libs/python/computer-server/computer_server/handlers/linux.pyindex af127ce0f6..261189ede3 100644--- a/libs/python/computer-server/computer_server/handlers/linux.py+++ b/libs/python/computer-server/computer_server/handlers/linux.py@@ -537,73 +537,4 @@ class LinuxAutomationHandler(BaseAutomationHandler):except Exception as e:return {"success": False, "error": str(e)}- # Clipboard Actions- async def copy_to_clipboard(self) -> Dict[str, Any]:- """Get the current content of the clipboard.-- Returns:- Dict[str, Any]: A dictionary containing success status and clipboard content,- or error message if failed.- """- try:- import pyperclip-- content = pyperclip.paste()- return {"success": True, "content": content}- except Exception as e:- return {"success": False, "error": str(e)}-- async def set_clipboard(self, text: str) -> Dict[str, Any]:- """Set the clipboard content to the specified text.-- Args:- text: The text to copy to the clipboard.-- Returns:- Dict[str, Any]: A dictionary with success status and error message if failed.- """- try:- import pyperclip-- pyperclip.copy(text)- return {"success": True}- except Exception as e:- return {"success": False, "error": str(e)}-- # Command Execution- async def run_command(self, command: str) -> Dict[str, Any]:- """Execute a shell command asynchronously.-- When IS_CUA_ANDROID is set, routes the command through `adb shell`- so execution runs inside the Android emulator.-- Args:- command: The shell command to execute.-- Returns:- Dict[str, Any]: A dictionary containing success status, stdout, stderr,- and return code, or error message if failed.- """- try:- if os.environ.get("IS_CUA_ANDROID") == "true":- # Route through adb shell to run inside the emulator- process = await asyncio.create_subprocess_exec(- "adb",- "shell",- command,- stdout=asyncio.subprocess.PIPE,- stderr=asyncio.subprocess.PIPE,- )- else:- process = await asyncio.create_subprocess_shell(- command, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE- )- stdout, stderr = await process.communicate()- return {- "success": True,- "stdout": stdout.decode() if stdout else "",- "stderr": stderr.decode() if stderr else "",- "return_code": process.returncode,- }- except Exception as e:- return {"success": False, "error": str(e)}+ # Clipboard and run_command inherited from BaseAutomationHandlerdiff --git a/libs/python/computer-server/computer_server/handlers/macos.py b/libs/python/computer-server/computer_server/handlers/macos.pyindex 56ba5a672d..3846508b33 100644--- a/libs/python/computer-server/computer_server/handlers/macos.py+++ b/libs/python/computer-server/computer_server/handlers/macos.py@@ -1408,66 +1408,4 @@ class MacOSAutomationHandler(BaseAutomationHandler):except Exception as e:return {"success": False, "error": str(e)}- # Clipboard Actions- async def copy_to_clipboard(self) -> Dict[str, Any]:- """Get the current content of the system clipboard.-- Returns:- Dictionary containing success status and clipboard content or error message- """- try:- import pyperclip-- content = pyperclip.paste()- return {"success": True, "content": content}- except Exception as e:- return {"success": False, "error": str(e)}-- async def set_clipboard(self, text: str) -> Dict[str, Any]:- """Set the content of the system clipboard.-- Args:- text: Text to copy to the clipboard-- Returns:- Dictionary containing success status and error message if failed- """- try:- import pyperclip-- pyperclip.copy(text)- return {"success": True}- except Exception as e:- return {"success": False, "error": str(e)}-- async def run_command(self, command: str) -> Dict[str, Any]:- """Run a shell command and return its output.-- Args:- command: Shell command to execute-- Returns:- Dictionary containing success status, stdout, stderr, and return code- """- try:- if os.environ.get("IS_CUA_ANDROID") == "true":- process = await asyncio.create_subprocess_exec(- "adb",- "shell",- command,- stdout=asyncio.subprocess.PIPE,- stderr=asyncio.subprocess.PIPE,- )- else:- process = await asyncio.create_subprocess_shell(- command, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE- )- stdout, stderr = await process.communicate()- return {- "success": True,- "stdout": stdout.decode() if stdout else "",- "stderr": stderr.decode() if stderr else "",- "return_code": process.returncode,- }- except Exception as e:- return {"success": False, "error": str(e)}+ # Clipboard and run_command inherited from BaseAutomationHandlerdiff --git a/libs/python/computer-server/computer_server/handlers/vnc.py b/libs/python/computer-server/computer_server/handlers/vnc.pynew file mode 100644index 0000000000..c691e01ab2--- /dev/null+++ b/libs/python/computer-server/computer_server/handlers/vnc.py@@ -0,0 +1,499 @@+"""+VNC backend for automation and accessibility handlers.++A cross-platform alternative to the native (OS-specific) handlers. Connects to+any VNC server to perform screenshots, mouse, keyboard, and scroll operations+over the RFB protocol. Works with Linux, macOS, and Windows targets — the only+requirement is a reachable VNC server.++Built on vncdotool (Twisted-based RFB client). The Twisted reactor runs in a+background daemon thread; all handler methods bridge from asyncio via+`asyncio.to_thread`.++Usage:+ # Via CLI+ python -m computer_server --vnc-host 192.168.64.1 --vnc-port 5900 --vnc-password secret++ # Via env vars+ CUA_VNC_HOST=192.168.64.1 CUA_VNC_PORT=5900 CUA_VNC_PASSWORD=secret python -m computer_server+"""++import asyncio+import base64+import logging+import threading+from io import BytesIO+from typing import Any, Dict, List, Optional, Tuple++from .base import BaseAccessibilityHandler, BaseAutomationHandler++logger = logging.getLogger(__name__)++# Key name aliases: map cua-computer-server key names → vncdotool key names.+# vncdotool's KEYMAP already covers most X11 keysym names; these bridge+# the naming differences used by the rest of the computer-server API.+_KEY_ALIASES = {+ "return": "enter",+ "escape": "esc",+ "backspace": "bsp",+ "delete": "del",+ "page_up": "pgup",+ "pageup": "pgup",+ "page_down": "pgdn",+ "pagedown": "pgdn",+ "insert": "ins",+ "caps_lock": "caplk",+ "num_lock": "numlk",+ "shift_l": "lshift",+ "shift_r": "rshift",+ "ctrl_l": "lctrl",+ "ctrl_r": "rctrl",+ "control": "ctrl",+ "control_l": "lctrl",+ "control_r": "rctrl",+ "alt_l": "lalt",+ "alt_r": "ralt",+ "meta_l": "lmeta",+ "meta_r": "rmeta",+ "super_l": "lsuper",+ "super_r": "rsuper",+ # macOS-specific names+ "command": "alt",+ "cmd": "alt",+ "option": "meta",+ "option_l": "lmeta",+ "option_r": "rmeta",+}++# Button name → vncdotool button number (1-indexed)+_BUTTON_MAP = {"left": 1, "middle": 2, "right": 3}+++def _translate_key(key: str) -> str:+ """Translate a key name to vncdotool's KEYMAP name."""+ lower = key.lower()+ return _KEY_ALIASES.get(lower, lower)+++# ---------------------------------------------------------------------------+# Lazy vncdotool client wrapper+# ---------------------------------------------------------------------------+++class _VNCConnection:+ """Manages a vncdotool ThreadedVNCClientProxy lifecycle.++ All methods are synchronous (blocking) — the handler calls them via+ asyncio.to_thread to avoid blocking the event loop.+ """++ def __init__(self, host: str, port: int, password: str = ""):+ self._host = host+ self._port = port+ self._password = password+ self._client = None+ self._lock = threading.Lock()+ # Track cursor position locally (vncdotool also tracks via client.x/y)+ self._cursor_x: int = 0+ self._cursor_y: int = 0++ def _ensure_connected(self):+ """Lazily connect to the VNC server. Returns the vncdotool client."""+ if self._client is not None:+ return self._client+ with self._lock:+ if self._client is not None:+ return self._client+ from vncdotool import api++ server_str = f"{self._host}::{self._port}"+ logger.info(f"VNC connecting to {self._host}:{self._port}")+ client = api.connect(+ server_str,+ password=self._password or None,+ )+ client.timeout = 30+ self._client = client+ logger.info("VNC connected")+ return client++ def disconnect(self):+ with self._lock:+ if self._client is not None:+ try:+ self._client.disconnect()+ except Exception:+ pass+ self._client = None++ def _reset_on_error(self):+ """Disconnect so next call reconnects."""+ try:+ self.disconnect()+ except Exception:+ pass++ # -- Screenshot ---------------------------------------------------------++ def capture_screenshot(self) -> bytes:+ """Capture the screen and return PNG bytes."""+ client = self._ensure_connected()+ # refreshScreen updates the internal screen image+ client.refreshScreen(incremental=False)+ screen = client.protocol.screen+ buf = BytesIO()+ screen.save(buf, format="PNG")+ return buf.getvalue()++ # -- Mouse --------------------------------------------------------------++ def mouse_move(self, x: int, y: int):+ client = self._ensure_connected()+ client.mouseMove(x, y)+ self._cursor_x = x+ self._cursor_y = y++ def mouse_click(self, x: int, y: int, button: int = 1, clicks: int = 1):+ client = self._ensure_connected()+ client.mouseMove(x, y)+ self._cursor_x = x+ self._cursor_y = y+ for _ in range(clicks):+ client.mousePress(button)++ def mouse_down(self, x: int, y: int, button: int = 1):+ client = self._ensure_connected()+ client.mouseMove(x, y)+ self._cursor_x = x+ self._cursor_y = y+ client.mouseDown(button)++ def mouse_up(self, x: int, y: int, button: int = 1):+ client = self._ensure_connected()+ client.mouseMove(x, y)+ self._cursor_x = x+ self._cursor_y = y+ client.mouseUp(button)++ def mouse_drag(self, x: int, y: int, step: int = 1):+ client = self._ensure_connected()+ client.mouseDrag(x, y, step=step)+ self._cursor_x = x+ self._cursor_y = y++ def scroll(self, x: int, y: int):+ """Scroll. y>0 = down (button 5), y<0 = up (button 4)."""+ client = self._ensure_connected()+ if y > 0:+ for _ in range(y):+ client.mousePress(5)+ elif y < 0:+ for _ in range(abs(y)):+ client.mousePress(4)+ # Horizontal: button 6 (right), button 7 (left) — not universally supported+ if x > 0:+ for _ in range(x):+ client.mousePress(6)+ elif x < 0:+ for _ in range(abs(x)):+ client.mousePress(7)++ # -- Keyboard -----------------------------------------------------------++ def key_press(self, key: str):+ client = self._ensure_connected()+ client.keyPress(_translate_key(key))++ def key_down(self, key: str):+ client = self._ensure_connected()+ client.keyDown(_translate_key(key))++ def key_up(self, key: str):+ client = self._ensure_connected()+ client.keyUp(_translate_key(key))++ def type_text(self, text: str):+ """Type text character by character, handling shift for uppercase/symbols."""+ client = self._ensure_connected()+ for ch in text:+ if ch == " ":+ client.keyPress("space")+ elif ch == "\n":+ client.keyPress("enter")+ elif ch == "\t":+ client.keyPress("tab")+ else:+ # vncdotool's keyPress handles single characters directly+ client.keyPress(ch)++ def hotkey(self, keys: List[str]):+ """Press a key combination (e.g. ['command', 'a'])."""+ client = self._ensure_connected()+ translated = [_translate_key(k) for k in keys]+ # vncdotool supports "modifier-key" syntax: "alt-a", "ctrl-shift-c"+ combo = "-".join(translated)+ client.keyPress(combo)++ # -- Clipboard ----------------------------------------------------------++ def paste_text(self, text: str):+ client = self._ensure_connected()+ client.paste(text)++ # -- Info ---------------------------------------------------------------++ @property+ def screen_size(self) -> Tuple[int, int]:+ client = self._ensure_connected()+ proto = client.protocol+ if proto is not None and proto.screen is not None:+ return proto.screen.size+ return (0, 0)++ @property+ def cursor_position(self) -> Tuple[int, int]:+ return self._cursor_x, self._cursor_y+++# ---------------------------------------------------------------------------+# Automation Handler+# ---------------------------------------------------------------------------+++class VNCAutomationHandler(BaseAutomationHandler):+ """Cross-platform automation handler that operates via VNC/RFB protocol.++ Works with any OS target (Linux, macOS, Windows) that has a VNC server.+ All operations go through the network, bypassing local permission systems.+ """++ def __init__(+ self,+ host: str = "127.0.0.1",+ port: int = 5900,+ password: str = "",+ ):+ self._conn = _VNCConnection(host, port, password)++ def _resolve_coords(self, x: Optional[int], y: Optional[int]) -> Tuple[int, int]:+ if x is not None and y is not None:+ return x, y+ return self._conn.cursor_position++ # -- Screenshot ---------------------------------------------------------++ async def screenshot(self) -> Dict[str, Any]:+ try:+ png_bytes = await asyncio.to_thread(self._conn.capture_screenshot)+ image_data = base64.b64encode(png_bytes).decode()+ return {"success": True, "image_data": image_data}+ except Exception as e:+ logger.error(f"VNC screenshot error: {e}")+ self._conn._reset_on_error()+ return {"success": False, "error": f"VNC screenshot error: {e}"}++ # -- Mouse actions ------------------------------------------------------++ async def left_click(self, x: Optional[int] = None, y: Optional[int] = None) -> Dict[str, Any]:+ try:+ cx, cy = self._resolve_coords(x, y)+ await asyncio.to_thread(self._conn.mouse_click, cx, cy, 1)+ return {"success": True}+ except Exception as e:+ return {"success": False, "error": str(e)}++ async def right_click(self, x: Optional[int] = None, y: Optional[int] = None) -> Dict[str, Any]:+ try:+ cx, cy = self._resolve_coords(x, y)+ await asyncio.to_thread(self._conn.mouse_click, cx, cy, 3)+ return {"success": True}+ except Exception as e:+ return {"success": False, "error": str(e)}++ async def middle_click(+ self, x: Optional[int] = None, y: Optional[int] = None+ ) -> Dict[str, Any]:+ try:+ cx, cy = self._resolve_coords(x, y)+ await asyncio.to_thread(self._conn.mouse_click, cx, cy, 2)+ return {"success": True}+ except Exception as e:+ return {"success": False, "error": str(e)}++ async def double_click(+ self, x: Optional[int] = None, y: Optional[int] = None+ ) -> Dict[str, Any]:+ try:+ cx, cy = self._resolve_coords(x, y)+ await asyncio.to_thread(self._conn.mouse_click, cx, cy, 1, 2)+ return {"success": True}+ except Exception as e:+ return {"success": False, "error": str(e)}++ async def move_cursor(self, x: int, y: int) -> Dict[str, Any]:+ try:+ await asyncio.to_thread(self._conn.mouse_move, x, y)+ return {"success": True}+ except Exception as e:+ return {"success": False, "error": str(e)}++ async def mouse_down(+ self, x: Optional[int] = None, y: Optional[int] = None, button: str = "left"+ ) -> Dict[str, Any]:+ try:+ cx, cy = self._resolve_coords(x, y)+ btn = _BUTTON_MAP.get(button, 1)+ await asyncio.to_thread(self._conn.mouse_down, cx, cy, btn)+ return {"success": True}+ except Exception as e:+ return {"success": False, "error": str(e)}++ async def mouse_up(+ self, x: Optional[int] = None, y: Optional[int] = None, button: str = "left"+ ) -> Dict[str, Any]:+ try:+ cx, cy = self._resolve_coords(x, y)+ btn = _BUTTON_MAP.get(button, 1)+ await asyncio.to_thread(self._conn.mouse_up, cx, cy, btn)+ return {"success": True}+ except Exception as e:+ return {"success": False, "error": str(e)}++ async def drag_to(+ self, x: int, y: int, button: str = "left", duration: float = 0.5+ ) -> Dict[str, Any]:+ try:+ btn = _BUTTON_MAP.get(button, 1)+ cx, cy = self._conn.cursor_position++ def _drag():+ self._conn.mouse_down(cx, cy, btn)+ step = max(1, int(1 / max(duration, 0.01) * 5))+ self._conn.mouse_drag(x, y, step=step)+ self._conn.mouse_up(x, y, btn)++ await asyncio.to_thread(_drag)+ return {"success": True}+ except Exception as e:+ return {"success": False, "error": str(e)}++ async def drag(+ self, path: List[Tuple[int, int]], button: str = "left", duration: float = 0.5+ ) -> Dict[str, Any]:+ try:+ if not path:+ return {"success": False, "error": "Empty path"}+ btn = _BUTTON_MAP.get(button, 1)++ def _drag_path():+ self._conn.mouse_down(path[0][0], path[0][1], btn)+ for px, py in path[1:]:+ self._conn.mouse_move(px, py)+ self._conn.mouse_up(path[-1][0], path[-1][1], btn)++ await asyncio.to_thread(_drag_path)+ return {"success": True}+ except Exception as e:+ return {"success": False, "error": str(e)}++ # -- Keyboard actions ---------------------------------------------------++ async def type_text(self, text: str) -> Dict[str, Any]:+ try:+ await asyncio.to_thread(self._conn.type_text, text)+ return {"success": True}+ except Exception as e:+ return {"success": False, "error": str(e)}++ async def press_key(self, key: str) -> Dict[str, Any]:+ try:+ await asyncio.to_thread(self._conn.key_press, key)+ return {"success": True}+ except Exception as e:+ return {"success": False, "error": str(e)}++ async def key_down(self, key: str) -> Dict[str, Any]:+ try:+ await asyncio.to_thread(self._conn.key_down, key)+ return {"success": True}+ except Exception as e:+ return {"success": False, "error": str(e)}++ async def key_up(self, key: str) -> Dict[str, Any]:+ try:+ await asyncio.to_thread(self._conn.key_up, key)+ return {"success": True}+ except Exception as e:+ return {"success": False, "error": str(e)}++ async def hotkey(self, keys: List[str]) -> Dict[str, Any]:+ try:+ await asyncio.to_thread(self._conn.hotkey, keys)+ return {"success": True}+ except Exception as e:+ return {"success": False, "error": str(e)}++ # -- Scrolling ----------------------------------------------------------++ async def scroll(self, x: int, y: int) -> Dict[str, Any]:+ try:+ await asyncio.to_thread(self._conn.scroll, x, y)+ return {"success": True}+ except Exception as e:+ return {"success": False, "error": str(e)}++ async def scroll_down(self, clicks: int = 1) -> Dict[str, Any]:+ return await self.scroll(0, clicks)++ async def scroll_up(self, clicks: int = 1) -> Dict[str, Any]:+ return await self.scroll(0, -clicks)++ # -- Screen info --------------------------------------------------------++ async def get_screen_size(self) -> Dict[str, Any]:+ try:+ w, h = await asyncio.to_thread(lambda: self._conn.screen_size)+ return {"success": True, "size": {"width": w, "height": h}}+ except Exception as e:+ return {"success": False, "error": str(e)}++ async def get_cursor_position(self) -> Dict[str, Any]:+ try:+ x, y = self._conn.cursor_position+ return {"success": True, "position": {"x": x, "y": y}}+ except Exception as e:+ return {"success": False, "error": str(e)}++ # Clipboard and run_command inherited from BaseAutomationHandler+++# ---------------------------------------------------------------------------+# Accessibility Handler (stub)+# ---------------------------------------------------------------------------+++class VNCAccessibilityHandler(BaseAccessibilityHandler):+ """Stub accessibility handler for VNC — no accessibility tree available."""++ async def get_accessibility_tree(self) -> Dict[str, Any]:+ return {+ "success": True,+ "tree": {+ "role": "Window",+ "title": "VNC Remote Desktop",+ "position": {"x": 0, "y": 0},+ "size": {"width": 0, "height": 0},+ "children": [],+ },+ }++ async def find_element(+ self,+ role: Optional[str] = None,+ title: Optional[str] = None,+ value: Optional[str] = None,+ ) -> Dict[str, Any]:+ return {+ "success": False,+ "error": "Accessibility tree not available over VNC",+ }diff --git a/libs/python/computer-server/computer_server/handlers/windows.py b/libs/python/computer-server/computer_server/handlers/windows.pyindex fb6ec6c85c..c4af537a84 100644--- a/libs/python/computer-server/computer_server/handlers/windows.py+++ b/libs/python/computer-server/computer_server/handlers/windows.py@@ -695,42 +695,9 @@ class WindowsAutomationHandler(BaseAutomationHandler):except Exception as e:return {"success": False, "error": str(e)}- # Clipboard Actions- async def copy_to_clipboard(self) -> Dict[str, Any]:- """Get the current content of the clipboard.+ # Clipboard inherited from BaseAutomationHandler- Returns:- Dict[str, Any]: A dictionary containing the success status and either- clipboard content or an error message.- Structure: {"success": bool, "content": str} or- {"success": bool, "error": str}- """- try:- import pyperclip-- content = pyperclip.paste()- return {"success": True, "content": content}- except Exception as e:- return {"success": False, "error": str(e)}-- async def set_clipboard(self, text: str) -> Dict[str, Any]:- """Set the clipboard content to the specified text.-- Args:- text (str): The text to copy to the clipboard.-- Returns:- Dict[str, Any]: A dictionary with success status and optional error message.- """- try:- import pyperclip-- pyperclip.copy(text)- return {"success": True}- except Exception as e:- return {"success": False, "error": str(e)}-- # Command Execution+ # Command Execution (Windows override for multi-encoding support)async def run_command(self, command: str) -> Dict[str, Any]:"""Execute a shell command asynchronously.diff --git a/libs/python/computer-server/pyproject.toml b/libs/python/computer-server/pyproject.tomlindex a1f1dfc67b..89eae4a580 100644--- a/libs/python/computer-server/pyproject.toml+++ b/libs/python/computer-server/pyproject.toml@@ -49,6 +49,9 @@ linux = [windows = ["pywin32>=310"]+vnc = [+ "vncdotool>=1.2.0"+][project.urls]homepage = "https://github.com/trycua/cua"diff --git a/libs/python/computer/computer/computer.py b/libs/python/computer/computer/computer.pyindex 599b8dbfd9..f24f87b966 100644--- a/libs/python/computer/computer/computer.py+++ b/libs/python/computer/computer/computer.py@@ -112,6 +112,10 @@ class Computer:api_key: Optional[str] = None,experiments: Optional[List[str]] = None,timeout: int = 100,+ backend: Literal["native", "vnc"] = "native",+ vnc_host: Optional[str] = None,+ vnc_port: int = 5900,+ vnc_password: str = "",run_opts: Optional[Dict[str, Any]] = None,):"""Initialize a new Computer instance.@@ -143,6 +147,10 @@ class Computer:api_key: Optional API key for cloud providers (defaults to CUA_API_KEY environment variable)experiments: Optional list of experimental features to enable (e.g. ["app-use"])timeout: Timeout in seconds for connecting to the computer interface+ backend: Handler backend for the computer-server: 'native' (OS-specific) or 'vnc'. Defaults to 'native'.+ vnc_host: VNC server host (required when backend='vnc')+ vnc_port: VNC server port (default: 5900)+ vnc_password: VNC server passwordrun_opts: Optional dictionary of provider-specific run options."""@@ -194,6 +202,10 @@ class Computer:self.api_port = 8000self.experiments = experiments or []+ self.backend = backend+ self.vnc_host = vnc_host+ self.vnc_port = vnc_port+ self.vnc_password = vnc_passwordif "app-use" in self.experiments:assert self.os_type == "macos", "App use experiment is only supported on macOS"@@ -532,6 +544,16 @@ class Computer:if self.shared_directories:run_opts["shared_directories"] = shared_dirs.copy()+ # Pass backend configuration as env vars for the computer-server+ if self.backend == "vnc":+ run_opts.setdefault("env", {})+ run_opts["env"]["CUA_BACKEND"] = "vnc"+ if self.vnc_host:+ run_opts["env"]["CUA_VNC_HOST"] = self.vnc_host+ run_opts["env"]["CUA_VNC_PORT"] = str(self.vnc_port)+ if self.vnc_password:+ run_opts["env"]["CUA_VNC_PASSWORD"] = self.vnc_password+# Merge custom run_optsrun_opts.update(self.custom_run_opts)diff --git a/libs/python/cua-cli/tests/commands/test_sandbox.py b/libs/python/cua-cli/tests/commands/test_sandbox.pyindex d53fdf64df..89c4ef6694 100644--- a/libs/python/cua-cli/tests/commands/test_sandbox.py+++ b/libs/python/cua-cli/tests/commands/test_sandbox.py@@ -476,7 +476,9 @@ class TestCmdShell:subparsers = parser.add_subparsers()sandbox.register_parser(subparsers)- args = parser.parse_args(["sb", "shell", "--cols", "120", "--rows", "40", "my-sandbox", "ls"])+ args = parser.parse_args(+ ["sb", "shell", "--cols", "120", "--rows", "40", "my-sandbox", "ls"]+ )assert args.cols == 120assert args.rows == 40assert args.name == "my-sandbox"