Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

feat(computer-server): add VNC backend with --backend flag (#1196)

Francesco Bonacci5d7f13c3feb4
diff --git a/docs/content/docs/cua/reference/agent-sdk/index.mdx b/docs/content/docs/cua/reference/agent-sdk/index.mdx
index 16ef2f1808..ece0060541 100644
--- a/docs/content/docs/cua/reference/agent-sdk/index.mdx
+++ b/docs/content/docs/cua/reference/agent-sdk/index.mdx
@@ -7,7 +7,7 @@ description: Python API reference for building computer-use agents
AUTO-GENERATED FILE - DO NOT EDIT DIRECTLY
Generated by: npx tsx scripts/docs-generators/python-sdk.ts
Source: libs/python/agent/agent
- Version: 0.7.30
+ Version: 0.7.36
*/}
import { Callout } from 'fumadocs-ui/components/callout';
@@ -17,7 +17,7 @@ import { VersionHeader } from '@/components/version-selector';
<VersionHeader
versions={[{"version":"0.7","href":"/cua/reference/agent-sdk","isCurrent":true},{"version":"0.6","href":"/cua/reference/agent-sdk/v0.6/api","isCurrent":false},{"version":"0.5","href":"/cua/reference/agent-sdk/v0.5/api","isCurrent":false},{"version":"0.4","href":"/cua/reference/agent-sdk/v0.4/api","isCurrent":false},{"version":"0.3","href":"/cua/reference/agent-sdk/v0.3/api","isCurrent":false}]}
currentVersion="0.7"
- fullVersion="0.7.30"
+ fullVersion="0.7.36"
packageName="cua-agent"
/>
@@ -1346,3 +1346,12 @@ async def on_computer_call_end(self, item: Dict[str, Any], result: List[Dict[str
Called when a computer call has completed.
Saves screenshots and computer call output.
+
+#### TrajectorySaverCallback.on_function_call_end
+
+```python
+async def on_function_call_end(self, item: Dict[str, Any], result: List[Dict[str, Any]]) -> None
+```
+
+Called when a function call has completed.
+Saves screenshots and function call output for GPT 5.4 / BrowserTool.
diff --git a/docs/content/docs/cua/reference/cli/index.mdx b/docs/content/docs/cua/reference/cli/index.mdx
index 165ac3ee38..f7fa87bf0b 100644
--- a/docs/content/docs/cua/reference/cli/index.mdx
+++ b/docs/content/docs/cua/reference/cli/index.mdx
@@ -7,7 +7,7 @@ description: Python API reference for the Cua command-line interface
AUTO-GENERATED FILE - DO NOT EDIT DIRECTLY
Generated by: npx tsx scripts/docs-generators/python-sdk.ts
Source: libs/python/cua-cli/cua_cli
- Version: 0.1.7
+ Version: 0.1.8
*/}
import { Callout } from 'fumadocs-ui/components/callout';
@@ -17,7 +17,7 @@ import { VersionHeader } from '@/components/version-selector';
<VersionHeader
versions={[{"version":"0.1","href":"/cua/reference/cli","isCurrent":true}]}
currentVersion="0.1"
- fullVersion="0.1.7"
+ fullVersion="0.1.8"
packageName="cua-cli"
/>
diff --git a/docs/content/docs/cua/reference/computer-sdk/index.mdx b/docs/content/docs/cua/reference/computer-sdk/index.mdx
index 89d2d7e013..40d15551f4 100644
--- a/docs/content/docs/cua/reference/computer-sdk/index.mdx
+++ b/docs/content/docs/cua/reference/computer-sdk/index.mdx
@@ -7,7 +7,7 @@ description: Python API reference for controlling virtual machines and computer
AUTO-GENERATED FILE - DO NOT EDIT DIRECTLY
Generated by: npx tsx scripts/docs-generators/python-sdk.ts
Source: libs/python/computer/computer
- Version: 0.5.16
+ Version: 0.5.17
*/}
import { Callout } from 'fumadocs-ui/components/callout';
@@ -17,7 +17,7 @@ import { VersionHeader } from '@/components/version-selector';
<VersionHeader
versions={[{"version":"0.5","href":"/cua/reference/computer-sdk","isCurrent":true},{"version":"0.4","href":"/cua/reference/computer-sdk/v0.4/api","isCurrent":false},{"version":"0.3","href":"/cua/reference/computer-sdk/v0.3/api","isCurrent":false}]}
currentVersion="0.5"
- fullVersion="0.5.16"
+ fullVersion="0.5.17"
packageName="cua-computer"
/>
@@ -41,7 +41,7 @@ Computer is the main class for interacting with the computer.
### Constructor
```python
-Computer(self, display: Union[Display, Dict[str, int], str] = '1024x768', memory: str = '8GB', cpu: str = '4', os_type: OSType = 'macos', name: str = '', image: Optional[str] = None, shared_directories: Optional[List[str]] = None, use_host_computer_server: bool = False, verbosity: Union[int, LogLevel] = logging.INFO, telemetry_enabled: bool = True, provider_type: Union[str, VMProviderType] = VMProviderType.LUME, provider_port: Optional[int] = 7777, noVNC_port: Optional[int] = 8006, api_port: Optional[int] = None, host: str = 'localhost', api_host: Optional[str] = None, storage: Optional[str] = None, ephemeral: bool = False, api_key: Optional[str] = None, experiments: Optional[List[str]] = None, timeout: int = 100, run_opts: Optional[Dict[str, Any]] = None)
+Computer(self, display: Union[Display, Dict[str, int], str] = '1024x768', memory: str = '8GB', cpu: str = '4', os_type: OSType = 'macos', name: str = '', image: Optional[str] = None, shared_directories: Optional[List[str]] = None, use_host_computer_server: bool = False, verbosity: Union[int, LogLevel] = logging.INFO, telemetry_enabled: bool = True, provider_type: Union[str, VMProviderType] = VMProviderType.LUME, provider_port: Optional[int] = 7777, noVNC_port: Optional[int] = 8006, api_port: Optional[int] = None, host: str = 'localhost', api_host: Optional[str] = None, storage: Optional[str] = None, ephemeral: bool = False, api_key: Optional[str] = None, experiments: Optional[List[str]] = None, timeout: int = 100, backend: Literal['native', 'vnc'] = 'native', vnc_host: Optional[str] = None, vnc_port: int = 5900, vnc_password: str = '', run_opts: Optional[Dict[str, Any]] = None)
```
### Attributes
@@ -61,6 +61,10 @@ Computer(self, display: Union[Display, Dict[str, int], str] = '1024x768', memory
| `api_key` | `Any` | |
| `timeout` | `Any` | |
| `experiments` | `Any` | |
+| `backend` | `Any` | |
+| `vnc_host` | `Any` | |
+| `vnc_port` | `Any` | |
+| `vnc_password` | `Any` | |
| `custom_run_opts` | `Any` | |
| `storage` | `Any` | |
| `shared_path` | `Any` | |
diff --git a/docs/content/docs/cuabench/reference/api.mdx b/docs/content/docs/cuabench/reference/api.mdx
index a6ebc9abf3..4258fd13bd 100644
--- a/docs/content/docs/cuabench/reference/api.mdx
+++ b/docs/content/docs/cuabench/reference/api.mdx
@@ -7,7 +7,7 @@ description: Python API reference for the desktop automation benchmarking framew
AUTO-GENERATED FILE - DO NOT EDIT DIRECTLY
Generated by: npx tsx scripts/docs-generators/python-sdk.ts
Source: libs/cua-bench/cua_bench
- Version: 0.2.4
+ Version: 0.2.6
*/}
import { Callout } from 'fumadocs-ui/components/callout';
@@ -17,7 +17,7 @@ import { VersionHeader } from '@/components/version-selector';
<VersionHeader
versions={[{"version":"0.2","href":"/cuabench/reference","isCurrent":true}]}
currentVersion="0.2"
- fullVersion="0.2.4"
+ fullVersion="0.2.6"
packageName="cua-bench"
/>
@@ -3595,7 +3595,7 @@ TaskRunner(self, agent_image: str = DEFAULT_AGENT_IMAGE, env_hostname: str = 'cu
#### TaskRunner.run_task
```python
-async def run_task(self, env_path: Path, task_index: int, env_type: str, golden_name: Optional[str] = None, agent: Optional[str] = None, agent_image: Optional[str] = None, agent_command: Optional[List[str]] = None, agent_import_path: Optional[str] = None, model: Optional[str] = None, max_steps: int = 100, oracle: bool = False, memory: str = '8G', cpus: str = '8', vnc_port: Optional[int] = None, api_port: Optional[int] = None, output_dir: Optional[str] = None, stream_agent_logs: bool = False, timeout: Optional[int] = None, cleanup_before: bool = True, remove_images_after: bool = False, provider_type: Optional[str] = None) -> TaskResult
+async def run_task(self, env_path: Path, task_index: int, env_type: str, golden_name: Optional[str] = None, agent: Optional[str] = None, agent_image: Optional[str] = None, agent_command: Optional[List[str]] = None, agent_import_path: Optional[str] = None, model: Optional[str] = None, max_steps: int = 100, oracle: bool = False, memory: str = '8G', cpus: str = '8', vnc_port: Optional[int] = None, api_port: Optional[int] = None, output_dir: Optional[str] = None, stream_agent_logs: bool = False, timeout: Optional[int] = None, cleanup_before: bool = True, remove_images_after: bool = False, provider_type: Optional[str] = None, dev_paths: Optional[List[str]] = None) -> TaskResult
```
Run a task with 2-container architecture.
diff --git a/libs/lume/scripts/setup-cua-computer.sh b/libs/lume/scripts/setup-cua-computer.sh
new file mode 100755
index 0000000000..c5d7f5aaa8
--- /dev/null
+++ b/libs/lume/scripts/setup-cua-computer.sh
@@ -0,0 +1,476 @@
+#!/bin/bash
+#
+# setup-cua-computer.sh
+# Makes a Lume macOS VM image cua-computer compatible
+#
+# This script runs ON the VM itself (not the host).
+# It installs cua-computer-server, cua-agent, Playwright, and configures
+# the VM for headless CUA operation (auto-login, disable sleep, LaunchAgent).
+#
+# Usage:
+# lume ssh <vm-name> "curl -fsSL <url> | bash"
+# lume ssh <vm-name> < setup-cua-computer.sh
+# # Or copy and run directly inside the VM:
+# ./setup-cua-computer.sh [--yes] [--port PORT]
+#
+# Prerequisites:
+# - macOS VM created via lume (with SSH enabled, user lume/lume)
+#
+
+set -e
+
+# ============================================
+# Configuration
+# ============================================
+
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+YELLOW='\033[1;33m'
+BLUE='\033[0;34m'
+NC='\033[0m'
+
+AUTO_YES=false
+CUA_SERVER_PORT=8443
+VM_PASSWORD="${VM_PASSWORD:-lume}"
+CUA_DIR="$HOME/.cua-server"
+
+# Parse arguments
+while [ "$#" -gt 0 ]; do
+ case "$1" in
+ --yes|-y)
+ AUTO_YES=true
+ ;;
+ --port)
+ CUA_SERVER_PORT="$2"
+ shift
+ ;;
+ --help)
+ echo "Usage: $0 [OPTIONS]"
+ echo ""
+ echo "Options:"
+ echo " --yes, -y Non-interactive mode (accept all prompts)"
+ echo " --port PORT CUA computer server port (default: 8443)"
+ echo " --help Show this help message"
+ exit 0
+ ;;
+ *)
+ echo -e "${RED}Unknown option: $1${NC}"
+ exit 1
+ ;;
+ esac
+ shift
+done
+
+CURRENT_USER="$(whoami)"
+USER_HOME="$HOME"
+
+# ============================================
+# Helpers
+# ============================================
+
+print_phase() {
+ echo ""
+ echo -e "${BLUE}========================================${NC}"
+ echo -e "${BLUE} $1${NC}"
+ echo -e "${BLUE}========================================${NC}"
+ echo ""
+}
+
+confirm() {
+ local prompt="$1"
+ if [ "$AUTO_YES" = true ]; then
+ echo -e "${YELLOW}${prompt} [Y/n] ${NC}y (auto)"
+ return 0
+ fi
+ read -r -p "$(echo -e "${YELLOW}${prompt} [Y/n] ${NC}")" response
+ case "$response" in
+ [nN][oO]|[nN]) return 1 ;;
+ *) return 0 ;;
+ esac
+}
+
+ensure_brew() {
+ if [ -f /opt/homebrew/bin/brew ]; then
+ eval "$(/opt/homebrew/bin/brew shellenv)"
+ fi
+}
+
+# Create an askpass helper for non-interactive sudo
+setup_sudo() {
+ if [ "$AUTO_YES" = true ]; then
+ ASKPASS_SCRIPT=$(mktemp)
+ printf '#!/bin/bash\necho "%s"\n' "$VM_PASSWORD" > "$ASKPASS_SCRIPT"
+ chmod +x "$ASKPASS_SCRIPT"
+ export SUDO_ASKPASS="$ASKPASS_SCRIPT"
+ sudo -A -v 2>/dev/null || true
+ fi
+}
+
+cleanup_sudo() {
+ if [ -n "${ASKPASS_SCRIPT:-}" ] && [ -f "$ASKPASS_SCRIPT" ]; then
+ rm -f "$ASKPASS_SCRIPT"
+ fi
+}
+trap cleanup_sudo EXIT
+
+# ============================================
+# Phase 1: Prerequisites (Homebrew + Python)
+# ============================================
+
+install_homebrew() {
+ print_phase "Phase 1: Homebrew"
+
+ if command -v brew &>/dev/null; then
+ echo -e "${GREEN}Homebrew already installed$(brew --version | head -1 | sed 's/Homebrew //' | sed 's/^/ (v/')${NC})"
+ ensure_brew
+ return 0
+ fi
+
+ if ! confirm "Install Homebrew?"; then
+ echo -e "${YELLOW}Skipping Homebrew. Later steps will fail.${NC}"
+ return 1
+ fi
+
+ setup_sudo
+
+ if [ "$AUTO_YES" = true ]; then
+ NONINTERACTIVE=1 SUDO_ASKPASS="$ASKPASS_SCRIPT" /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)"
+ else
+ echo -e "${YELLOW}Homebrew requires sudo access. Enter your password if prompted.${NC}"
+ /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)"
+ fi
+
+ # Add brew to shell profile
+ if ! grep -q 'brew shellenv' "$HOME/.zprofile" 2>/dev/null; then
+ echo 'eval "$(/opt/homebrew/bin/brew shellenv)"' >> "$HOME/.zprofile"
+ fi
+
+ ensure_brew
+ echo -e "${GREEN}Homebrew installed${NC}"
+}
+
+install_python() {
+ print_phase "Phase 2: Python 3.13"
+
+ ensure_brew
+
+ if brew list [email protected] &>/dev/null; then
+ echo -e "${GREEN}Python 3.13 already installed${NC}"
+ else
+ if ! confirm "Install Python 3.13 via Homebrew?"; then
+ echo -e "${YELLOW}Skipping Python 3.13.${NC}"
+ return 1
+ fi
+ brew install [email protected]
+ echo -e "${GREEN}Python 3.13 installed${NC}"
+ fi
+
+ export PATH="/opt/homebrew/opt/[email protected]/bin:$PATH"
+}
+
+install_system_deps() {
+ print_phase "Phase 3: System Dependencies"
+
+ ensure_brew
+
+ local deps_needed=()
+ for pkg in ffmpeg; do
+ if ! brew list "$pkg" &>/dev/null; then
+ deps_needed+=("$pkg")
+ fi
+ done
+
+ if [ ${#deps_needed[@]} -eq 0 ]; then
+ echo -e "${GREEN}All system dependencies already installed${NC}"
+ return 0
+ fi
+
+ if confirm "Install system dependencies (${deps_needed[*]})?"; then
+ brew install "${deps_needed[@]}"
+ echo -e "${GREEN}System dependencies installed${NC}"
+ fi
+}
+
+# ============================================
+# Phase 4: CUA Server + Agent
+# ============================================
+
+setup_cua_server() {
+ print_phase "Phase 4: CUA Computer Server & Agent"
+
+ ensure_brew
+ export PATH="/opt/homebrew/opt/[email protected]/bin:$PATH"
+
+ if ! confirm "Install cua-computer-server and cua-agent?"; then
+ echo -e "${YELLOW}Skipping CUA server setup.${NC}"
+ return 1
+ fi
+
+ mkdir -p "$CUA_DIR"
+
+ # Create venv if needed
+ if [ ! -d "$CUA_DIR/venv" ]; then
+ echo "Creating Python 3.13 virtual environment..."
+ /opt/homebrew/opt/[email protected]/bin/python3.13 -m venv "$CUA_DIR/venv"
+ fi
+
+ echo "Installing cua-computer-server..."
+ source "$CUA_DIR/venv/bin/activate"
+ pip install --upgrade pip
+ pip install cua-computer-server
+
+ echo "Installing cua-agent..."
+ pip install 'cua-agent[all]'
+
+ # Playwright + Firefox (non-blocking)
+ echo "Installing Playwright and Firefox browser..."
+ pip install playwright && playwright install firefox || \
+ echo -e "${YELLOW}Playwright install failed (non-blocking, BrowserTool may not work)${NC}"
+
+ deactivate
+
+ echo -e "${GREEN}CUA server and agent installed${NC}"
+}
+
+# ============================================
+# Phase 5: Startup Script + LaunchAgent
+# ============================================
+
+create_startup_script() {
+ print_phase "Phase 5: Startup Script & LaunchAgent"
+
+ if ! confirm "Create CUA server startup script and LaunchAgent?"; then
+ echo -e "${YELLOW}Skipping LaunchAgent setup.${NC}"
+ return 1
+ fi
+
+ local STARTUP_SCRIPT="$CUA_DIR/start_server.sh"
+
+ cat > "$STARTUP_SCRIPT" <<'SCRIPT_EOF'
+#!/bin/bash
+
+LOG_FILE="$HOME/.cua-server/server.log"
+echo "=== CUA Server startup at $(date) ===" >> "$LOG_FILE"
+
+# Wait for window server
+sleep 10
+
+cd "$HOME/.cua-server"
+source venv/bin/activate
+
+export DISPLAY=:0
+
+# Update packages
+echo "Updating cua-agent..." >> "$LOG_FILE"
+pip install --upgrade --no-input "cua-agent[all]" >> "$LOG_FILE" 2>&1 || true
+
+echo "Updating cua-computer-server..." >> "$LOG_FILE"
+pip install --upgrade --no-input cua-computer-server >> "$LOG_FILE" 2>&1 || true
+
+# Ensure Playwright Firefox
+echo "Ensuring Playwright Firefox..." >> "$LOG_FILE"
+pip install --upgrade --no-input playwright >> "$LOG_FILE" 2>&1 || true
+playwright install firefox >> "$LOG_FILE" 2>&1 || true
+
+# Start server
+echo "Starting CUA computer server on port __PORT__..." >> "$LOG_FILE"
+python -m computer_server --port __PORT__ >> "$LOG_FILE" 2>&1
+SCRIPT_EOF
+
+ # Stamp the port
+ sed -i '' "s/__PORT__/$CUA_SERVER_PORT/g" "$STARTUP_SCRIPT"
+ chmod +x "$STARTUP_SCRIPT"
+
+ # Create LaunchAgent
+ local SERVICE_NAME="com.trycua.computer_server"
+ local PLIST_PATH="$USER_HOME/Library/LaunchAgents/$SERVICE_NAME.plist"
+
+ mkdir -p "$USER_HOME/Library/LaunchAgents"
+
+ # Unload existing
+ launchctl unload "$PLIST_PATH" 2>/dev/null || true
+
+ cat > "$PLIST_PATH" <<EOF
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+ <key>Label</key>
+ <string>$SERVICE_NAME</string>
+
+ <key>WorkingDirectory</key>
+ <string>$CUA_DIR</string>
+
+ <key>EnvironmentVariables</key>
+ <dict>
+ <key>PATH</key>
+ <string>/opt/homebrew/bin:/opt/homebrew/opt/[email protected]/bin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin</string>
+ <key>HOME</key>
+ <string>$USER_HOME</string>
+ <key>DISPLAY</key>
+ <string>:0</string>
+ </dict>
+
+ <key>ProgramArguments</key>
+ <array>
+ <string>/bin/bash</string>
+ <string>$CUA_DIR/start_server.sh</string>
+ </array>
+
+ <key>RunAtLoad</key>
+ <true/>
+
+ <key>KeepAlive</key>
+ <true/>
+
+ <key>StandardOutPath</key>
+ <string>/tmp/computer_server.log</string>
+
+ <key>StandardErrorPath</key>
+ <string>/tmp/computer_server.error.log</string>
+
+ <key>ProcessType</key>
+ <string>Interactive</string>
+
+ <key>SessionType</key>
+ <string>Aqua</string>
+</dict>
+</plist>
+EOF
+
+ chmod 644 "$PLIST_PATH"
+ touch /tmp/computer_server.log /tmp/computer_server.error.log
+
+ echo -e "${GREEN}Startup script and LaunchAgent created${NC}"
+ echo "LaunchAgent will start CUA server on port $CUA_SERVER_PORT at login"
+}
+
+# ============================================
+# Phase 6: System Configuration
+# ============================================
+
+configure_system() {
+ print_phase "Phase 6: System Configuration"
+
+ if ! confirm "Disable screen lock, screensaver, and sleep?"; then
+ echo -e "${YELLOW}Skipping system configuration.${NC}"
+ return 0
+ fi
+
+ setup_sudo
+
+ # Disable screensaver
+ defaults -currentHost write com.apple.screensaver idleTime 0
+ defaults write com.apple.screensaver askForPassword -int 0
+ defaults write com.apple.screensaver askForPasswordDelay -int 0
+
+ # Disable sleep
+ if [ "$AUTO_YES" = true ]; then
+ sudo -A pmset -a displaysleep 0 sleep 0 disksleep 0
+ else
+ sudo pmset -a displaysleep 0 sleep 0 disksleep 0
+ fi
+
+ # Enable full keyboard navigation (needed for accessibility automation)
+ defaults write NSGlobalDomain AppleKeyboardUIMode -int 3
+
+ echo -e "${GREEN}System configured (no sleep, no screensaver, full keyboard nav)${NC}"
+}
+
+# ============================================
+# Phase 7: Verification
+# ============================================
+
+verify() {
+ print_phase "Verification"
+
+ echo "System:"
+ echo " User: $CURRENT_USER"
+ sw_vers | sed 's/^/ /'
+
+ echo ""
+ echo "Installed:"
+
+ # Homebrew
+ if command -v brew &>/dev/null; then
+ echo -e " ${GREEN}✓${NC} Homebrew $(brew --version | head -1 | grep -oE '[0-9]+\.[0-9]+\.[0-9]+')"
+ else
+ echo -e " ${RED}✗${NC} Homebrew"
+ fi
+
+ # Python
+ ensure_brew
+ if brew list [email protected] &>/dev/null; then
+ echo -e " ${GREEN}✓${NC} Python 3.13"
+ else
+ echo -e " ${RED}✗${NC} Python 3.13"
+ fi
+
+ # CUA server
+ if [ -f "$CUA_DIR/venv/bin/python" ]; then
+ local server_ver
+ server_ver=$("$CUA_DIR/venv/bin/pip" show cua-computer-server 2>/dev/null | grep Version | awk '{print $2}')
+ echo -e " ${GREEN}✓${NC} cua-computer-server ${server_ver:-unknown}"
+ else
+ echo -e " ${RED}✗${NC} cua-computer-server"
+ fi
+
+ # Agent
+ if [ -f "$CUA_DIR/venv/bin/python" ]; then
+ local agent_ver
+ agent_ver=$("$CUA_DIR/venv/bin/pip" show cua-agent 2>/dev/null | grep Version | awk '{print $2}')
+ echo -e " ${GREEN}✓${NC} cua-agent ${agent_ver:-unknown}"
+ else
+ echo -e " ${RED}✗${NC} cua-agent"
+ fi
+
+ # Playwright
+ if "$CUA_DIR/venv/bin/python" -c "import playwright" 2>/dev/null; then
+ echo -e " ${GREEN}✓${NC} Playwright + Firefox"
+ else
+ echo -e " ${YELLOW}~${NC} Playwright (may not be installed)"
+ fi
+
+ # LaunchAgent
+ local PLIST="$USER_HOME/Library/LaunchAgents/com.trycua.computer_server.plist"
+ if [ -f "$PLIST" ]; then
+ echo -e " ${GREEN}✓${NC} LaunchAgent (port $CUA_SERVER_PORT)"
+ else
+ echo -e " ${RED}✗${NC} LaunchAgent"
+ fi
+
+ echo ""
+ echo -e "${GREEN}========================================${NC}"
+ echo -e "${GREEN} Setup Complete!${NC}"
+ echo -e "${GREEN}========================================${NC}"
+ echo ""
+ echo "CUA computer server will start on port $CUA_SERVER_PORT at next login."
+ echo ""
+ echo "To start it now (from the host):"
+ echo " lume ssh <vm-name> 'launchctl load ~/Library/LaunchAgents/com.trycua.computer_server.plist'"
+ echo ""
+ echo "To test from the host:"
+ echo " curl http://<vm-ip>:$CUA_SERVER_PORT/health"
+ echo ""
+}
+
+# ============================================
+# Main
+# ============================================
+
+echo -e "${BLUE}"
+echo " ⠀⣀⣀⡀⠀⠀⠀⠀⢀⣀⣀⣀⡀⠘⠋⢉⠙⣷⠀⠀ ⠀"
+echo " ⠀⠀⢀⣴⣿⡿⠋⣉⠁⣠⣾⣿⣿⣿⣿⡿⠿⣦⡈⠀⣿⡇⠃⠀"
+echo " ⠀⠀⠀⣽⣿⣧⠀⠃⢰⣿⣿⡏⠙⣿⠿⢧⣀⣼⣷⠀⡿⠃⠀⠀"
+echo " ⠀⠀⠀⠉⣿⣿⣦⠀⢿⣿⣿⣷⣾⡏⠀⠀⢹⣿⣿⠀⠀⠀⠀⠀⠀"
+echo -e " ⠀⠀⠀⠀⠀⠉⠛⠁⠈⠿⣿⣿⣿⣷⣄⣠⡼⠟⠁${NC} ${BLUE}CUA Computer Setup${NC}"
+echo -e "${BLUE} Make a Lume VM cua-computer compatible${NC}"
+echo ""
+
+install_homebrew
+install_python
+install_system_deps
+setup_cua_server
+create_startup_script
+configure_system
+verify
diff --git a/libs/python/agent/agent/adapters/yutori_adapter.py b/libs/python/agent/agent/adapters/yutori_adapter.py
index 1bf6eb4a2a..47b821f7c1 100644
--- a/libs/python/agent/agent/adapters/yutori_adapter.py
+++ b/libs/python/agent/agent/adapters/yutori_adapter.py
@@ -21,7 +21,7 @@ class YutoriAdapter(CustomLLM):
def _normalize_model(self, model: str) -> str:
"""Strip the yutori/ prefix to get the bare model name."""
if model.startswith("yutori/"):
- return model[len("yutori/"):]
+ return model[len("yutori/") :]
return model
def _resolve_api_key(self, kwargs: dict | None = None) -> str:
@@ -61,7 +61,13 @@ class YutoriAdapter(CustomLLM):
params["tool_choice"] = kwargs["tool_choice"]
# Forward optional generation params
- for key in ("temperature", "top_p", "max_completion_tokens", "max_tokens", "response_format"):
+ for key in (
+ "temperature",
+ "top_p",
+ "max_completion_tokens",
+ "max_tokens",
+ "response_format",
+ ):
if key in kwargs:
params[key] = kwargs[key]
diff --git a/libs/python/agent/agent/agent.py b/libs/python/agent/agent/agent.py
index 388a514e7b..70e93293b5 100644
--- a/libs/python/agent/agent/agent.py
+++ b/libs/python/agent/agent/agent.py
@@ -91,6 +91,7 @@ def get_json(obj: Any, max_depth: int = 10) -> Any:
# Handle enums — just use their value
import enum
+
if isinstance(o, enum.Enum):
return o.value
@@ -856,7 +857,9 @@ class ComputerAgent:
# Note: function_call_output only supports text, so we include the
# screenshot as a separate user message with image content
if is_terminate:
- output_content = json.dumps(action_result if action_result else {"terminated": True})
+ output_content = json.dumps(
+ action_result if action_result else {"terminated": True}
+ )
call_output = {
"type": "function_call_output",
"call_id": item.get("call_id"),
@@ -870,7 +873,9 @@ class ComputerAgent:
if action_result is not None:
output_content = json.dumps(action_result)
else:
- output_content = json.dumps({"success": True, "screenshot_captured": True})
+ output_content = json.dumps(
+ {"success": True, "screenshot_captured": True}
+ )
call_output = {
"type": "function_call_output",
"call_id": item.get("call_id"),
diff --git a/libs/python/agent/agent/callbacks/trajectory_saver.py b/libs/python/agent/agent/callbacks/trajectory_saver.py
index 81627882f7..5d091e3641 100644
--- a/libs/python/agent/agent/callbacks/trajectory_saver.py
+++ b/libs/python/agent/agent/callbacks/trajectory_saver.py
@@ -156,7 +156,10 @@ def extract_computer_call_outputs(
new_content = []
content_modified = False
for content_item in content:
- if isinstance(content_item, dict) and content_item.get("type") == "input_image":
+ if (
+ isinstance(content_item, dict)
+ and content_item.get("type") == "input_image"
+ ):
image_url = content_item.get("image_url", "")
if isinstance(image_url, str) and image_url.startswith("data:"):
# Generate a unique ID for this screenshot
@@ -559,6 +562,7 @@ class TrajectorySaverCallback(AsyncCallbackHandler):
# Try to evaluate as Python literal (for stringified dicts)
try:
import ast
+
output_dict = ast.literal_eval(output)
except (ValueError, SyntaxError):
continue
@@ -589,7 +593,12 @@ class TrajectorySaverCallback(AsyncCallbackHandler):
image_bytes = base64.b64decode(image_data)
# If we have coordinates, draw crosshair annotation
- if x_coord is not None and y_coord is not None and x_coord != 0 and y_coord != 0:
+ if (
+ x_coord is not None
+ and y_coord is not None
+ and x_coord != 0
+ and y_coord != 0
+ ):
annotated_image = self._draw_crosshair_on_image(
image_bytes, int(x_coord), int(y_coord)
)
@@ -610,7 +619,10 @@ class TrajectorySaverCallback(AsyncCallbackHandler):
content = result_item.get("content", [])
if isinstance(content, list):
for content_item in content:
- if isinstance(content_item, dict) and content_item.get("type") == "input_image":
+ if (
+ isinstance(content_item, dict)
+ and content_item.get("type") == "input_image"
+ ):
image_url = content_item.get("image_url", "")
if isinstance(image_url, str) and image_url.startswith("data:"):
try:
@@ -618,7 +630,12 @@ class TrajectorySaverCallback(AsyncCallbackHandler):
image_bytes = base64.b64decode(b64_payload)
# If we have coordinates, draw crosshair annotation
- if x_coord is not None and y_coord is not None and x_coord != 0 and y_coord != 0:
+ if (
+ x_coord is not None
+ and y_coord is not None
+ and x_coord != 0
+ and y_coord != 0
+ ):
annotated_image = self._draw_crosshair_on_image(
image_bytes, int(x_coord), int(y_coord)
)
diff --git a/libs/python/agent/agent/loops/gemini.py b/libs/python/agent/agent/loops/gemini.py
index 9df2127b2d..37c45dcad8 100644
--- a/libs/python/agent/agent/loops/gemini.py
+++ b/libs/python/agent/agent/loops/gemini.py
@@ -89,11 +89,7 @@ def _sanitize_for_json(obj: Any) -> Any:
return _sanitize_for_json(obj.model_dump())
# Handle objects with __dict__ (like Gemini SDK response objects)
if hasattr(obj, "__dict__"):
- return {
- k: _sanitize_for_json(v)
- for k, v in obj.__dict__.items()
- if not k.startswith("__")
- }
+ return {k: _sanitize_for_json(v) for k, v in obj.__dict__.items() if not k.startswith("__")}
# Fallback to string representation
return str(obj)
@@ -828,9 +824,7 @@ class GeminiComputerUseConfig(AsyncAgentConfig):
}
if _on_api_start:
- await _on_api_start(
- _sanitize_for_json(api_kwargs)
- )
+ await _on_api_start(_sanitize_for_json(api_kwargs))
response = client.models.generate_content(**api_kwargs)
diff --git a/libs/python/agent/agent/loops/openai.py b/libs/python/agent/agent/loops/openai.py
index e013e746e8..5648e67084 100644
--- a/libs/python/agent/agent/loops/openai.py
+++ b/libs/python/agent/agent/loops/openai.py
@@ -15,7 +15,9 @@ from ..decorators import register_agent
from ..types import AgentCapability, AgentResponse, Messages, Tools
-async def _map_computer_tool_to_openai(computer_handler: Any, use_native_tool: bool = True) -> Dict[str, Any]:
+async def _map_computer_tool_to_openai(
+ computer_handler: Any, use_native_tool: bool = True
+) -> Dict[str, Any]:
"""Map a computer tool to OpenAI's tool schema.
Args:
@@ -136,6 +138,7 @@ async def _map_computer_tool_to_openai(computer_handler: Any, use_native_tool: b
def _is_native_computer_use_model(model: str) -> bool:
"""Check if the model supports native computer_use_preview tool format."""
import re
+
# Only computer-use-preview models support native computer_use_preview tool
# GPT 5.4 does NOT support computer_use_preview - it uses function calling
return bool(re.search(r"computer-use-preview", model, re.IGNORECASE))
@@ -154,18 +157,22 @@ async def _prepare_tools_for_openai(tool_schemas: List[Dict[str, Any]], model: s
for schema in tool_schemas:
if schema["type"] == "computer":
# Map computer tool to OpenAI format (native or function based on model)
- computer_tool = await _map_computer_tool_to_openai(schema["computer"], use_native_tool=use_native)
+ computer_tool = await _map_computer_tool_to_openai(
+ schema["computer"], use_native_tool=use_native
+ )
openai_tools.append(computer_tool)
elif schema["type"] == "function":
# Function tools for Responses API need: {type, name, description, parameters}
# Note: parameters are at the root level, NOT nested under 'function'
func = schema["function"]
- openai_tools.append({
- "type": "function",
- "name": func["name"],
- "description": func.get("description", ""),
- "parameters": func.get("parameters", {}),
- })
+ openai_tools.append(
+ {
+ "type": "function",
+ "name": func["name"],
+ "description": func.get("description", ""),
+ "parameters": func.get("parameters", {}),
+ }
+ )
return openai_tools
@@ -387,10 +394,7 @@ Task: Click {instruction}. Output ONLY a click action on the target element.""",
continue
# Native format: computer_call with action dict
- if (
- item.get("type") == "computer_call"
- and isinstance(item.get("action"), dict)
- ):
+ if item.get("type") == "computer_call" and isinstance(item.get("action"), dict):
action = item["action"]
if action.get("x") is not None and action.get("y") is not None:
return (int(action.get("x")), int(action.get("y")))
diff --git a/libs/python/agent/agent/loops/yutori.py b/libs/python/agent/agent/loops/yutori.py
index 861fcdf453..0b38e0671e 100644
--- a/libs/python/agent/agent/loops/yutori.py
+++ b/libs/python/agent/agent/loops/yutori.py
@@ -370,16 +370,12 @@ class YutoriN1Config(AsyncAgentConfig):
}
],
}
- output_items.extend(
- convert_completion_messages_to_responses_items([fake_cm])
- )
+ output_items.extend(convert_completion_messages_to_responses_items([fake_cm]))
# Only use content_text once
content_text = ""
else:
# Custom tool — emit as function_call
- output_items.append(
- make_function_call_item(fn_name, args, call_id=tc_id)
- )
+ output_items.append(make_function_call_item(fn_name, args, call_id=tc_id))
else:
# No tool calls — task is complete
if content_text:
diff --git a/libs/python/agent/agent/tools/browser_tool.py b/libs/python/agent/agent/tools/browser_tool.py
index 95ff43fcb7..08b04f4424 100644
--- a/libs/python/agent/agent/tools/browser_tool.py
+++ b/libs/python/agent/agent/tools/browser_tool.py
@@ -188,7 +188,11 @@ class BrowserTool(BaseComputerTool):
],
"type": "string",
},
- "keys": {"description": "Required only by action=key.", "type": "array", "items": {"type": "string"}},
+ "keys": {
+ "description": "Required only by action=key.",
+ "type": "array",
+ "items": {"type": "string"},
+ },
"text": {"description": "Required only by action=type.", "type": "string"},
"coordinate": {
"description": "(x, y) coordinates for mouse actions. Required only by action=left_click, action=mouse_move, and action=type.",
diff --git a/libs/python/computer-server/computer_server/cli.py b/libs/python/computer-server/computer_server/cli.py
index aa68351321..3ea0c76f0d 100644
--- a/libs/python/computer-server/computer_server/cli.py
+++ b/libs/python/computer-server/computer_server/cli.py
@@ -10,8 +10,6 @@ import sys
import threading
from typing import List, Optional
-from .server import Server
-
logger = logging.getLogger(__name__)
@@ -72,6 +70,33 @@ def parse_args(args: Optional[List[str]] = None) -> argparse.Namespace:
help="Disable automatic server restart in watchdog",
)
+ # Backend selection
+ parser.add_argument(
+ "--backend",
+ choices=["native", "vnc"],
+ default="native",
+ help="Handler backend: 'native' uses OS-specific handlers, 'vnc' uses VNC (default: native)",
+ )
+
+ # VNC backend options
+ parser.add_argument(
+ "--vnc-host",
+ type=str,
+ help="VNC server host (required when --backend=vnc)",
+ )
+ parser.add_argument(
+ "--vnc-port",
+ type=int,
+ default=5900,
+ help="VNC server port (default: 5900)",
+ )
+ parser.add_argument(
+ "--vnc-password",
+ type=str,
+ default="",
+ help="VNC server password",
+ )
+
return parser.parse_args(args)
@@ -86,6 +111,21 @@ def main() -> None:
stream=sys.stderr, # Use stderr for MCP compatibility
)
+ # Set backend env vars from CLI args before Server import triggers handler creation
+ if args.backend == "vnc" or args.vnc_host:
+ if not args.vnc_host and not os.environ.get("CUA_VNC_HOST"):
+ parser_err = "--vnc-host is required when using --backend=vnc"
+ logger.error(parser_err)
+ sys.exit(1)
+ os.environ["CUA_BACKEND"] = "vnc"
+ if args.vnc_host:
+ os.environ["CUA_VNC_HOST"] = args.vnc_host
+ os.environ["CUA_VNC_PORT"] = str(args.vnc_port)
+ if args.vnc_password:
+ os.environ["CUA_VNC_PASSWORD"] = args.vnc_password
+ vnc_host = args.vnc_host or os.environ.get("CUA_VNC_HOST")
+ logger.info(f"VNC backend enabled → {vnc_host}:{args.vnc_port}")
+
# Check if watchdog should be enabled
container_name = os.environ.get("CONTAINER_NAME")
enable_watchdog = (args.watchdog or bool(container_name)) and not sys.platform.startswith("win")
@@ -151,6 +191,10 @@ def main() -> None:
else:
logger.info("HTTP mode (no SSL certificates provided)")
+ # Import Server lazily so env vars (e.g. CUA_VNC_HOST) are set before
+ # the module-level handler factory runs in main.py.
+ from .server import Server
+
server = Server(host=args.host, port=args.port, log_level=args.log_level, **ssl_args)
try:
diff --git a/libs/python/computer-server/computer_server/handlers/base.py b/libs/python/computer-server/computer_server/handlers/base.py
index d38ef553a0..833b21b416 100644
--- a/libs/python/computer-server/computer_server/handlers/base.py
+++ b/libs/python/computer-server/computer_server/handlers/base.py
@@ -1,6 +1,11 @@
+import asyncio
+import logging
+import sys
from abc import ABC, abstractmethod
from typing import Any, Dict, List, Optional, Tuple
+logger = logging.getLogger(__name__)
+
class BaseAccessibilityHandler(ABC):
"""Abstract base class for OS-specific accessibility handlers."""
@@ -322,17 +327,54 @@ class BaseAutomationHandler(ABC):
pass
# Clipboard Actions
- @abstractmethod
async def copy_to_clipboard(self) -> Dict[str, Any]:
- """Get the current clipboard content."""
- pass
+ """Get the current clipboard content using pyperclip."""
+ try:
+ import pyperclip
+
+ content = pyperclip.paste()
+ return {"success": True, "content": content}
+ except Exception as e:
+ return {"success": False, "error": str(e)}
- @abstractmethod
async def set_clipboard(self, text: str) -> Dict[str, Any]:
- """Set the clipboard content."""
- pass
+ """Set the clipboard content using pyperclip."""
+ try:
+ import pyperclip
- @abstractmethod
+ pyperclip.copy(text)
+ return {"success": True}
+ except Exception as e:
+ return {"success": False, "error": str(e)}
+
+ # Command Execution
async def run_command(self, command: str) -> Dict[str, Any]:
- """Run a command and return the output."""
- pass
+ """Run a shell command locally and return its output.
+
+ When IS_CUA_ANDROID env var is set, routes the command through
+ ``adb shell`` so execution runs inside the Android emulator.
+ """
+ import os
+
+ try:
+ if os.environ.get("IS_CUA_ANDROID") == "true":
+ process = await asyncio.create_subprocess_exec(
+ "adb",
+ "shell",
+ command,
+ stdout=asyncio.subprocess.PIPE,
+ stderr=asyncio.subprocess.PIPE,
+ )
+ else:
+ process = await asyncio.create_subprocess_shell(
+ command, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE
+ )
+ stdout, stderr = await process.communicate()
+ return {
+ "success": True,
+ "stdout": stdout.decode() if stdout else "",
+ "stderr": stderr.decode() if stderr else "",
+ "return_code": process.returncode,
+ }
+ except Exception as e:
+ return {"success": False, "error": str(e)}
diff --git a/libs/python/computer-server/computer_server/handlers/factory.py b/libs/python/computer-server/computer_server/handlers/factory.py
index 26a0792d5f..00029fa10c 100644
--- a/libs/python/computer-server/computer_server/handlers/factory.py
+++ b/libs/python/computer-server/computer_server/handlers/factory.py
@@ -1,3 +1,5 @@
+import logging
+import os
from typing import Tuple
from computer_server.diorama.base import BaseDioramaHandler
@@ -11,6 +13,8 @@ from .base import (
BaseWindowHandler,
)
+logger = logging.getLogger(__name__)
+
OS_TYPE = get_current_os()
if OS_TYPE == "android":
@@ -55,7 +59,28 @@ class HandlerFactory:
NotImplementedError: If the current OS is not supported
RuntimeError: If unable to determine the current OS
"""
- if OS_TYPE == "android":
+ backend = os.environ.get("CUA_BACKEND", "native")
+ vnc_host = os.environ.get("CUA_VNC_HOST")
+ if backend == "vnc" or vnc_host:
+ if not vnc_host:
+ raise RuntimeError(
+ "CUA_VNC_HOST must be set when using VNC backend "
+ "(--backend=vnc requires --vnc-host)"
+ )
+ from .vnc import VNCAccessibilityHandler, VNCAutomationHandler
+
+ vnc_port = int(os.environ.get("CUA_VNC_PORT", "5900"))
+ vnc_password = os.environ.get("CUA_VNC_PASSWORD", "")
+ logger.info(f"Using VNC backend → {vnc_host}:{vnc_port}")
+ return (
+ VNCAccessibilityHandler(),
+ VNCAutomationHandler(host=vnc_host, port=vnc_port, password=vnc_password),
+ BaseDioramaHandler(),
+ GenericFileHandler(),
+ GenericDesktopHandler(),
+ GenericWindowHandler(),
+ )
+ elif OS_TYPE == "android":
return (
AndroidAccessibilityHandler(),
AndroidAutomationHandler(),
diff --git a/libs/python/computer-server/computer_server/handlers/linux.py b/libs/python/computer-server/computer_server/handlers/linux.py
index af127ce0f6..261189ede3 100644
--- a/libs/python/computer-server/computer_server/handlers/linux.py
+++ b/libs/python/computer-server/computer_server/handlers/linux.py
@@ -537,73 +537,4 @@ class LinuxAutomationHandler(BaseAutomationHandler):
except Exception as e:
return {"success": False, "error": str(e)}
- # Clipboard Actions
- async def copy_to_clipboard(self) -> Dict[str, Any]:
- """Get the current content of the clipboard.
-
- Returns:
- Dict[str, Any]: A dictionary containing success status and clipboard content,
- or error message if failed.
- """
- try:
- import pyperclip
-
- content = pyperclip.paste()
- return {"success": True, "content": content}
- except Exception as e:
- return {"success": False, "error": str(e)}
-
- async def set_clipboard(self, text: str) -> Dict[str, Any]:
- """Set the clipboard content to the specified text.
-
- Args:
- text: The text to copy to the clipboard.
-
- Returns:
- Dict[str, Any]: A dictionary with success status and error message if failed.
- """
- try:
- import pyperclip
-
- pyperclip.copy(text)
- return {"success": True}
- except Exception as e:
- return {"success": False, "error": str(e)}
-
- # Command Execution
- async def run_command(self, command: str) -> Dict[str, Any]:
- """Execute a shell command asynchronously.
-
- When IS_CUA_ANDROID is set, routes the command through `adb shell`
- so execution runs inside the Android emulator.
-
- Args:
- command: The shell command to execute.
-
- Returns:
- Dict[str, Any]: A dictionary containing success status, stdout, stderr,
- and return code, or error message if failed.
- """
- try:
- if os.environ.get("IS_CUA_ANDROID") == "true":
- # Route through adb shell to run inside the emulator
- process = await asyncio.create_subprocess_exec(
- "adb",
- "shell",
- command,
- stdout=asyncio.subprocess.PIPE,
- stderr=asyncio.subprocess.PIPE,
- )
- else:
- process = await asyncio.create_subprocess_shell(
- command, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE
- )
- stdout, stderr = await process.communicate()
- return {
- "success": True,
- "stdout": stdout.decode() if stdout else "",
- "stderr": stderr.decode() if stderr else "",
- "return_code": process.returncode,
- }
- except Exception as e:
- return {"success": False, "error": str(e)}
+ # Clipboard and run_command inherited from BaseAutomationHandler
diff --git a/libs/python/computer-server/computer_server/handlers/macos.py b/libs/python/computer-server/computer_server/handlers/macos.py
index 56ba5a672d..3846508b33 100644
--- a/libs/python/computer-server/computer_server/handlers/macos.py
+++ b/libs/python/computer-server/computer_server/handlers/macos.py
@@ -1408,66 +1408,4 @@ class MacOSAutomationHandler(BaseAutomationHandler):
except Exception as e:
return {"success": False, "error": str(e)}
- # Clipboard Actions
- async def copy_to_clipboard(self) -> Dict[str, Any]:
- """Get the current content of the system clipboard.
-
- Returns:
- Dictionary containing success status and clipboard content or error message
- """
- try:
- import pyperclip
-
- content = pyperclip.paste()
- return {"success": True, "content": content}
- except Exception as e:
- return {"success": False, "error": str(e)}
-
- async def set_clipboard(self, text: str) -> Dict[str, Any]:
- """Set the content of the system clipboard.
-
- Args:
- text: Text to copy to the clipboard
-
- Returns:
- Dictionary containing success status and error message if failed
- """
- try:
- import pyperclip
-
- pyperclip.copy(text)
- return {"success": True}
- except Exception as e:
- return {"success": False, "error": str(e)}
-
- async def run_command(self, command: str) -> Dict[str, Any]:
- """Run a shell command and return its output.
-
- Args:
- command: Shell command to execute
-
- Returns:
- Dictionary containing success status, stdout, stderr, and return code
- """
- try:
- if os.environ.get("IS_CUA_ANDROID") == "true":
- process = await asyncio.create_subprocess_exec(
- "adb",
- "shell",
- command,
- stdout=asyncio.subprocess.PIPE,
- stderr=asyncio.subprocess.PIPE,
- )
- else:
- process = await asyncio.create_subprocess_shell(
- command, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE
- )
- stdout, stderr = await process.communicate()
- return {
- "success": True,
- "stdout": stdout.decode() if stdout else "",
- "stderr": stderr.decode() if stderr else "",
- "return_code": process.returncode,
- }
- except Exception as e:
- return {"success": False, "error": str(e)}
+ # Clipboard and run_command inherited from BaseAutomationHandler
diff --git a/libs/python/computer-server/computer_server/handlers/vnc.py b/libs/python/computer-server/computer_server/handlers/vnc.py
new file mode 100644
index 0000000000..c691e01ab2
--- /dev/null
+++ b/libs/python/computer-server/computer_server/handlers/vnc.py
@@ -0,0 +1,499 @@
+"""
+VNC backend for automation and accessibility handlers.
+
+A cross-platform alternative to the native (OS-specific) handlers. Connects to
+any VNC server to perform screenshots, mouse, keyboard, and scroll operations
+over the RFB protocol. Works with Linux, macOS, and Windows targets — the only
+requirement is a reachable VNC server.
+
+Built on vncdotool (Twisted-based RFB client). The Twisted reactor runs in a
+background daemon thread; all handler methods bridge from asyncio via
+`asyncio.to_thread`.
+
+Usage:
+ # Via CLI
+ python -m computer_server --vnc-host 192.168.64.1 --vnc-port 5900 --vnc-password secret
+
+ # Via env vars
+ CUA_VNC_HOST=192.168.64.1 CUA_VNC_PORT=5900 CUA_VNC_PASSWORD=secret python -m computer_server
+"""
+
+import asyncio
+import base64
+import logging
+import threading
+from io import BytesIO
+from typing import Any, Dict, List, Optional, Tuple
+
+from .base import BaseAccessibilityHandler, BaseAutomationHandler
+
+logger = logging.getLogger(__name__)
+
+# Key name aliases: map cua-computer-server key names → vncdotool key names.
+# vncdotool's KEYMAP already covers most X11 keysym names; these bridge
+# the naming differences used by the rest of the computer-server API.
+_KEY_ALIASES = {
+ "return": "enter",
+ "escape": "esc",
+ "backspace": "bsp",
+ "delete": "del",
+ "page_up": "pgup",
+ "pageup": "pgup",
+ "page_down": "pgdn",
+ "pagedown": "pgdn",
+ "insert": "ins",
+ "caps_lock": "caplk",
+ "num_lock": "numlk",
+ "shift_l": "lshift",
+ "shift_r": "rshift",
+ "ctrl_l": "lctrl",
+ "ctrl_r": "rctrl",
+ "control": "ctrl",
+ "control_l": "lctrl",
+ "control_r": "rctrl",
+ "alt_l": "lalt",
+ "alt_r": "ralt",
+ "meta_l": "lmeta",
+ "meta_r": "rmeta",
+ "super_l": "lsuper",
+ "super_r": "rsuper",
+ # macOS-specific names
+ "command": "alt",
+ "cmd": "alt",
+ "option": "meta",
+ "option_l": "lmeta",
+ "option_r": "rmeta",
+}
+
+# Button name → vncdotool button number (1-indexed)
+_BUTTON_MAP = {"left": 1, "middle": 2, "right": 3}
+
+
+def _translate_key(key: str) -> str:
+ """Translate a key name to vncdotool's KEYMAP name."""
+ lower = key.lower()
+ return _KEY_ALIASES.get(lower, lower)
+
+
+# ---------------------------------------------------------------------------
+# Lazy vncdotool client wrapper
+# ---------------------------------------------------------------------------
+
+
+class _VNCConnection:
+ """Manages a vncdotool ThreadedVNCClientProxy lifecycle.
+
+ All methods are synchronous (blocking) — the handler calls them via
+ asyncio.to_thread to avoid blocking the event loop.
+ """
+
+ def __init__(self, host: str, port: int, password: str = ""):
+ self._host = host
+ self._port = port
+ self._password = password
+ self._client = None
+ self._lock = threading.Lock()
+ # Track cursor position locally (vncdotool also tracks via client.x/y)
+ self._cursor_x: int = 0
+ self._cursor_y: int = 0
+
+ def _ensure_connected(self):
+ """Lazily connect to the VNC server. Returns the vncdotool client."""
+ if self._client is not None:
+ return self._client
+ with self._lock:
+ if self._client is not None:
+ return self._client
+ from vncdotool import api
+
+ server_str = f"{self._host}::{self._port}"
+ logger.info(f"VNC connecting to {self._host}:{self._port}")
+ client = api.connect(
+ server_str,
+ password=self._password or None,
+ )
+ client.timeout = 30
+ self._client = client
+ logger.info("VNC connected")
+ return client
+
+ def disconnect(self):
+ with self._lock:
+ if self._client is not None:
+ try:
+ self._client.disconnect()
+ except Exception:
+ pass
+ self._client = None
+
+ def _reset_on_error(self):
+ """Disconnect so next call reconnects."""
+ try:
+ self.disconnect()
+ except Exception:
+ pass
+
+ # -- Screenshot ---------------------------------------------------------
+
+ def capture_screenshot(self) -> bytes:
+ """Capture the screen and return PNG bytes."""
+ client = self._ensure_connected()
+ # refreshScreen updates the internal screen image
+ client.refreshScreen(incremental=False)
+ screen = client.protocol.screen
+ buf = BytesIO()
+ screen.save(buf, format="PNG")
+ return buf.getvalue()
+
+ # -- Mouse --------------------------------------------------------------
+
+ def mouse_move(self, x: int, y: int):
+ client = self._ensure_connected()
+ client.mouseMove(x, y)
+ self._cursor_x = x
+ self._cursor_y = y
+
+ def mouse_click(self, x: int, y: int, button: int = 1, clicks: int = 1):
+ client = self._ensure_connected()
+ client.mouseMove(x, y)
+ self._cursor_x = x
+ self._cursor_y = y
+ for _ in range(clicks):
+ client.mousePress(button)
+
+ def mouse_down(self, x: int, y: int, button: int = 1):
+ client = self._ensure_connected()
+ client.mouseMove(x, y)
+ self._cursor_x = x
+ self._cursor_y = y
+ client.mouseDown(button)
+
+ def mouse_up(self, x: int, y: int, button: int = 1):
+ client = self._ensure_connected()
+ client.mouseMove(x, y)
+ self._cursor_x = x
+ self._cursor_y = y
+ client.mouseUp(button)
+
+ def mouse_drag(self, x: int, y: int, step: int = 1):
+ client = self._ensure_connected()
+ client.mouseDrag(x, y, step=step)
+ self._cursor_x = x
+ self._cursor_y = y
+
+ def scroll(self, x: int, y: int):
+ """Scroll. y>0 = down (button 5), y<0 = up (button 4)."""
+ client = self._ensure_connected()
+ if y > 0:
+ for _ in range(y):
+ client.mousePress(5)
+ elif y < 0:
+ for _ in range(abs(y)):
+ client.mousePress(4)
+ # Horizontal: button 6 (right), button 7 (left) — not universally supported
+ if x > 0:
+ for _ in range(x):
+ client.mousePress(6)
+ elif x < 0:
+ for _ in range(abs(x)):
+ client.mousePress(7)
+
+ # -- Keyboard -----------------------------------------------------------
+
+ def key_press(self, key: str):
+ client = self._ensure_connected()
+ client.keyPress(_translate_key(key))
+
+ def key_down(self, key: str):
+ client = self._ensure_connected()
+ client.keyDown(_translate_key(key))
+
+ def key_up(self, key: str):
+ client = self._ensure_connected()
+ client.keyUp(_translate_key(key))
+
+ def type_text(self, text: str):
+ """Type text character by character, handling shift for uppercase/symbols."""
+ client = self._ensure_connected()
+ for ch in text:
+ if ch == " ":
+ client.keyPress("space")
+ elif ch == "\n":
+ client.keyPress("enter")
+ elif ch == "\t":
+ client.keyPress("tab")
+ else:
+ # vncdotool's keyPress handles single characters directly
+ client.keyPress(ch)
+
+ def hotkey(self, keys: List[str]):
+ """Press a key combination (e.g. ['command', 'a'])."""
+ client = self._ensure_connected()
+ translated = [_translate_key(k) for k in keys]
+ # vncdotool supports "modifier-key" syntax: "alt-a", "ctrl-shift-c"
+ combo = "-".join(translated)
+ client.keyPress(combo)
+
+ # -- Clipboard ----------------------------------------------------------
+
+ def paste_text(self, text: str):
+ client = self._ensure_connected()
+ client.paste(text)
+
+ # -- Info ---------------------------------------------------------------
+
+ @property
+ def screen_size(self) -> Tuple[int, int]:
+ client = self._ensure_connected()
+ proto = client.protocol
+ if proto is not None and proto.screen is not None:
+ return proto.screen.size
+ return (0, 0)
+
+ @property
+ def cursor_position(self) -> Tuple[int, int]:
+ return self._cursor_x, self._cursor_y
+
+
+# ---------------------------------------------------------------------------
+# Automation Handler
+# ---------------------------------------------------------------------------
+
+
+class VNCAutomationHandler(BaseAutomationHandler):
+ """Cross-platform automation handler that operates via VNC/RFB protocol.
+
+ Works with any OS target (Linux, macOS, Windows) that has a VNC server.
+ All operations go through the network, bypassing local permission systems.
+ """
+
+ def __init__(
+ self,
+ host: str = "127.0.0.1",
+ port: int = 5900,
+ password: str = "",
+ ):
+ self._conn = _VNCConnection(host, port, password)
+
+ def _resolve_coords(self, x: Optional[int], y: Optional[int]) -> Tuple[int, int]:
+ if x is not None and y is not None:
+ return x, y
+ return self._conn.cursor_position
+
+ # -- Screenshot ---------------------------------------------------------
+
+ async def screenshot(self) -> Dict[str, Any]:
+ try:
+ png_bytes = await asyncio.to_thread(self._conn.capture_screenshot)
+ image_data = base64.b64encode(png_bytes).decode()
+ return {"success": True, "image_data": image_data}
+ except Exception as e:
+ logger.error(f"VNC screenshot error: {e}")
+ self._conn._reset_on_error()
+ return {"success": False, "error": f"VNC screenshot error: {e}"}
+
+ # -- Mouse actions ------------------------------------------------------
+
+ async def left_click(self, x: Optional[int] = None, y: Optional[int] = None) -> Dict[str, Any]:
+ try:
+ cx, cy = self._resolve_coords(x, y)
+ await asyncio.to_thread(self._conn.mouse_click, cx, cy, 1)
+ return {"success": True}
+ except Exception as e:
+ return {"success": False, "error": str(e)}
+
+ async def right_click(self, x: Optional[int] = None, y: Optional[int] = None) -> Dict[str, Any]:
+ try:
+ cx, cy = self._resolve_coords(x, y)
+ await asyncio.to_thread(self._conn.mouse_click, cx, cy, 3)
+ return {"success": True}
+ except Exception as e:
+ return {"success": False, "error": str(e)}
+
+ async def middle_click(
+ self, x: Optional[int] = None, y: Optional[int] = None
+ ) -> Dict[str, Any]:
+ try:
+ cx, cy = self._resolve_coords(x, y)
+ await asyncio.to_thread(self._conn.mouse_click, cx, cy, 2)
+ return {"success": True}
+ except Exception as e:
+ return {"success": False, "error": str(e)}
+
+ async def double_click(
+ self, x: Optional[int] = None, y: Optional[int] = None
+ ) -> Dict[str, Any]:
+ try:
+ cx, cy = self._resolve_coords(x, y)
+ await asyncio.to_thread(self._conn.mouse_click, cx, cy, 1, 2)
+ return {"success": True}
+ except Exception as e:
+ return {"success": False, "error": str(e)}
+
+ async def move_cursor(self, x: int, y: int) -> Dict[str, Any]:
+ try:
+ await asyncio.to_thread(self._conn.mouse_move, x, y)
+ return {"success": True}
+ except Exception as e:
+ return {"success": False, "error": str(e)}
+
+ async def mouse_down(
+ self, x: Optional[int] = None, y: Optional[int] = None, button: str = "left"
+ ) -> Dict[str, Any]:
+ try:
+ cx, cy = self._resolve_coords(x, y)
+ btn = _BUTTON_MAP.get(button, 1)
+ await asyncio.to_thread(self._conn.mouse_down, cx, cy, btn)
+ return {"success": True}
+ except Exception as e:
+ return {"success": False, "error": str(e)}
+
+ async def mouse_up(
+ self, x: Optional[int] = None, y: Optional[int] = None, button: str = "left"
+ ) -> Dict[str, Any]:
+ try:
+ cx, cy = self._resolve_coords(x, y)
+ btn = _BUTTON_MAP.get(button, 1)
+ await asyncio.to_thread(self._conn.mouse_up, cx, cy, btn)
+ return {"success": True}
+ except Exception as e:
+ return {"success": False, "error": str(e)}
+
+ async def drag_to(
+ self, x: int, y: int, button: str = "left", duration: float = 0.5
+ ) -> Dict[str, Any]:
+ try:
+ btn = _BUTTON_MAP.get(button, 1)
+ cx, cy = self._conn.cursor_position
+
+ def _drag():
+ self._conn.mouse_down(cx, cy, btn)
+ step = max(1, int(1 / max(duration, 0.01) * 5))
+ self._conn.mouse_drag(x, y, step=step)
+ self._conn.mouse_up(x, y, btn)
+
+ await asyncio.to_thread(_drag)
+ return {"success": True}
+ except Exception as e:
+ return {"success": False, "error": str(e)}
+
+ async def drag(
+ self, path: List[Tuple[int, int]], button: str = "left", duration: float = 0.5
+ ) -> Dict[str, Any]:
+ try:
+ if not path:
+ return {"success": False, "error": "Empty path"}
+ btn = _BUTTON_MAP.get(button, 1)
+
+ def _drag_path():
+ self._conn.mouse_down(path[0][0], path[0][1], btn)
+ for px, py in path[1:]:
+ self._conn.mouse_move(px, py)
+ self._conn.mouse_up(path[-1][0], path[-1][1], btn)
+
+ await asyncio.to_thread(_drag_path)
+ return {"success": True}
+ except Exception as e:
+ return {"success": False, "error": str(e)}
+
+ # -- Keyboard actions ---------------------------------------------------
+
+ async def type_text(self, text: str) -> Dict[str, Any]:
+ try:
+ await asyncio.to_thread(self._conn.type_text, text)
+ return {"success": True}
+ except Exception as e:
+ return {"success": False, "error": str(e)}
+
+ async def press_key(self, key: str) -> Dict[str, Any]:
+ try:
+ await asyncio.to_thread(self._conn.key_press, key)
+ return {"success": True}
+ except Exception as e:
+ return {"success": False, "error": str(e)}
+
+ async def key_down(self, key: str) -> Dict[str, Any]:
+ try:
+ await asyncio.to_thread(self._conn.key_down, key)
+ return {"success": True}
+ except Exception as e:
+ return {"success": False, "error": str(e)}
+
+ async def key_up(self, key: str) -> Dict[str, Any]:
+ try:
+ await asyncio.to_thread(self._conn.key_up, key)
+ return {"success": True}
+ except Exception as e:
+ return {"success": False, "error": str(e)}
+
+ async def hotkey(self, keys: List[str]) -> Dict[str, Any]:
+ try:
+ await asyncio.to_thread(self._conn.hotkey, keys)
+ return {"success": True}
+ except Exception as e:
+ return {"success": False, "error": str(e)}
+
+ # -- Scrolling ----------------------------------------------------------
+
+ async def scroll(self, x: int, y: int) -> Dict[str, Any]:
+ try:
+ await asyncio.to_thread(self._conn.scroll, x, y)
+ return {"success": True}
+ except Exception as e:
+ return {"success": False, "error": str(e)}
+
+ async def scroll_down(self, clicks: int = 1) -> Dict[str, Any]:
+ return await self.scroll(0, clicks)
+
+ async def scroll_up(self, clicks: int = 1) -> Dict[str, Any]:
+ return await self.scroll(0, -clicks)
+
+ # -- Screen info --------------------------------------------------------
+
+ async def get_screen_size(self) -> Dict[str, Any]:
+ try:
+ w, h = await asyncio.to_thread(lambda: self._conn.screen_size)
+ return {"success": True, "size": {"width": w, "height": h}}
+ except Exception as e:
+ return {"success": False, "error": str(e)}
+
+ async def get_cursor_position(self) -> Dict[str, Any]:
+ try:
+ x, y = self._conn.cursor_position
+ return {"success": True, "position": {"x": x, "y": y}}
+ except Exception as e:
+ return {"success": False, "error": str(e)}
+
+ # Clipboard and run_command inherited from BaseAutomationHandler
+
+
+# ---------------------------------------------------------------------------
+# Accessibility Handler (stub)
+# ---------------------------------------------------------------------------
+
+
+class VNCAccessibilityHandler(BaseAccessibilityHandler):
+ """Stub accessibility handler for VNC — no accessibility tree available."""
+
+ async def get_accessibility_tree(self) -> Dict[str, Any]:
+ return {
+ "success": True,
+ "tree": {
+ "role": "Window",
+ "title": "VNC Remote Desktop",
+ "position": {"x": 0, "y": 0},
+ "size": {"width": 0, "height": 0},
+ "children": [],
+ },
+ }
+
+ async def find_element(
+ self,
+ role: Optional[str] = None,
+ title: Optional[str] = None,
+ value: Optional[str] = None,
+ ) -> Dict[str, Any]:
+ return {
+ "success": False,
+ "error": "Accessibility tree not available over VNC",
+ }
diff --git a/libs/python/computer-server/computer_server/handlers/windows.py b/libs/python/computer-server/computer_server/handlers/windows.py
index fb6ec6c85c..c4af537a84 100644
--- a/libs/python/computer-server/computer_server/handlers/windows.py
+++ b/libs/python/computer-server/computer_server/handlers/windows.py
@@ -695,42 +695,9 @@ class WindowsAutomationHandler(BaseAutomationHandler):
except Exception as e:
return {"success": False, "error": str(e)}
- # Clipboard Actions
- async def copy_to_clipboard(self) -> Dict[str, Any]:
- """Get the current content of the clipboard.
+ # Clipboard inherited from BaseAutomationHandler
- Returns:
- Dict[str, Any]: A dictionary containing the success status and either
- clipboard content or an error message.
- Structure: {"success": bool, "content": str} or
- {"success": bool, "error": str}
- """
- try:
- import pyperclip
-
- content = pyperclip.paste()
- return {"success": True, "content": content}
- except Exception as e:
- return {"success": False, "error": str(e)}
-
- async def set_clipboard(self, text: str) -> Dict[str, Any]:
- """Set the clipboard content to the specified text.
-
- Args:
- text (str): The text to copy to the clipboard.
-
- Returns:
- Dict[str, Any]: A dictionary with success status and optional error message.
- """
- try:
- import pyperclip
-
- pyperclip.copy(text)
- return {"success": True}
- except Exception as e:
- return {"success": False, "error": str(e)}
-
- # Command Execution
+ # Command Execution (Windows override for multi-encoding support)
async def run_command(self, command: str) -> Dict[str, Any]:
"""Execute a shell command asynchronously.
diff --git a/libs/python/computer-server/pyproject.toml b/libs/python/computer-server/pyproject.toml
index a1f1dfc67b..89eae4a580 100644
--- a/libs/python/computer-server/pyproject.toml
+++ b/libs/python/computer-server/pyproject.toml
@@ -49,6 +49,9 @@ linux = [
windows = [
"pywin32>=310"
]
+vnc = [
+ "vncdotool>=1.2.0"
+]
[project.urls]
homepage = "https://github.com/trycua/cua"
diff --git a/libs/python/computer/computer/computer.py b/libs/python/computer/computer/computer.py
index 599b8dbfd9..f24f87b966 100644
--- a/libs/python/computer/computer/computer.py
+++ b/libs/python/computer/computer/computer.py
@@ -112,6 +112,10 @@ class Computer:
api_key: Optional[str] = None,
experiments: Optional[List[str]] = None,
timeout: int = 100,
+ backend: Literal["native", "vnc"] = "native",
+ vnc_host: Optional[str] = None,
+ vnc_port: int = 5900,
+ vnc_password: str = "",
run_opts: Optional[Dict[str, Any]] = None,
):
"""Initialize a new Computer instance.
@@ -143,6 +147,10 @@ class Computer:
api_key: Optional API key for cloud providers (defaults to CUA_API_KEY environment variable)
experiments: Optional list of experimental features to enable (e.g. ["app-use"])
timeout: Timeout in seconds for connecting to the computer interface
+ backend: Handler backend for the computer-server: 'native' (OS-specific) or 'vnc'. Defaults to 'native'.
+ vnc_host: VNC server host (required when backend='vnc')
+ vnc_port: VNC server port (default: 5900)
+ vnc_password: VNC server password
run_opts: Optional dictionary of provider-specific run options.
"""
@@ -194,6 +202,10 @@ class Computer:
self.api_port = 8000
self.experiments = experiments or []
+ self.backend = backend
+ self.vnc_host = vnc_host
+ self.vnc_port = vnc_port
+ self.vnc_password = vnc_password
if "app-use" in self.experiments:
assert self.os_type == "macos", "App use experiment is only supported on macOS"
@@ -532,6 +544,16 @@ class Computer:
if self.shared_directories:
run_opts["shared_directories"] = shared_dirs.copy()
+ # Pass backend configuration as env vars for the computer-server
+ if self.backend == "vnc":
+ run_opts.setdefault("env", {})
+ run_opts["env"]["CUA_BACKEND"] = "vnc"
+ if self.vnc_host:
+ run_opts["env"]["CUA_VNC_HOST"] = self.vnc_host
+ run_opts["env"]["CUA_VNC_PORT"] = str(self.vnc_port)
+ if self.vnc_password:
+ run_opts["env"]["CUA_VNC_PASSWORD"] = self.vnc_password
+
# Merge custom run_opts
run_opts.update(self.custom_run_opts)
diff --git a/libs/python/cua-cli/tests/commands/test_sandbox.py b/libs/python/cua-cli/tests/commands/test_sandbox.py
index d53fdf64df..89c4ef6694 100644
--- a/libs/python/cua-cli/tests/commands/test_sandbox.py
+++ b/libs/python/cua-cli/tests/commands/test_sandbox.py
@@ -476,7 +476,9 @@ class TestCmdShell:
subparsers = parser.add_subparsers()
sandbox.register_parser(subparsers)
- args = parser.parse_args(["sb", "shell", "--cols", "120", "--rows", "40", "my-sandbox", "ls"])
+ args = parser.parse_args(
+ ["sb", "shell", "--cols", "120", "--rows", "40", "my-sandbox", "ls"]
+ )
assert args.cols == 120
assert args.rows == 40
assert args.name == "my-sandbox"