Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
feat(lume, computer-server): VNC backend rewrite and VirtioFS port discovery (#1205)
Francesco Bonaccif73a68be2ac3
diff --git a/docs/content/docs/lume/reference/cli-reference.mdx b/docs/content/docs/lume/reference/cli-reference.mdxindex a8dd1355b5..1a0e8475d6 100644--- a/docs/content/docs/lume/reference/cli-reference.mdx+++ b/docs/content/docs/lume/reference/cli-reference.mdx@@ -7,16 +7,16 @@ description: Command Line Interface reference for LumeAUTO-GENERATED FILE - DO NOT EDIT DIRECTLYGenerated by: npx tsx scripts/docs-generators/lume.tsSource: libs/lume/src/Commands/*.swift- Version: 0.2.86+ Version: 0.3.0*/}import { Callout } from 'fumadocs-ui/components/callout';import { VersionHeader } from '@/components/version-selector';<VersionHeader- versions={[{"version":"0.2","href":"/lume/reference/cli-reference","isCurrent":true}]}- currentVersion="0.2"- fullVersion="0.2.86"+ versions={[{"version":"0.3","href":"/lume/reference/cli-reference","isCurrent":true},{"version":"0.2","href":"/lume/reference/v0.2/cli-reference","isCurrent":false}]}+ currentVersion="0.3"+ fullVersion="0.3.0"packageName="lume"installCommand="curl -fsSL https://raw.githubusercontent.com/trycua/cua/main/libs/lume/scripts/install.sh | bash"/>diff --git a/docs/content/docs/lume/reference/http-api.mdx b/docs/content/docs/lume/reference/http-api.mdxindex 8034a487cb..79f38961df 100644--- a/docs/content/docs/lume/reference/http-api.mdx+++ b/docs/content/docs/lume/reference/http-api.mdx@@ -7,7 +7,7 @@ description: HTTP API reference for Lume serverAUTO-GENERATED FILE - DO NOT EDIT DIRECTLYGenerated by: npx tsx scripts/docs-generators/lume.tsSource: libs/lume/src/Server/*.swift- Version: 0.2.86+ Version: 0.3.0*/}import { Callout } from 'fumadocs-ui/components/callout';@@ -15,9 +15,9 @@ import { Tabs, Tab } from 'fumadocs-ui/components/tabs';import { VersionHeader } from '@/components/version-selector';<VersionHeader- versions={[{"version":"0.2","href":"/lume/reference/cli-reference","isCurrent":true}]}- currentVersion="0.2"- fullVersion="0.2.86"+ versions={[{"version":"0.3","href":"/lume/reference/cli-reference","isCurrent":true},{"version":"0.2","href":"/lume/reference/v0.2/cli-reference","isCurrent":false}]}+ currentVersion="0.3"+ fullVersion="0.3.0"packageName="lume"installCommand="curl -fsSL https://raw.githubusercontent.com/trycua/cua/main/libs/lume/scripts/install.sh | bash"/>diff --git a/libs/lume/scripts/setup-cua-computer.sh b/libs/lume/scripts/setup-cua.shsimilarity index 50%rename from libs/lume/scripts/setup-cua-computer.shrename to libs/lume/scripts/setup-cua.shindex c5d7f5aaa8..1e4b117b89 100755--- a/libs/lume/scripts/setup-cua-computer.sh+++ b/libs/lume/scripts/setup-cua.sh@@ -1,21 +1,52 @@#!/bin/bash#-# setup-cua-computer.sh-# Makes a Lume macOS VM image cua-computer compatible+# setup-cua.sh+# Makes a Lume macOS VM fully cua-compatible## This script runs ON the VM itself (not the host).-# It installs cua-computer-server, cua-agent, Playwright, and configures-# the VM for headless CUA operation (auto-login, disable sleep, LaunchAgent).+# It installs cua-computer-server, cua-agent, Playwright, noVNC (web-based VNC+# proxying to lume's built-in VNC on the host), supervisor, and configures the+# VM for headless CUA operation (auto-login, disable sleep, LaunchAgent).+#+# In Lume VMs, the display is provided by the Virtualization Framework and+# exposed through lume's built-in VNC server on the host. macOS Screen Sharing+# inside the VM has no framebuffer to capture. Therefore, noVNC/websockify+# inside the VM proxies to lume's VNC on the host (via the gateway IP).## Usage:-# lume ssh <vm-name> "curl -fsSL <url> | bash"-# lume ssh <vm-name> < setup-cua-computer.sh+# # Copy the script into the VM first, then run with a generous timeout:+# lume ssh <vm-name> "cat > /tmp/setup.sh && chmod +x /tmp/setup.sh" < setup-cua.sh+# lume ssh <vm-name> --timeout 600 "bash /tmp/setup.sh --yes"+## # Or copy and run directly inside the VM:-# ./setup-cua-computer.sh [--yes] [--port PORT]+# ./setup-cua.sh [--yes] [--port PORT] [--host-vnc-port PORT]+#+# Important: Start the VM with a fixed VNC port and password so websockify can find it:+# lume run <vm-name> --vnc-port 5901 --vnc-password lume+#+# Note: Port 5900 is often used by the host's Screen Sharing service, so we+# default to 5901. Use --host-vnc-port to match whatever --vnc-port you pass+# to lume run.+#+# Accessing noVNC remotely (from outside the host):+# 1. SSH tunnel: ssh -L 6080:<vm-ip>:6080 user@host+# 2. Open: http://localhost:6080/vnc.html?host=localhost&port=6080+# The ?host=localhost&port=6080 params ensure the websocket connects through+# the tunnel instead of trying to reach the VM IP directly.+#+# Note: The default lume ssh timeout is 60s, which is too short for this script+# (Homebrew + Xcode CLT + pip installs can take 5-10 minutes). Use --timeout 600+# or --timeout 0 (unlimited) when running via lume ssh.## Prerequisites:# - macOS VM created via lume (with SSH enabled, user lume/lume)#+# Host troubleshooting:+# If VMs fail to start with "Unable to access security information", the host's+# login keychain is likely locked (common after host reboot/crash on macOS Sequoia+).+# Fix: security unlock-keychain -p '<host-password>' ~/Library/Keychains/login.keychain-db+# Prevention: enable auto-login on the host so the keychain unlocks at boot.+#set -e@@ -33,6 +64,9 @@ AUTO_YES=falseCUA_SERVER_PORT=8443VM_PASSWORD="${VM_PASSWORD:-lume}"CUA_DIR="$HOME/.cua-server"+NOVNC_PORT=6080+HOST_VNC_PORT="${HOST_VNC_PORT:-5901}"+VNC_PASSWORD="${VNC_PASSWORD:-lume}"# Parse argumentswhile [ "$#" -gt 0 ]; do@@ -44,13 +78,28 @@ while [ "$#" -gt 0 ]; doCUA_SERVER_PORT="$2"shift;;+ --host-vnc-port)+ HOST_VNC_PORT="$2"+ shift+ ;;+ --vnc-password)+ VNC_PASSWORD="$2"+ shift+ ;;--help)echo "Usage: $0 [OPTIONS]"echo ""echo "Options:"- echo " --yes, -y Non-interactive mode (accept all prompts)"- echo " --port PORT CUA computer server port (default: 8443)"- echo " --help Show this help message"+ echo " --yes, -y Non-interactive mode (accept all prompts)"+ echo " --port PORT CUA computer server port (default: 8443)"+ echo " --host-vnc-port PORT Lume VNC port on the host (default: 5901)"+ echo " --vnc-password PWD Lume VNC password (from 'lume ls' output)"+ echo " --help Show this help message"+ echo ""+ echo "Environment variables:"+ echo " VM_PASSWORD VM user password (default: lume)"+ echo " HOST_VNC_PORT Lume VNC port on the host (default: 5901)"+ echo " VNC_PASSWORD Lume VNC password (from 'lume ls' output)"exit 0;;*)@@ -64,6 +113,12 @@ doneCURRENT_USER="$(whoami)"USER_HOME="$HOME"+# Auto-detect host gateway IP+GATEWAY_IP=$(route get default 2>/dev/null | grep gateway | awk '{print $2}')+if [ -z "$GATEWAY_IP" ]; then+ GATEWAY_IP="192.168.64.1"+fi+# ============================================# Helpers# ============================================@@ -174,7 +229,7 @@ install_system_deps() {ensure_brewlocal deps_needed=()- for pkg in ffmpeg; do+ for pkg in ffmpeg supervisor wget; doif ! brew list "$pkg" &>/dev/null; thendeps_needed+=("$pkg")fi@@ -227,17 +282,94 @@ setup_cua_server() {pip install playwright && playwright install firefox || \echo -e "${YELLOW}Playwright install failed (non-blocking, BrowserTool may not work)${NC}"+ # Install websockify for noVNC and vncdotool for VNC backend+ echo "Installing websockify and vncdotool..."+ pip install websockify vncdotool+deactivateecho -e "${GREEN}CUA server and agent installed${NC}"}# ============================================-# Phase 5: Startup Script + LaunchAgent+# Phase 5: noVNC + Websockify+# ============================================++setup_novnc() {+ print_phase "Phase 5: noVNC (Web-based VNC)"++ if ! confirm "Install noVNC and websockify (web VNC on port $NOVNC_PORT)?"; then+ echo -e "${YELLOW}Skipping noVNC setup.${NC}"+ return 0+ fi++ ensure_brew++ local NOVNC_DIR="/usr/local/share/novnc"++ setup_sudo++ # Download noVNC+ echo "Installing noVNC..."+ cd /tmp+ rm -rf noVNC-master noVNC-master.zip+ wget -q https://github.com/trycua/noVNC/archive/refs/heads/master.zip -O noVNC-master.zip+ unzip -q noVNC-master.zip++ if [ "$AUTO_YES" = true ]; then+ sudo -A rm -rf "$NOVNC_DIR"+ sudo -A mkdir -p "$NOVNC_DIR"+ sudo -A mv noVNC-master/* "$NOVNC_DIR/"+ sudo -A ln -sf "$NOVNC_DIR/vnc.html" "$NOVNC_DIR/index.html"+ sudo -A chmod -R 755 "$NOVNC_DIR"+ else+ sudo rm -rf "$NOVNC_DIR"+ sudo mkdir -p "$NOVNC_DIR"+ sudo mv noVNC-master/* "$NOVNC_DIR/"+ sudo ln -sf "$NOVNC_DIR/vnc.html" "$NOVNC_DIR/index.html"+ sudo chmod -R 755 "$NOVNC_DIR"+ fi++ rm -rf noVNC-master noVNC-master.zip++ # Configure supervisor for websockify+ # Proxies to lume's built-in VNC on the host via the gateway IP.+ # The host VNC port must match the --vnc-port used with lume run.+ local SUPERVISOR_CONF_DIR="/opt/homebrew/etc/supervisor.d"+ local SUPERVISOR_LOG_DIR="/opt/homebrew/var/log/supervisor"+ local VENV_PYTHON="$CUA_DIR/venv/bin/python3"++ mkdir -p "$SUPERVISOR_CONF_DIR"+ mkdir -p "$SUPERVISOR_LOG_DIR"++ cat > "$SUPERVISOR_CONF_DIR/websockify.ini" <<EOF+[program:websockify]+command=$VENV_PYTHON -m websockify --web $NOVNC_DIR $NOVNC_PORT $GATEWAY_IP:$HOST_VNC_PORT+directory=$NOVNC_DIR+autostart=true+autorestart=true+startsecs=5+startretries=3+stopwaitsecs=10+redirect_stderr=true+stdout_logfile=$SUPERVISOR_LOG_DIR/websockify.log+stdout_logfile_maxbytes=10MB+stdout_logfile_backups=5+environment=HOME="$USER_HOME",PATH="/opt/homebrew/bin:/usr/local/bin:/usr/bin:/bin",DISPLAY=":0"+EOF++ chmod 644 "$SUPERVISOR_CONF_DIR/websockify.ini"++ echo -e "${GREEN}noVNC installed${NC}"+ echo " websockify will proxy port $NOVNC_PORT -> $GATEWAY_IP:$HOST_VNC_PORT (lume VNC)"+}++# ============================================+# Phase 6: Startup Script + LaunchAgent# ============================================create_startup_script() {- print_phase "Phase 5: Startup Script & LaunchAgent"+ print_phase "Phase 6: Startup Script & LaunchAgent"if ! confirm "Create CUA server startup script and LaunchAgent?"; thenecho -e "${YELLOW}Skipping LaunchAgent setup.${NC}"@@ -260,6 +392,48 @@ source venv/bin/activateexport DISPLAY=:0+# Auto-detect host gateway IP for VNC backend+GATEWAY_IP=$(route get default 2>/dev/null | grep gateway | awk '{print $2}')+if [ -z "$GATEWAY_IP" ]; then+ GATEWAY_IP="192.168.64.1"+fi++# Read VNC config from lume shared directory if available.+# Lume automatically shares a "lume-config" VirtioFS volume containing vnc.env+# with the assigned VNC port and password for this VM.+LUME_CONFIG_MOUNT="/tmp/lume-config"+LUME_CONFIG="$LUME_CONFIG_MOUNT/vnc.env"+LUME_VNC_PORT=""+LUME_VNC_PASSWORD=""++# Try to mount the lume-config VirtioFS share and read VNC config+mkdir -p "$LUME_CONFIG_MOUNT" 2>/dev/null+if mount | grep -q "lume-config"; then+ echo "lume-config already mounted" >> "$LOG_FILE"+elif mount_virtiofs lume-config "$LUME_CONFIG_MOUNT" 2>/dev/null; then+ echo "Mounted lume-config VirtioFS share" >> "$LOG_FILE"+else+ echo "lume-config VirtioFS share not available, using defaults" >> "$LOG_FILE"+fi++# Wait up to 10s for vnc.env to appear (written by host after VNC starts)+for i in $(seq 1 10); do+ if [ -f "$LUME_CONFIG" ]; then+ eval "$(cat "$LUME_CONFIG")"+ LUME_VNC_PORT="${VNC_PORT:-}"+ LUME_VNC_PASSWORD="${VNC_PASSWORD:-}"+ echo "Read VNC config from lume shared dir: port=$LUME_VNC_PORT" >> "$LOG_FILE"+ break+ fi+ sleep 1+done++# Set VNC backend env vars — prefer lume-config values, fall back to defaults+export CUA_BACKEND=vnc+export CUA_VNC_HOST="$GATEWAY_IP"+export CUA_VNC_PORT="${LUME_VNC_PORT:-__VNC_PORT__}"+export CUA_VNC_PASSWORD="${LUME_VNC_PASSWORD:-__VNC_PASSWORD__}"+# Update packagesecho "Updating cua-agent..." >> "$LOG_FILE"pip install --upgrade --no-input "cua-agent[all]" >> "$LOG_FILE" 2>&1 || true@@ -272,13 +446,16 @@ echo "Ensuring Playwright Firefox..." >> "$LOG_FILE"pip install --upgrade --no-input playwright >> "$LOG_FILE" 2>&1 || trueplaywright install firefox >> "$LOG_FILE" 2>&1 || true-# Start server-echo "Starting CUA computer server on port __PORT__..." >> "$LOG_FILE"+# Start server with VNC backend pointing at lume's VNC on the host+echo "Starting CUA computer server (vnc backend) on port __PORT__..." >> "$LOG_FILE"+echo " VNC target: $GATEWAY_IP:__VNC_PORT__ (password: __VNC_PASSWORD__)" >> "$LOG_FILE"python -m computer_server --port __PORT__ >> "$LOG_FILE" 2>&1SCRIPT_EOF- # Stamp the port+ # Stamp the port and VNC settingssed -i '' "s/__PORT__/$CUA_SERVER_PORT/g" "$STARTUP_SCRIPT"+ sed -i '' "s/__VNC_PORT__/$HOST_VNC_PORT/g" "$STARTUP_SCRIPT"+ sed -i '' "s/__VNC_PASSWORD__/$VNC_PASSWORD/g" "$STARTUP_SCRIPT"chmod +x "$STARTUP_SCRIPT"# Create LaunchAgent@@ -309,6 +486,12 @@ SCRIPT_EOF<string>$USER_HOME</string><key>DISPLAY</key><string>:0</string>+ <key>CUA_BACKEND</key>+ <string>vnc</string>+ <key>CUA_VNC_PORT</key>+ <string>$HOST_VNC_PORT</string>+ <key>CUA_VNC_PASSWORD</key>+ <string>$VNC_PASSWORD</string></dict><key>ProgramArguments</key>@@ -346,13 +529,13 @@ EOF}# ============================================-# Phase 6: System Configuration+# Phase 7: System Configuration + Auto-Login# ============================================configure_system() {- print_phase "Phase 6: System Configuration"+ print_phase "Phase 7: System Configuration"- if ! confirm "Disable screen lock, screensaver, and sleep?"; then+ if ! confirm "Disable screen lock, screensaver, sleep, and enable auto-login?"; thenecho -e "${YELLOW}Skipping system configuration.${NC}"return 0fi@@ -374,11 +557,67 @@ configure_system() {# Enable full keyboard navigation (needed for accessibility automation)defaults write NSGlobalDomain AppleKeyboardUIMode -int 3- echo -e "${GREEN}System configured (no sleep, no screensaver, full keyboard nav)${NC}"+ # Configure auto-login (ensures GUI session + unlocked keychain on boot)+ echo "Configuring auto-login for $CURRENT_USER..."++ if [ "$AUTO_YES" = true ]; then+ sudo -A defaults write /Library/Preferences/com.apple.loginwindow autoLoginUser "$CURRENT_USER"+ else+ sudo defaults write /Library/Preferences/com.apple.loginwindow autoLoginUser "$CURRENT_USER"+ fi++ # Create kcpassword file (XOR-encrypted password for auto-login)+ python3 << PYEOF+password = "$VM_PASSWORD"+key = bytes.fromhex("7d895223d2bcddeaa3b91f")+padded_len = ((len(password) + 11) // 12) * 12+padded = password + "\x00" * (padded_len - len(password))+result = bytearray()+for i, char in enumerate(padded.encode("utf-8")):+ result.append(char ^ key[i % len(key)])+with open("/tmp/kcpassword", "wb") as f:+ f.write(result)+PYEOF++ if [ "$AUTO_YES" = true ]; then+ sudo -A mv /tmp/kcpassword /etc/kcpassword+ sudo -A chown root:wheel /etc/kcpassword+ sudo -A chmod 600 /etc/kcpassword+ else+ sudo mv /tmp/kcpassword /etc/kcpassword+ sudo chown root:wheel /etc/kcpassword+ sudo chmod 600 /etc/kcpassword+ fi++ echo -e "${GREEN}System configured (no sleep, no screensaver, full keyboard nav, auto-login)${NC}"}# ============================================-# Phase 7: Verification+# Phase 8: Start Services+# ============================================++start_services() {+ print_phase "Phase 8: Start Services"++ ensure_brew++ # Start supervisor (manages websockify)+ if command -v brew &>/dev/null; then+ echo "Starting supervisor..."+ brew services start supervisor 2>/dev/null || true+ sleep 3++ if command -v supervisorctl &>/dev/null; then+ supervisorctl reread 2>/dev/null || true+ supervisorctl update 2>/dev/null || true+ fi+ fi++ echo -e "${GREEN}Services started${NC}"+}++# ============================================+# Phase 9: Verification# ============================================verify() {@@ -386,6 +625,7 @@ verify() {echo "System:"echo " User: $CURRENT_USER"+ echo " Gateway: $GATEWAY_IP"sw_vers | sed 's/^/ /'echo ""@@ -431,6 +671,22 @@ verify() {echo -e " ${YELLOW}~${NC} Playwright (may not be installed)"fi+ # noVNC / websockify+ if command -v supervisorctl &>/dev/null && supervisorctl status websockify 2>/dev/null | grep -q RUNNING; then+ echo -e " ${GREEN}✓${NC} noVNC + websockify (port $NOVNC_PORT -> $GATEWAY_IP:$HOST_VNC_PORT)"+ elif [ -d "/usr/local/share/novnc" ]; then+ echo -e " ${YELLOW}~${NC} noVNC installed (websockify may not be running yet)"+ else+ echo -e " ${RED}✗${NC} noVNC"+ fi++ # Auto-login+ if defaults read /Library/Preferences/com.apple.loginwindow autoLoginUser 2>/dev/null | grep -q "$CURRENT_USER"; then+ echo -e " ${GREEN}✓${NC} Auto-login ($CURRENT_USER)"+ else+ echo -e " ${YELLOW}~${NC} Auto-login (may need reboot)"+ fi+# LaunchAgentlocal PLIST="$USER_HOME/Library/LaunchAgents/com.trycua.computer_server.plist"if [ -f "$PLIST" ]; then@@ -444,13 +700,29 @@ verify() {echo -e "${GREEN} Setup Complete!${NC}"echo -e "${GREEN}========================================${NC}"echo ""- echo "CUA computer server will start on port $CUA_SERVER_PORT at next login."+ echo "Services:"+ echo " CUA server: port $CUA_SERVER_PORT (starts at login via LaunchAgent)"+ echo " noVNC: port $NOVNC_PORT -> $GATEWAY_IP:$HOST_VNC_PORT (lume VNC)"+ echo ""+ echo "Start the VM with a fixed VNC port and password (from the host):"+ echo " lume run <vm-name> --vnc-port $HOST_VNC_PORT --vnc-password $VNC_PASSWORD"echo ""- echo "To start it now (from the host):"+ echo "To start CUA server now (from the host):"echo " lume ssh <vm-name> 'launchctl load ~/Library/LaunchAgents/com.trycua.computer_server.plist'"echo ""echo "To test from the host:"- echo " curl http://<vm-ip>:$CUA_SERVER_PORT/health"+ echo " curl http://<vm-ip>:$CUA_SERVER_PORT/status"+ echo ""+ echo "To access noVNC from the host network:"+ echo " http://<vm-ip>:$NOVNC_PORT/vnc.html"+ echo ""+ echo "To access noVNC remotely (from outside the host):"+ echo " 1. SSH tunnel: ssh -L $NOVNC_PORT:<vm-ip>:$NOVNC_PORT user@host"+ echo " 2. Open: http://localhost:$NOVNC_PORT/vnc.html?host=localhost&port=$NOVNC_PORT"+ echo ""+ echo "VNC password: $VNC_PASSWORD"+ echo ""+ echo "NOTE: A reboot is recommended for auto-login to take effect."echo ""}@@ -463,14 +735,16 @@ echo " ⠀⣀⣀⡀⠀⠀⠀⠀⢀⣀⣀⣀⡀⠘⠋⢉⠙⣷⠀⠀ ⠀"echo " ⠀⠀⢀⣴⣿⡿⠋⣉⠁⣠⣾⣿⣿⣿⣿⡿⠿⣦⡈⠀⣿⡇⠃⠀"echo " ⠀⠀⠀⣽⣿⣧⠀⠃⢰⣿⣿⡏⠙⣿⠿⢧⣀⣼⣷⠀⡿⠃⠀⠀"echo " ⠀⠀⠀⠉⣿⣿⣦⠀⢿⣿⣿⣷⣾⡏⠀⠀⢹⣿⣿⠀⠀⠀⠀⠀⠀"-echo -e " ⠀⠀⠀⠀⠀⠉⠛⠁⠈⠿⣿⣿⣿⣷⣄⣠⡼⠟⠁${NC} ${BLUE}CUA Computer Setup${NC}"-echo -e "${BLUE} Make a Lume VM cua-computer compatible${NC}"+echo -e " ⠀⠀⠀⠀⠀⠉⠛⠁⠈⠿⣿⣿⣿⣷⣄⣠⡼⠟⠁${NC} ${BLUE}CUA Setup${NC}"+echo -e "${BLUE} Make a Lume VM cua-compatible${NC}"echo ""install_homebrewinstall_pythoninstall_system_depssetup_cua_server+setup_novnccreate_startup_scriptconfigure_system+start_servicesverifydiff --git a/libs/lume/src/Commands/Run.swift b/libs/lume/src/Commands/Run.swiftindex b7188ec9ae..835634d358 100644--- a/libs/lume/src/Commands/Run.swift+++ b/libs/lume/src/Commands/Run.swift@@ -45,6 +45,11 @@ struct Run: AsyncParsableCommand {help: "Port to use for the VNC server. Defaults to 0 (auto-assign)")var vncPort: Int = 0+ @Option(+ name: [.customLong("vnc-password")],+ help: "Password for the VNC server. Defaults to a random passphrase")+ var vncPassword: String?+@Option(help: "For MacOS VMs only, boot into the VM in recovery mode")var recoveryMode: Bool = false@@ -133,6 +138,7 @@ struct Run: AsyncParsableCommand {registry: registry,organization: organization,vncPort: vncPort,+ vncPassword: vncPassword,recoveryMode: recoveryMode,storage: storage,usbMassStoragePaths: parsedUSBStorageDevices.isEmpty ? nil : parsedUSBStorageDevices,diff --git a/libs/lume/src/LumeController.swift b/libs/lume/src/LumeController.swiftindex 5105cd44f4..349824016f 100644--- a/libs/lume/src/LumeController.swift+++ b/libs/lume/src/LumeController.swift@@ -904,6 +904,7 @@ final class LumeController {registry: String = "ghcr.io",organization: String = "trycua",vncPort: Int = 0,+ vncPassword: String? = nil,recoveryMode: Bool = false,storage: String? = nil,usbMassStoragePaths: [Path]? = nil,@@ -1004,6 +1005,7 @@ final class LumeController {sharedDirectories: sharedDirectories,mount: mount,vncPort: vncPort,+ vncPassword: vncPassword,recoveryMode: recoveryMode,usbMassStoragePaths: usbMassStoragePaths,networkMode: networkMode,diff --git a/libs/lume/src/VM/VM.swift b/libs/lume/src/VM/VM.swiftindex 8e2d2b9c2d..4da637d85c 100644--- a/libs/lume/src/VM/VM.swift+++ b/libs/lume/src/VM/VM.swift@@ -136,7 +136,7 @@ class VM {func run(noDisplay: Bool, sharedDirectories: [SharedDirectory], mount: Path?, vncPort: Int = 0,- recoveryMode: Bool = false, usbMassStoragePaths: [Path]? = nil,+ vncPassword: String? = nil, recoveryMode: Bool = false, usbMassStoragePaths: [Path]? = nil,networkMode: NetworkMode? = nil, clipboard: Bool = false) async throws {Logger.info(@@ -215,13 +215,29 @@ class VM {// Create and configure the VMdo {+ // Create a lume-config shared directory so the guest can discover+ // the VNC port/password at boot. The directory is created empty now+ // and populated after the VNC server starts (VirtioFS exposes live+ // host directory contents, so the guest will see the file once written).+ let lumeConfigDir = FileManager.default.temporaryDirectory+ .appendingPathComponent("lume-config-\(vmDirContext.name)")+ try? FileManager.default.createDirectory(at: lumeConfigDir, withIntermediateDirectories: true)+ // Remove stale vnc.env from a previous run so the guest doesn't+ // read outdated port/password before the new file is written.+ try? FileManager.default.removeItem(+ at: lumeConfigDir.appendingPathComponent("vnc.env"))+ let lumeConfigSharedDir = SharedDirectory(+ hostPath: lumeConfigDir.path, tag: "lume-config", readOnly: true)+ var allSharedDirectories = sharedDirectories+ allSharedDirectories.append(lumeConfigSharedDir)+Logger.info("Creating virtualization service context", metadata: ["name": vmDirContext.name])let config = try createVMVirtualizationServiceContext(cpuCount: cpuCount,memorySize: memorySize,display: vmDirContext.config.display.string,- sharedDirectories: sharedDirectories,+ sharedDirectories: allSharedDirectories,mount: mount,recoveryMode: recoveryMode,usbMassStoragePaths: usbMassStoragePaths,@@ -246,7 +262,21 @@ class VM {"port": "\(vncPort)",])let vncInfo = try await setupSession(- port: vncPort, sharedDirectories: sharedDirectories)+ port: vncPort, password: vncPassword, sharedDirectories: sharedDirectories)++ // Write VNC connection info to the shared config directory so the+ // guest can read it via mount_virtiofs.+ // URL format: vnc://:password@host:port — URLComponents needs http:// to parse correctly.+ if let components = URLComponents(string: vncInfo.replacingOccurrences(of: "vnc://", with: "http://")),+ let port = components.port {+ let password = components.password ?? ""+ let envContent = "VNC_PORT=\(port)\nVNC_PASSWORD=\(password)\n"+ try? envContent.write(+ to: lumeConfigDir.appendingPathComponent("vnc.env"),+ atomically: true, encoding: .utf8)+ Logger.info("Wrote VNC config to shared directory", metadata: [+ "port": "\(port)", "path": lumeConfigDir.path])+ }Logger.info("VNC setup successful", metadata: ["name": vmDirContext.name, "vncInfo": vncInfo])@@ -658,12 +688,12 @@ class VM {}/// Sets up the VNC service and returns the VNC URL- private func startVNCService(port: Int = 0) async throws -> String {+ private func startVNCService(port: Int = 0, password: String? = nil) async throws -> String {guard let service = virtualizationService else {throw VMError.internalError("Virtualization service not initialized")}- try await vncService.start(port: port, virtualMachine: service.getVirtualMachine())+ try await vncService.start(port: port, password: password, virtualMachine: service.getVirtualMachine())guard let url = vncService.url else {throw VMError.vncNotConfigured@@ -692,10 +722,10 @@ class VM {/// Main session setup method that handles VNC and persists session dataprivate func setupSession(- port: Int = 0, sharedDirectories: [SharedDirectory] = []+ port: Int = 0, password: String? = nil, sharedDirectories: [SharedDirectory] = []) async throws -> String {// Start the VNC service and get the URL- let url = try await startVNCService(port: port)+ let url = try await startVNCService(port: port, password: password)// Save the session datasaveSessionData(url: url, sharedDirectories: sharedDirectories)@@ -901,11 +931,22 @@ class VM {// Create and configure the VMdo {+ // Create lume-config shared directory for VNC discovery+ let lumeConfigDir = FileManager.default.temporaryDirectory+ .appendingPathComponent("lume-config-\(vmDirContext.name)")+ try? FileManager.default.createDirectory(at: lumeConfigDir, withIntermediateDirectories: true)+ try? FileManager.default.removeItem(+ at: lumeConfigDir.appendingPathComponent("vnc.env"))+ let lumeConfigSharedDir = SharedDirectory(+ hostPath: lumeConfigDir.path, tag: "lume-config", readOnly: true)+ var allSharedDirectories = sharedDirectories+ allSharedDirectories.append(lumeConfigSharedDir)+let config = try createVMVirtualizationServiceContext(cpuCount: cpuCount,memorySize: memorySize,display: vmDirContext.config.display.string,- sharedDirectories: sharedDirectories,+ sharedDirectories: allSharedDirectories,mount: mount,recoveryMode: recoveryMode,usbMassStoragePaths: usbImagePaths@@ -916,6 +957,16 @@ class VM {port: vncPort, sharedDirectories: sharedDirectories)Logger.info("VNC info", metadata: ["vncInfo": vncInfo])+ // Write VNC config to shared directory for guest discovery+ if let components = URLComponents(string: vncInfo.replacingOccurrences(of: "vnc://", with: "http://")),+ let port = components.port {+ let password = components.password ?? ""+ let envContent = "VNC_PORT=\(port)\nVNC_PASSWORD=\(password)\n"+ try? envContent.write(+ to: lumeConfigDir.appendingPathComponent("vnc.env"),+ atomically: true, encoding: .utf8)+ }+// Start the VMguard let service = virtualizationService else {throw VMError.internalError("Virtualization service not initialized")diff --git a/libs/lume/src/VNC/VNCService.swift b/libs/lume/src/VNC/VNCService.swiftindex ed41c3b684..67fdbb204c 100644--- a/libs/lume/src/VNC/VNCService.swift+++ b/libs/lume/src/VNC/VNCService.swift@@ -8,7 +8,7 @@ import Network@MainActorprotocol VNCService {var url: String? { get }- func start(port: Int, virtualMachine: Any?) async throws+ func start(port: Int, password: String?, virtualMachine: Any?) async throwsfunc stop()func openClient(url: String) async throws@@ -69,8 +69,8 @@ final class DefaultVNCService: VNCService {}}- func start(port: Int, virtualMachine: Any?) async throws {- let password = Array(PassphraseGenerator().prefix(4)).joined(separator: "-")+ func start(port: Int, password: String? = nil, virtualMachine: Any?) async throws {+ let password = password ?? Array(PassphraseGenerator().prefix(4)).joined(separator: "-")let securityConfiguration = Dynamic._VZVNCAuthenticationSecurityConfiguration(password: password)// Store password for later VNC client connectiondiff --git a/libs/lume/tests/Mocks/MockVM.swift b/libs/lume/tests/Mocks/MockVM.swiftindex 1e24d08da9..75a68ddf3c 100644--- a/libs/lume/tests/Mocks/MockVM.swift+++ b/libs/lume/tests/Mocks/MockVM.swift@@ -23,13 +23,13 @@ class MockVM: VM {override func run(noDisplay: Bool, sharedDirectories: [SharedDirectory], mount: Path?, vncPort: Int = 0,- recoveryMode: Bool = false, usbMassStoragePaths: [Path]? = nil,+ vncPassword: String? = nil, recoveryMode: Bool = false, usbMassStoragePaths: [Path]? = nil,networkMode: NetworkMode? = nil, clipboard: Bool = false) async throws {mockIsRunning = truetry await super.run(noDisplay: noDisplay, sharedDirectories: sharedDirectories, mount: mount,- vncPort: vncPort, recoveryMode: recoveryMode,+ vncPort: vncPort, vncPassword: vncPassword, recoveryMode: recoveryMode,usbMassStoragePaths: usbMassStoragePaths,networkMode: networkMode, clipboard: clipboard)diff --git a/libs/lume/tests/Mocks/MockVNCService.swift b/libs/lume/tests/Mocks/MockVNCService.swiftindex 215053d0f6..1bc8f16d8d 100644--- a/libs/lume/tests/Mocks/MockVNCService.swift+++ b/libs/lume/tests/Mocks/MockVNCService.swift@@ -29,7 +29,7 @@ final class MockVNCService: VNCService {return nil}- func start(port: Int, virtualMachine: Any?) async throws {+ func start(port: Int, password: String? = nil, virtualMachine: Any?) async throws {isRunning = trueurl = "vnc://localhost:\(port)"_attachedVM = virtualMachinediff --git a/libs/python/computer-server/computer_server/cli.py b/libs/python/computer-server/computer_server/cli.pyindex 3ea0c76f0d..ca3ef52551 100644--- a/libs/python/computer-server/computer_server/cli.py+++ b/libs/python/computer-server/computer_server/cli.py@@ -120,7 +120,9 @@ def main() -> None:os.environ["CUA_BACKEND"] = "vnc"if args.vnc_host:os.environ["CUA_VNC_HOST"] = args.vnc_host- os.environ["CUA_VNC_PORT"] = str(args.vnc_port)+ # Only override env vars from CLI args when explicitly provided+ if args.vnc_port != 5900 or "CUA_VNC_PORT" not in os.environ:+ os.environ["CUA_VNC_PORT"] = str(args.vnc_port)if args.vnc_password:os.environ["CUA_VNC_PASSWORD"] = args.vnc_passwordvnc_host = args.vnc_host or os.environ.get("CUA_VNC_HOST")diff --git a/libs/python/computer-server/computer_server/handlers/vnc.py b/libs/python/computer-server/computer_server/handlers/vnc.pyindex c691e01ab2..7285907b82 100644--- a/libs/python/computer-server/computer_server/handlers/vnc.py+++ b/libs/python/computer-server/computer_server/handlers/vnc.py@@ -21,7 +21,6 @@ Usage:import asyncioimport base64import logging-import threadingfrom io import BytesIOfrom typing import Any, Dict, List, Optional, Tuple@@ -81,7 +80,12 @@ def _translate_key(key: str) -> str:class _VNCConnection:- """Manages a vncdotool ThreadedVNCClientProxy lifecycle.+ """Manages vncdotool client connections to a VNC server.++ Uses fresh connections per operation to avoid a bug in vncdotool's+ ThreadedVNCClientProxy where the Twisted deferred chain corrupts+ coordinate state after operations like refreshScreen. Each public+ method creates its own connection, executes, and disconnects.All methods are synchronous (blocking) — the handler calls them viaasyncio.to_thread to avoid blocking the event loop.@@ -91,164 +95,272 @@ class _VNCConnection:self._host = hostself._port = portself._password = password- self._client = None- self._lock = threading.Lock()- # Track cursor position locally (vncdotool also tracks via client.x/y)+ # Track cursor position locallyself._cursor_x: int = 0self._cursor_y: int = 0- def _ensure_connected(self):- """Lazily connect to the VNC server. Returns the vncdotool client."""- if self._client is not None:- return self._client- with self._lock:- if self._client is not None:- return self._client- from vncdotool import api-- server_str = f"{self._host}::{self._port}"- logger.info(f"VNC connecting to {self._host}:{self._port}")- client = api.connect(- server_str,- password=self._password or None,+ def _with_client(self, fn):+ """Execute fn(client) with a fresh VNC connection via the global Twisted reactor.++ Starts the reactor in a background thread on first use, then reuses it.+ ``fn(client)`` receives a raw ``VNCDoToolClient`` whose methods return+ Twisted Deferreds.+ """+ import threading++ from twisted.internet import defer, reactor+ from vncdotool.client import VNCDoToolFactory++ result_holder: list = [None]+ error_holder: list = [None]+ done_event = threading.Event()++ def _work():+ factory = VNCDoToolFactory()+ factory.password = self._password or None++ @defer.inlineCallbacks+ def _do():+ client = None+ try:+ reactor.connectTCP(self._host, self._port, factory)+ client = yield factory.deferred+ res = yield defer.maybeDeferred(fn, client)+ result_holder[0] = res+ except Exception as e:+ error_holder[0] = e+ finally:+ if client is not None and hasattr(client, "transport") and client.transport:+ client.transport.loseConnection()+ done_event.set()++ _do()++ # Ensure the reactor is running in a background thread+ if not reactor.running:+ t = threading.Thread(+ target=reactor.run,+ kwargs={"installSignalHandlers": False},+ daemon=True,)- client.timeout = 30- self._client = client- logger.info("VNC connected")- return client+ t.start()++ reactor.callFromThread(_work)+ done_event.wait(timeout=30)+ if not done_event.is_set():+ raise TimeoutError("VNC operation timed out")+ if error_holder[0] is not None:+ raise error_holder[0]+ return result_holder[0]def disconnect(self):- with self._lock:- if self._client is not None:- try:- self._client.disconnect()- except Exception:- pass- self._client = None+ pass # No persistent connection to closedef _reset_on_error(self):- """Disconnect so next call reconnects."""- try:- self.disconnect()- except Exception:- pass+ pass # No persistent connection to reset# -- Screenshot ---------------------------------------------------------def capture_screenshot(self) -> bytes:"""Capture the screen and return PNG bytes."""- client = self._ensure_connected()- # refreshScreen updates the internal screen image- client.refreshScreen(incremental=False)- screen = client.protocol.screen- buf = BytesIO()- screen.save(buf, format="PNG")- return buf.getvalue()+ from twisted.internet import defer++ @defer.inlineCallbacks+ def _do(client):+ yield client.refreshScreen(incremental=False)+ screen = client.screen+ buf = BytesIO()+ screen.save(buf, format="PNG")+ defer.returnValue(buf.getvalue())++ return self._with_client(_do)# -- Mouse --------------------------------------------------------------def mouse_move(self, x: int, y: int):- client = self._ensure_connected()- client.mouseMove(x, y)+ self._with_client(lambda c: c.mouseMove(x, y))self._cursor_x = xself._cursor_y = ydef mouse_click(self, x: int, y: int, button: int = 1, clicks: int = 1):- client = self._ensure_connected()- client.mouseMove(x, y)+ from twisted.internet import defer++ @defer.inlineCallbacks+ def _do(client):+ yield client.mouseMove(x, y)+ for _ in range(clicks):+ yield client.mousePress(button)++ self._with_client(_do)self._cursor_x = xself._cursor_y = y- for _ in range(clicks):- client.mousePress(button)def mouse_down(self, x: int, y: int, button: int = 1):- client = self._ensure_connected()- client.mouseMove(x, y)+ from twisted.internet import defer++ @defer.inlineCallbacks+ def _do(client):+ yield client.mouseMove(x, y)+ yield client.mouseDown(button)++ self._with_client(_do)self._cursor_x = xself._cursor_y = y- client.mouseDown(button)def mouse_up(self, x: int, y: int, button: int = 1):- client = self._ensure_connected()- client.mouseMove(x, y)+ from twisted.internet import defer++ @defer.inlineCallbacks+ def _do(client):+ yield client.mouseMove(x, y)+ yield client.mouseUp(button)++ self._with_client(_do)self._cursor_x = xself._cursor_y = y- client.mouseUp(button)def mouse_drag(self, x: int, y: int, step: int = 1):- client = self._ensure_connected()- client.mouseDrag(x, y, step=step)+ from twisted.internet import defer++ sx, sy = self._cursor_x, self._cursor_y++ @defer.inlineCallbacks+ def _do(client):+ # Move in increments from current position to target+ dx, dy = x - sx, y - sy+ steps = max(abs(dx), abs(dy)) // max(step, 1)+ steps = max(steps, 1)+ yield client.mouseDown(1)+ for i in range(1, steps + 1):+ ix = sx + dx * i // steps+ iy = sy + dy * i // steps+ yield client.mouseMove(ix, iy)+ yield client.mouseUp(1)++ self._with_client(_do)self._cursor_x = xself._cursor_y = y+ def drag_to(+ self, start_x: int, start_y: int, end_x: int, end_y: int, button: int = 1, step: int = 5+ ):+ """Drag from start to end on a single connection."""+ from twisted.internet import defer++ @defer.inlineCallbacks+ def _do(client):+ yield client.mouseMove(start_x, start_y)+ yield client.mouseDown(button)+ # Manual drag in increments instead of mouseDrag (avoids doPoll)+ dx, dy = end_x - start_x, end_y - start_y+ steps = max(abs(dx), abs(dy)) // max(step, 1)+ steps = max(steps, 1)+ for i in range(1, steps + 1):+ ix = start_x + dx * i // steps+ iy = start_y + dy * i // steps+ yield client.mouseMove(ix, iy)+ yield client.mouseUp(button)++ self._with_client(_do)+ self._cursor_x = end_x+ self._cursor_y = end_y++ def drag_path(self, path: List[Tuple[int, int]], button: int = 1):+ """Drag along a path of points on a single connection."""+ from twisted.internet import defer++ @defer.inlineCallbacks+ def _do(client):+ yield client.mouseMove(path[0][0], path[0][1])+ yield client.mouseDown(button)+ for px, py in path[1:]:+ yield client.mouseMove(px, py)+ yield client.mouseUp(button)++ self._with_client(_do)+ self._cursor_x = path[-1][0]+ self._cursor_y = path[-1][1]+def scroll(self, x: int, y: int):- """Scroll. y>0 = down (button 5), y<0 = up (button 4)."""- client = self._ensure_connected()- if y > 0:- for _ in range(y):- client.mousePress(5)- elif y < 0:- for _ in range(abs(y)):- client.mousePress(4)- # Horizontal: button 6 (right), button 7 (left) — not universally supported- if x > 0:- for _ in range(x):- client.mousePress(6)- elif x < 0:- for _ in range(abs(x)):- client.mousePress(7)+ """Scroll. y>0 = up, y<0 = down (matches macOS native handler convention).++ Uses arrow key presses instead of mouse buttons 4/5 because Apple's+ _VZVNCServer (used by Lume) does not translate RFB mouse buttons 4-7+ into scroll wheel events.+ """+ from twisted.internet import defer++ @defer.inlineCallbacks+ def _do(client):+ if y > 0:+ for _ in range(y):+ yield client.keyPress("up")+ elif y < 0:+ for _ in range(abs(y)):+ yield client.keyPress("down")+ if x > 0:+ for _ in range(x):+ yield client.keyPress("right")+ elif x < 0:+ for _ in range(abs(x)):+ yield client.keyPress("left")++ self._with_client(_do)# -- Keyboard -----------------------------------------------------------def key_press(self, key: str):- client = self._ensure_connected()- client.keyPress(_translate_key(key))+ self._with_client(lambda c: c.keyPress(_translate_key(key)))def key_down(self, key: str):- client = self._ensure_connected()- client.keyDown(_translate_key(key))+ self._with_client(lambda c: c.keyDown(_translate_key(key)))def key_up(self, key: str):- client = self._ensure_connected()- client.keyUp(_translate_key(key))+ self._with_client(lambda c: c.keyUp(_translate_key(key)))def type_text(self, text: str):"""Type text character by character, handling shift for uppercase/symbols."""- client = self._ensure_connected()- for ch in text:- if ch == " ":- client.keyPress("space")- elif ch == "\n":- client.keyPress("enter")- elif ch == "\t":- client.keyPress("tab")- else:- # vncdotool's keyPress handles single characters directly- client.keyPress(ch)+ from twisted.internet import defer++ @defer.inlineCallbacks+ def _do(client):+ for ch in text:+ if ch == " ":+ yield client.keyPress("space")+ elif ch == "\n":+ yield client.keyPress("enter")+ elif ch == "\t":+ yield client.keyPress("tab")+ else:+ yield client.keyPress(ch)++ self._with_client(_do)def hotkey(self, keys: List[str]):"""Press a key combination (e.g. ['command', 'a'])."""- client = self._ensure_connected()translated = [_translate_key(k) for k in keys]- # vncdotool supports "modifier-key" syntax: "alt-a", "ctrl-shift-c"combo = "-".join(translated)- client.keyPress(combo)+ self._with_client(lambda c: c.keyPress(combo))# -- Clipboard ----------------------------------------------------------def paste_text(self, text: str):- client = self._ensure_connected()- client.paste(text)+ self._with_client(lambda c: c.paste(text))# -- Info ---------------------------------------------------------------@propertydef screen_size(self) -> Tuple[int, int]:- client = self._ensure_connected()- proto = client.protocol- if proto is not None and proto.screen is not None:- return proto.screen.size- return (0, 0)+ from twisted.internet import defer++ @defer.inlineCallbacks+ def _do(client):+ yield client.refreshScreen(incremental=False)+ if client.screen is not None:+ defer.returnValue(client.screen.size)+ defer.returnValue((0, 0))++ return self._with_client(_do)@propertydef cursor_position(self) -> Tuple[int, int]:@@ -365,14 +477,8 @@ class VNCAutomationHandler(BaseAutomationHandler):try:btn = _BUTTON_MAP.get(button, 1)cx, cy = self._conn.cursor_position-- def _drag():- self._conn.mouse_down(cx, cy, btn)- step = max(1, int(1 / max(duration, 0.01) * 5))- self._conn.mouse_drag(x, y, step=step)- self._conn.mouse_up(x, y, btn)-- await asyncio.to_thread(_drag)+ step = max(1, int(1 / max(duration, 0.01) * 5))+ await asyncio.to_thread(self._conn.drag_to, cx, cy, x, y, btn, step)return {"success": True}except Exception as e:return {"success": False, "error": str(e)}@@ -384,14 +490,7 @@ class VNCAutomationHandler(BaseAutomationHandler):if not path:return {"success": False, "error": "Empty path"}btn = _BUTTON_MAP.get(button, 1)-- def _drag_path():- self._conn.mouse_down(path[0][0], path[0][1], btn)- for px, py in path[1:]:- self._conn.mouse_move(px, py)- self._conn.mouse_up(path[-1][0], path[-1][1], btn)-- await asyncio.to_thread(_drag_path)+ await asyncio.to_thread(self._conn.drag_path, path, btn)return {"success": True}except Exception as e:return {"success": False, "error": str(e)}@@ -443,10 +542,10 @@ class VNCAutomationHandler(BaseAutomationHandler):return {"success": False, "error": str(e)}async def scroll_down(self, clicks: int = 1) -> Dict[str, Any]:- return await self.scroll(0, clicks)+ return await self.scroll(0, -clicks)async def scroll_up(self, clicks: int = 1) -> Dict[str, Any]:- return await self.scroll(0, -clicks)+ return await self.scroll(0, clicks)# -- Screen info --------------------------------------------------------