Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

feat(lume, computer-server): VNC backend rewrite and VirtioFS port discovery (#1205)

Francesco Bonaccif73a68be2ac3
diff --git a/docs/content/docs/lume/reference/cli-reference.mdx b/docs/content/docs/lume/reference/cli-reference.mdx
index a8dd1355b5..1a0e8475d6 100644
--- a/docs/content/docs/lume/reference/cli-reference.mdx
+++ b/docs/content/docs/lume/reference/cli-reference.mdx
@@ -7,16 +7,16 @@ description: Command Line Interface reference for Lume
AUTO-GENERATED FILE - DO NOT EDIT DIRECTLY
Generated by: npx tsx scripts/docs-generators/lume.ts
Source: libs/lume/src/Commands/*.swift
- Version: 0.2.86
+ Version: 0.3.0
*/}
import { Callout } from 'fumadocs-ui/components/callout';
import { VersionHeader } from '@/components/version-selector';
<VersionHeader
- versions={[{"version":"0.2","href":"/lume/reference/cli-reference","isCurrent":true}]}
- currentVersion="0.2"
- fullVersion="0.2.86"
+ versions={[{"version":"0.3","href":"/lume/reference/cli-reference","isCurrent":true},{"version":"0.2","href":"/lume/reference/v0.2/cli-reference","isCurrent":false}]}
+ currentVersion="0.3"
+ fullVersion="0.3.0"
packageName="lume"
installCommand="curl -fsSL https://raw.githubusercontent.com/trycua/cua/main/libs/lume/scripts/install.sh | bash"
/>
diff --git a/docs/content/docs/lume/reference/http-api.mdx b/docs/content/docs/lume/reference/http-api.mdx
index 8034a487cb..79f38961df 100644
--- a/docs/content/docs/lume/reference/http-api.mdx
+++ b/docs/content/docs/lume/reference/http-api.mdx
@@ -7,7 +7,7 @@ description: HTTP API reference for Lume server
AUTO-GENERATED FILE - DO NOT EDIT DIRECTLY
Generated by: npx tsx scripts/docs-generators/lume.ts
Source: libs/lume/src/Server/*.swift
- Version: 0.2.86
+ Version: 0.3.0
*/}
import { Callout } from 'fumadocs-ui/components/callout';
@@ -15,9 +15,9 @@ import { Tabs, Tab } from 'fumadocs-ui/components/tabs';
import { VersionHeader } from '@/components/version-selector';
<VersionHeader
- versions={[{"version":"0.2","href":"/lume/reference/cli-reference","isCurrent":true}]}
- currentVersion="0.2"
- fullVersion="0.2.86"
+ versions={[{"version":"0.3","href":"/lume/reference/cli-reference","isCurrent":true},{"version":"0.2","href":"/lume/reference/v0.2/cli-reference","isCurrent":false}]}
+ currentVersion="0.3"
+ fullVersion="0.3.0"
packageName="lume"
installCommand="curl -fsSL https://raw.githubusercontent.com/trycua/cua/main/libs/lume/scripts/install.sh | bash"
/>
diff --git a/libs/lume/scripts/setup-cua-computer.sh b/libs/lume/scripts/setup-cua.sh
similarity index 50%
rename from libs/lume/scripts/setup-cua-computer.sh
rename to libs/lume/scripts/setup-cua.sh
index c5d7f5aaa8..1e4b117b89 100755
--- a/libs/lume/scripts/setup-cua-computer.sh
+++ b/libs/lume/scripts/setup-cua.sh
@@ -1,21 +1,52 @@
#!/bin/bash
#
-# setup-cua-computer.sh
-# Makes a Lume macOS VM image cua-computer compatible
+# setup-cua.sh
+# Makes a Lume macOS VM fully cua-compatible
#
# This script runs ON the VM itself (not the host).
-# It installs cua-computer-server, cua-agent, Playwright, and configures
-# the VM for headless CUA operation (auto-login, disable sleep, LaunchAgent).
+# It installs cua-computer-server, cua-agent, Playwright, noVNC (web-based VNC
+# proxying to lume's built-in VNC on the host), supervisor, and configures the
+# VM for headless CUA operation (auto-login, disable sleep, LaunchAgent).
+#
+# In Lume VMs, the display is provided by the Virtualization Framework and
+# exposed through lume's built-in VNC server on the host. macOS Screen Sharing
+# inside the VM has no framebuffer to capture. Therefore, noVNC/websockify
+# inside the VM proxies to lume's VNC on the host (via the gateway IP).
#
# Usage:
-# lume ssh <vm-name> "curl -fsSL <url> | bash"
-# lume ssh <vm-name> < setup-cua-computer.sh
+# # Copy the script into the VM first, then run with a generous timeout:
+# lume ssh <vm-name> "cat > /tmp/setup.sh && chmod +x /tmp/setup.sh" < setup-cua.sh
+# lume ssh <vm-name> --timeout 600 "bash /tmp/setup.sh --yes"
+#
# # Or copy and run directly inside the VM:
-# ./setup-cua-computer.sh [--yes] [--port PORT]
+# ./setup-cua.sh [--yes] [--port PORT] [--host-vnc-port PORT]
+#
+# Important: Start the VM with a fixed VNC port and password so websockify can find it:
+# lume run <vm-name> --vnc-port 5901 --vnc-password lume
+#
+# Note: Port 5900 is often used by the host's Screen Sharing service, so we
+# default to 5901. Use --host-vnc-port to match whatever --vnc-port you pass
+# to lume run.
+#
+# Accessing noVNC remotely (from outside the host):
+# 1. SSH tunnel: ssh -L 6080:<vm-ip>:6080 user@host
+# 2. Open: http://localhost:6080/vnc.html?host=localhost&port=6080
+# The ?host=localhost&port=6080 params ensure the websocket connects through
+# the tunnel instead of trying to reach the VM IP directly.
+#
+# Note: The default lume ssh timeout is 60s, which is too short for this script
+# (Homebrew + Xcode CLT + pip installs can take 5-10 minutes). Use --timeout 600
+# or --timeout 0 (unlimited) when running via lume ssh.
#
# Prerequisites:
# - macOS VM created via lume (with SSH enabled, user lume/lume)
#
+# Host troubleshooting:
+# If VMs fail to start with "Unable to access security information", the host's
+# login keychain is likely locked (common after host reboot/crash on macOS Sequoia+).
+# Fix: security unlock-keychain -p '<host-password>' ~/Library/Keychains/login.keychain-db
+# Prevention: enable auto-login on the host so the keychain unlocks at boot.
+#
set -e
@@ -33,6 +64,9 @@ AUTO_YES=false
CUA_SERVER_PORT=8443
VM_PASSWORD="${VM_PASSWORD:-lume}"
CUA_DIR="$HOME/.cua-server"
+NOVNC_PORT=6080
+HOST_VNC_PORT="${HOST_VNC_PORT:-5901}"
+VNC_PASSWORD="${VNC_PASSWORD:-lume}"
# Parse arguments
while [ "$#" -gt 0 ]; do
@@ -44,13 +78,28 @@ while [ "$#" -gt 0 ]; do
CUA_SERVER_PORT="$2"
shift
;;
+ --host-vnc-port)
+ HOST_VNC_PORT="$2"
+ shift
+ ;;
+ --vnc-password)
+ VNC_PASSWORD="$2"
+ shift
+ ;;
--help)
echo "Usage: $0 [OPTIONS]"
echo ""
echo "Options:"
- echo " --yes, -y Non-interactive mode (accept all prompts)"
- echo " --port PORT CUA computer server port (default: 8443)"
- echo " --help Show this help message"
+ echo " --yes, -y Non-interactive mode (accept all prompts)"
+ echo " --port PORT CUA computer server port (default: 8443)"
+ echo " --host-vnc-port PORT Lume VNC port on the host (default: 5901)"
+ echo " --vnc-password PWD Lume VNC password (from 'lume ls' output)"
+ echo " --help Show this help message"
+ echo ""
+ echo "Environment variables:"
+ echo " VM_PASSWORD VM user password (default: lume)"
+ echo " HOST_VNC_PORT Lume VNC port on the host (default: 5901)"
+ echo " VNC_PASSWORD Lume VNC password (from 'lume ls' output)"
exit 0
;;
*)
@@ -64,6 +113,12 @@ done
CURRENT_USER="$(whoami)"
USER_HOME="$HOME"
+# Auto-detect host gateway IP
+GATEWAY_IP=$(route get default 2>/dev/null | grep gateway | awk '{print $2}')
+if [ -z "$GATEWAY_IP" ]; then
+ GATEWAY_IP="192.168.64.1"
+fi
+
# ============================================
# Helpers
# ============================================
@@ -174,7 +229,7 @@ install_system_deps() {
ensure_brew
local deps_needed=()
- for pkg in ffmpeg; do
+ for pkg in ffmpeg supervisor wget; do
if ! brew list "$pkg" &>/dev/null; then
deps_needed+=("$pkg")
fi
@@ -227,17 +282,94 @@ setup_cua_server() {
pip install playwright && playwright install firefox || \
echo -e "${YELLOW}Playwright install failed (non-blocking, BrowserTool may not work)${NC}"
+ # Install websockify for noVNC and vncdotool for VNC backend
+ echo "Installing websockify and vncdotool..."
+ pip install websockify vncdotool
+
deactivate
echo -e "${GREEN}CUA server and agent installed${NC}"
}
# ============================================
-# Phase 5: Startup Script + LaunchAgent
+# Phase 5: noVNC + Websockify
+# ============================================
+
+setup_novnc() {
+ print_phase "Phase 5: noVNC (Web-based VNC)"
+
+ if ! confirm "Install noVNC and websockify (web VNC on port $NOVNC_PORT)?"; then
+ echo -e "${YELLOW}Skipping noVNC setup.${NC}"
+ return 0
+ fi
+
+ ensure_brew
+
+ local NOVNC_DIR="/usr/local/share/novnc"
+
+ setup_sudo
+
+ # Download noVNC
+ echo "Installing noVNC..."
+ cd /tmp
+ rm -rf noVNC-master noVNC-master.zip
+ wget -q https://github.com/trycua/noVNC/archive/refs/heads/master.zip -O noVNC-master.zip
+ unzip -q noVNC-master.zip
+
+ if [ "$AUTO_YES" = true ]; then
+ sudo -A rm -rf "$NOVNC_DIR"
+ sudo -A mkdir -p "$NOVNC_DIR"
+ sudo -A mv noVNC-master/* "$NOVNC_DIR/"
+ sudo -A ln -sf "$NOVNC_DIR/vnc.html" "$NOVNC_DIR/index.html"
+ sudo -A chmod -R 755 "$NOVNC_DIR"
+ else
+ sudo rm -rf "$NOVNC_DIR"
+ sudo mkdir -p "$NOVNC_DIR"
+ sudo mv noVNC-master/* "$NOVNC_DIR/"
+ sudo ln -sf "$NOVNC_DIR/vnc.html" "$NOVNC_DIR/index.html"
+ sudo chmod -R 755 "$NOVNC_DIR"
+ fi
+
+ rm -rf noVNC-master noVNC-master.zip
+
+ # Configure supervisor for websockify
+ # Proxies to lume's built-in VNC on the host via the gateway IP.
+ # The host VNC port must match the --vnc-port used with lume run.
+ local SUPERVISOR_CONF_DIR="/opt/homebrew/etc/supervisor.d"
+ local SUPERVISOR_LOG_DIR="/opt/homebrew/var/log/supervisor"
+ local VENV_PYTHON="$CUA_DIR/venv/bin/python3"
+
+ mkdir -p "$SUPERVISOR_CONF_DIR"
+ mkdir -p "$SUPERVISOR_LOG_DIR"
+
+ cat > "$SUPERVISOR_CONF_DIR/websockify.ini" <<EOF
+[program:websockify]
+command=$VENV_PYTHON -m websockify --web $NOVNC_DIR $NOVNC_PORT $GATEWAY_IP:$HOST_VNC_PORT
+directory=$NOVNC_DIR
+autostart=true
+autorestart=true
+startsecs=5
+startretries=3
+stopwaitsecs=10
+redirect_stderr=true
+stdout_logfile=$SUPERVISOR_LOG_DIR/websockify.log
+stdout_logfile_maxbytes=10MB
+stdout_logfile_backups=5
+environment=HOME="$USER_HOME",PATH="/opt/homebrew/bin:/usr/local/bin:/usr/bin:/bin",DISPLAY=":0"
+EOF
+
+ chmod 644 "$SUPERVISOR_CONF_DIR/websockify.ini"
+
+ echo -e "${GREEN}noVNC installed${NC}"
+ echo " websockify will proxy port $NOVNC_PORT -> $GATEWAY_IP:$HOST_VNC_PORT (lume VNC)"
+}
+
+# ============================================
+# Phase 6: Startup Script + LaunchAgent
# ============================================
create_startup_script() {
- print_phase "Phase 5: Startup Script & LaunchAgent"
+ print_phase "Phase 6: Startup Script & LaunchAgent"
if ! confirm "Create CUA server startup script and LaunchAgent?"; then
echo -e "${YELLOW}Skipping LaunchAgent setup.${NC}"
@@ -260,6 +392,48 @@ source venv/bin/activate
export DISPLAY=:0
+# Auto-detect host gateway IP for VNC backend
+GATEWAY_IP=$(route get default 2>/dev/null | grep gateway | awk '{print $2}')
+if [ -z "$GATEWAY_IP" ]; then
+ GATEWAY_IP="192.168.64.1"
+fi
+
+# Read VNC config from lume shared directory if available.
+# Lume automatically shares a "lume-config" VirtioFS volume containing vnc.env
+# with the assigned VNC port and password for this VM.
+LUME_CONFIG_MOUNT="/tmp/lume-config"
+LUME_CONFIG="$LUME_CONFIG_MOUNT/vnc.env"
+LUME_VNC_PORT=""
+LUME_VNC_PASSWORD=""
+
+# Try to mount the lume-config VirtioFS share and read VNC config
+mkdir -p "$LUME_CONFIG_MOUNT" 2>/dev/null
+if mount | grep -q "lume-config"; then
+ echo "lume-config already mounted" >> "$LOG_FILE"
+elif mount_virtiofs lume-config "$LUME_CONFIG_MOUNT" 2>/dev/null; then
+ echo "Mounted lume-config VirtioFS share" >> "$LOG_FILE"
+else
+ echo "lume-config VirtioFS share not available, using defaults" >> "$LOG_FILE"
+fi
+
+# Wait up to 10s for vnc.env to appear (written by host after VNC starts)
+for i in $(seq 1 10); do
+ if [ -f "$LUME_CONFIG" ]; then
+ eval "$(cat "$LUME_CONFIG")"
+ LUME_VNC_PORT="${VNC_PORT:-}"
+ LUME_VNC_PASSWORD="${VNC_PASSWORD:-}"
+ echo "Read VNC config from lume shared dir: port=$LUME_VNC_PORT" >> "$LOG_FILE"
+ break
+ fi
+ sleep 1
+done
+
+# Set VNC backend env vars — prefer lume-config values, fall back to defaults
+export CUA_BACKEND=vnc
+export CUA_VNC_HOST="$GATEWAY_IP"
+export CUA_VNC_PORT="${LUME_VNC_PORT:-__VNC_PORT__}"
+export CUA_VNC_PASSWORD="${LUME_VNC_PASSWORD:-__VNC_PASSWORD__}"
+
# Update packages
echo "Updating cua-agent..." >> "$LOG_FILE"
pip install --upgrade --no-input "cua-agent[all]" >> "$LOG_FILE" 2>&1 || true
@@ -272,13 +446,16 @@ echo "Ensuring Playwright Firefox..." >> "$LOG_FILE"
pip install --upgrade --no-input playwright >> "$LOG_FILE" 2>&1 || true
playwright install firefox >> "$LOG_FILE" 2>&1 || true
-# Start server
-echo "Starting CUA computer server on port __PORT__..." >> "$LOG_FILE"
+# Start server with VNC backend pointing at lume's VNC on the host
+echo "Starting CUA computer server (vnc backend) on port __PORT__..." >> "$LOG_FILE"
+echo " VNC target: $GATEWAY_IP:__VNC_PORT__ (password: __VNC_PASSWORD__)" >> "$LOG_FILE"
python -m computer_server --port __PORT__ >> "$LOG_FILE" 2>&1
SCRIPT_EOF
- # Stamp the port
+ # Stamp the port and VNC settings
sed -i '' "s/__PORT__/$CUA_SERVER_PORT/g" "$STARTUP_SCRIPT"
+ sed -i '' "s/__VNC_PORT__/$HOST_VNC_PORT/g" "$STARTUP_SCRIPT"
+ sed -i '' "s/__VNC_PASSWORD__/$VNC_PASSWORD/g" "$STARTUP_SCRIPT"
chmod +x "$STARTUP_SCRIPT"
# Create LaunchAgent
@@ -309,6 +486,12 @@ SCRIPT_EOF
<string>$USER_HOME</string>
<key>DISPLAY</key>
<string>:0</string>
+ <key>CUA_BACKEND</key>
+ <string>vnc</string>
+ <key>CUA_VNC_PORT</key>
+ <string>$HOST_VNC_PORT</string>
+ <key>CUA_VNC_PASSWORD</key>
+ <string>$VNC_PASSWORD</string>
</dict>
<key>ProgramArguments</key>
@@ -346,13 +529,13 @@ EOF
}
# ============================================
-# Phase 6: System Configuration
+# Phase 7: System Configuration + Auto-Login
# ============================================
configure_system() {
- print_phase "Phase 6: System Configuration"
+ print_phase "Phase 7: System Configuration"
- if ! confirm "Disable screen lock, screensaver, and sleep?"; then
+ if ! confirm "Disable screen lock, screensaver, sleep, and enable auto-login?"; then
echo -e "${YELLOW}Skipping system configuration.${NC}"
return 0
fi
@@ -374,11 +557,67 @@ configure_system() {
# Enable full keyboard navigation (needed for accessibility automation)
defaults write NSGlobalDomain AppleKeyboardUIMode -int 3
- echo -e "${GREEN}System configured (no sleep, no screensaver, full keyboard nav)${NC}"
+ # Configure auto-login (ensures GUI session + unlocked keychain on boot)
+ echo "Configuring auto-login for $CURRENT_USER..."
+
+ if [ "$AUTO_YES" = true ]; then
+ sudo -A defaults write /Library/Preferences/com.apple.loginwindow autoLoginUser "$CURRENT_USER"
+ else
+ sudo defaults write /Library/Preferences/com.apple.loginwindow autoLoginUser "$CURRENT_USER"
+ fi
+
+ # Create kcpassword file (XOR-encrypted password for auto-login)
+ python3 << PYEOF
+password = "$VM_PASSWORD"
+key = bytes.fromhex("7d895223d2bcddeaa3b91f")
+padded_len = ((len(password) + 11) // 12) * 12
+padded = password + "\x00" * (padded_len - len(password))
+result = bytearray()
+for i, char in enumerate(padded.encode("utf-8")):
+ result.append(char ^ key[i % len(key)])
+with open("/tmp/kcpassword", "wb") as f:
+ f.write(result)
+PYEOF
+
+ if [ "$AUTO_YES" = true ]; then
+ sudo -A mv /tmp/kcpassword /etc/kcpassword
+ sudo -A chown root:wheel /etc/kcpassword
+ sudo -A chmod 600 /etc/kcpassword
+ else
+ sudo mv /tmp/kcpassword /etc/kcpassword
+ sudo chown root:wheel /etc/kcpassword
+ sudo chmod 600 /etc/kcpassword
+ fi
+
+ echo -e "${GREEN}System configured (no sleep, no screensaver, full keyboard nav, auto-login)${NC}"
}
# ============================================
-# Phase 7: Verification
+# Phase 8: Start Services
+# ============================================
+
+start_services() {
+ print_phase "Phase 8: Start Services"
+
+ ensure_brew
+
+ # Start supervisor (manages websockify)
+ if command -v brew &>/dev/null; then
+ echo "Starting supervisor..."
+ brew services start supervisor 2>/dev/null || true
+ sleep 3
+
+ if command -v supervisorctl &>/dev/null; then
+ supervisorctl reread 2>/dev/null || true
+ supervisorctl update 2>/dev/null || true
+ fi
+ fi
+
+ echo -e "${GREEN}Services started${NC}"
+}
+
+# ============================================
+# Phase 9: Verification
# ============================================
verify() {
@@ -386,6 +625,7 @@ verify() {
echo "System:"
echo " User: $CURRENT_USER"
+ echo " Gateway: $GATEWAY_IP"
sw_vers | sed 's/^/ /'
echo ""
@@ -431,6 +671,22 @@ verify() {
echo -e " ${YELLOW}~${NC} Playwright (may not be installed)"
fi
+ # noVNC / websockify
+ if command -v supervisorctl &>/dev/null && supervisorctl status websockify 2>/dev/null | grep -q RUNNING; then
+ echo -e " ${GREEN}✓${NC} noVNC + websockify (port $NOVNC_PORT -> $GATEWAY_IP:$HOST_VNC_PORT)"
+ elif [ -d "/usr/local/share/novnc" ]; then
+ echo -e " ${YELLOW}~${NC} noVNC installed (websockify may not be running yet)"
+ else
+ echo -e " ${RED}✗${NC} noVNC"
+ fi
+
+ # Auto-login
+ if defaults read /Library/Preferences/com.apple.loginwindow autoLoginUser 2>/dev/null | grep -q "$CURRENT_USER"; then
+ echo -e " ${GREEN}✓${NC} Auto-login ($CURRENT_USER)"
+ else
+ echo -e " ${YELLOW}~${NC} Auto-login (may need reboot)"
+ fi
+
# LaunchAgent
local PLIST="$USER_HOME/Library/LaunchAgents/com.trycua.computer_server.plist"
if [ -f "$PLIST" ]; then
@@ -444,13 +700,29 @@ verify() {
echo -e "${GREEN} Setup Complete!${NC}"
echo -e "${GREEN}========================================${NC}"
echo ""
- echo "CUA computer server will start on port $CUA_SERVER_PORT at next login."
+ echo "Services:"
+ echo " CUA server: port $CUA_SERVER_PORT (starts at login via LaunchAgent)"
+ echo " noVNC: port $NOVNC_PORT -> $GATEWAY_IP:$HOST_VNC_PORT (lume VNC)"
+ echo ""
+ echo "Start the VM with a fixed VNC port and password (from the host):"
+ echo " lume run <vm-name> --vnc-port $HOST_VNC_PORT --vnc-password $VNC_PASSWORD"
echo ""
- echo "To start it now (from the host):"
+ echo "To start CUA server now (from the host):"
echo " lume ssh <vm-name> 'launchctl load ~/Library/LaunchAgents/com.trycua.computer_server.plist'"
echo ""
echo "To test from the host:"
- echo " curl http://<vm-ip>:$CUA_SERVER_PORT/health"
+ echo " curl http://<vm-ip>:$CUA_SERVER_PORT/status"
+ echo ""
+ echo "To access noVNC from the host network:"
+ echo " http://<vm-ip>:$NOVNC_PORT/vnc.html"
+ echo ""
+ echo "To access noVNC remotely (from outside the host):"
+ echo " 1. SSH tunnel: ssh -L $NOVNC_PORT:<vm-ip>:$NOVNC_PORT user@host"
+ echo " 2. Open: http://localhost:$NOVNC_PORT/vnc.html?host=localhost&port=$NOVNC_PORT"
+ echo ""
+ echo "VNC password: $VNC_PASSWORD"
+ echo ""
+ echo "NOTE: A reboot is recommended for auto-login to take effect."
echo ""
}
@@ -463,14 +735,16 @@ echo " ⠀⣀⣀⡀⠀⠀⠀⠀⢀⣀⣀⣀⡀⠘⠋⢉⠙⣷⠀⠀ ⠀"
echo " ⠀⠀⢀⣴⣿⡿⠋⣉⠁⣠⣾⣿⣿⣿⣿⡿⠿⣦⡈⠀⣿⡇⠃⠀"
echo " ⠀⠀⠀⣽⣿⣧⠀⠃⢰⣿⣿⡏⠙⣿⠿⢧⣀⣼⣷⠀⡿⠃⠀⠀"
echo " ⠀⠀⠀⠉⣿⣿⣦⠀⢿⣿⣿⣷⣾⡏⠀⠀⢹⣿⣿⠀⠀⠀⠀⠀⠀"
-echo -e " ⠀⠀⠀⠀⠀⠉⠛⠁⠈⠿⣿⣿⣿⣷⣄⣠⡼⠟⠁${NC} ${BLUE}CUA Computer Setup${NC}"
-echo -e "${BLUE} Make a Lume VM cua-computer compatible${NC}"
+echo -e " ⠀⠀⠀⠀⠀⠉⠛⠁⠈⠿⣿⣿⣿⣷⣄⣠⡼⠟⠁${NC} ${BLUE}CUA Setup${NC}"
+echo -e "${BLUE} Make a Lume VM cua-compatible${NC}"
echo ""
install_homebrew
install_python
install_system_deps
setup_cua_server
+setup_novnc
create_startup_script
configure_system
+start_services
verify
diff --git a/libs/lume/src/Commands/Run.swift b/libs/lume/src/Commands/Run.swift
index b7188ec9ae..835634d358 100644
--- a/libs/lume/src/Commands/Run.swift
+++ b/libs/lume/src/Commands/Run.swift
@@ -45,6 +45,11 @@ struct Run: AsyncParsableCommand {
help: "Port to use for the VNC server. Defaults to 0 (auto-assign)")
var vncPort: Int = 0
+ @Option(
+ name: [.customLong("vnc-password")],
+ help: "Password for the VNC server. Defaults to a random passphrase")
+ var vncPassword: String?
+
@Option(help: "For MacOS VMs only, boot into the VM in recovery mode")
var recoveryMode: Bool = false
@@ -133,6 +138,7 @@ struct Run: AsyncParsableCommand {
registry: registry,
organization: organization,
vncPort: vncPort,
+ vncPassword: vncPassword,
recoveryMode: recoveryMode,
storage: storage,
usbMassStoragePaths: parsedUSBStorageDevices.isEmpty ? nil : parsedUSBStorageDevices,
diff --git a/libs/lume/src/LumeController.swift b/libs/lume/src/LumeController.swift
index 5105cd44f4..349824016f 100644
--- a/libs/lume/src/LumeController.swift
+++ b/libs/lume/src/LumeController.swift
@@ -904,6 +904,7 @@ final class LumeController {
registry: String = "ghcr.io",
organization: String = "trycua",
vncPort: Int = 0,
+ vncPassword: String? = nil,
recoveryMode: Bool = false,
storage: String? = nil,
usbMassStoragePaths: [Path]? = nil,
@@ -1004,6 +1005,7 @@ final class LumeController {
sharedDirectories: sharedDirectories,
mount: mount,
vncPort: vncPort,
+ vncPassword: vncPassword,
recoveryMode: recoveryMode,
usbMassStoragePaths: usbMassStoragePaths,
networkMode: networkMode,
diff --git a/libs/lume/src/VM/VM.swift b/libs/lume/src/VM/VM.swift
index 8e2d2b9c2d..4da637d85c 100644
--- a/libs/lume/src/VM/VM.swift
+++ b/libs/lume/src/VM/VM.swift
@@ -136,7 +136,7 @@ class VM {
func run(
noDisplay: Bool, sharedDirectories: [SharedDirectory], mount: Path?, vncPort: Int = 0,
- recoveryMode: Bool = false, usbMassStoragePaths: [Path]? = nil,
+ vncPassword: String? = nil, recoveryMode: Bool = false, usbMassStoragePaths: [Path]? = nil,
networkMode: NetworkMode? = nil, clipboard: Bool = false
) async throws {
Logger.info(
@@ -215,13 +215,29 @@ class VM {
// Create and configure the VM
do {
+ // Create a lume-config shared directory so the guest can discover
+ // the VNC port/password at boot. The directory is created empty now
+ // and populated after the VNC server starts (VirtioFS exposes live
+ // host directory contents, so the guest will see the file once written).
+ let lumeConfigDir = FileManager.default.temporaryDirectory
+ .appendingPathComponent("lume-config-\(vmDirContext.name)")
+ try? FileManager.default.createDirectory(at: lumeConfigDir, withIntermediateDirectories: true)
+ // Remove stale vnc.env from a previous run so the guest doesn't
+ // read outdated port/password before the new file is written.
+ try? FileManager.default.removeItem(
+ at: lumeConfigDir.appendingPathComponent("vnc.env"))
+ let lumeConfigSharedDir = SharedDirectory(
+ hostPath: lumeConfigDir.path, tag: "lume-config", readOnly: true)
+ var allSharedDirectories = sharedDirectories
+ allSharedDirectories.append(lumeConfigSharedDir)
+
Logger.info(
"Creating virtualization service context", metadata: ["name": vmDirContext.name])
let config = try createVMVirtualizationServiceContext(
cpuCount: cpuCount,
memorySize: memorySize,
display: vmDirContext.config.display.string,
- sharedDirectories: sharedDirectories,
+ sharedDirectories: allSharedDirectories,
mount: mount,
recoveryMode: recoveryMode,
usbMassStoragePaths: usbMassStoragePaths,
@@ -246,7 +262,21 @@ class VM {
"port": "\(vncPort)",
])
let vncInfo = try await setupSession(
- port: vncPort, sharedDirectories: sharedDirectories)
+ port: vncPort, password: vncPassword, sharedDirectories: sharedDirectories)
+
+ // Write VNC connection info to the shared config directory so the
+ // guest can read it via mount_virtiofs.
+ // URL format: vnc://:password@host:port — URLComponents needs http:// to parse correctly.
+ if let components = URLComponents(string: vncInfo.replacingOccurrences(of: "vnc://", with: "http://")),
+ let port = components.port {
+ let password = components.password ?? ""
+ let envContent = "VNC_PORT=\(port)\nVNC_PASSWORD=\(password)\n"
+ try? envContent.write(
+ to: lumeConfigDir.appendingPathComponent("vnc.env"),
+ atomically: true, encoding: .utf8)
+ Logger.info("Wrote VNC config to shared directory", metadata: [
+ "port": "\(port)", "path": lumeConfigDir.path])
+ }
Logger.info(
"VNC setup successful", metadata: ["name": vmDirContext.name, "vncInfo": vncInfo])
@@ -658,12 +688,12 @@ class VM {
}
/// Sets up the VNC service and returns the VNC URL
- private func startVNCService(port: Int = 0) async throws -> String {
+ private func startVNCService(port: Int = 0, password: String? = nil) async throws -> String {
guard let service = virtualizationService else {
throw VMError.internalError("Virtualization service not initialized")
}
- try await vncService.start(port: port, virtualMachine: service.getVirtualMachine())
+ try await vncService.start(port: port, password: password, virtualMachine: service.getVirtualMachine())
guard let url = vncService.url else {
throw VMError.vncNotConfigured
@@ -692,10 +722,10 @@ class VM {
/// Main session setup method that handles VNC and persists session data
private func setupSession(
- port: Int = 0, sharedDirectories: [SharedDirectory] = []
+ port: Int = 0, password: String? = nil, sharedDirectories: [SharedDirectory] = []
) async throws -> String {
// Start the VNC service and get the URL
- let url = try await startVNCService(port: port)
+ let url = try await startVNCService(port: port, password: password)
// Save the session data
saveSessionData(url: url, sharedDirectories: sharedDirectories)
@@ -901,11 +931,22 @@ class VM {
// Create and configure the VM
do {
+ // Create lume-config shared directory for VNC discovery
+ let lumeConfigDir = FileManager.default.temporaryDirectory
+ .appendingPathComponent("lume-config-\(vmDirContext.name)")
+ try? FileManager.default.createDirectory(at: lumeConfigDir, withIntermediateDirectories: true)
+ try? FileManager.default.removeItem(
+ at: lumeConfigDir.appendingPathComponent("vnc.env"))
+ let lumeConfigSharedDir = SharedDirectory(
+ hostPath: lumeConfigDir.path, tag: "lume-config", readOnly: true)
+ var allSharedDirectories = sharedDirectories
+ allSharedDirectories.append(lumeConfigSharedDir)
+
let config = try createVMVirtualizationServiceContext(
cpuCount: cpuCount,
memorySize: memorySize,
display: vmDirContext.config.display.string,
- sharedDirectories: sharedDirectories,
+ sharedDirectories: allSharedDirectories,
mount: mount,
recoveryMode: recoveryMode,
usbMassStoragePaths: usbImagePaths
@@ -916,6 +957,16 @@ class VM {
port: vncPort, sharedDirectories: sharedDirectories)
Logger.info("VNC info", metadata: ["vncInfo": vncInfo])
+ // Write VNC config to shared directory for guest discovery
+ if let components = URLComponents(string: vncInfo.replacingOccurrences(of: "vnc://", with: "http://")),
+ let port = components.port {
+ let password = components.password ?? ""
+ let envContent = "VNC_PORT=\(port)\nVNC_PASSWORD=\(password)\n"
+ try? envContent.write(
+ to: lumeConfigDir.appendingPathComponent("vnc.env"),
+ atomically: true, encoding: .utf8)
+ }
+
// Start the VM
guard let service = virtualizationService else {
throw VMError.internalError("Virtualization service not initialized")
diff --git a/libs/lume/src/VNC/VNCService.swift b/libs/lume/src/VNC/VNCService.swift
index ed41c3b684..67fdbb204c 100644
--- a/libs/lume/src/VNC/VNCService.swift
+++ b/libs/lume/src/VNC/VNCService.swift
@@ -8,7 +8,7 @@ import Network
@MainActor
protocol VNCService {
var url: String? { get }
- func start(port: Int, virtualMachine: Any?) async throws
+ func start(port: Int, password: String?, virtualMachine: Any?) async throws
func stop()
func openClient(url: String) async throws
@@ -69,8 +69,8 @@ final class DefaultVNCService: VNCService {
}
}
- func start(port: Int, virtualMachine: Any?) async throws {
- let password = Array(PassphraseGenerator().prefix(4)).joined(separator: "-")
+ func start(port: Int, password: String? = nil, virtualMachine: Any?) async throws {
+ let password = password ?? Array(PassphraseGenerator().prefix(4)).joined(separator: "-")
let securityConfiguration = Dynamic._VZVNCAuthenticationSecurityConfiguration(password: password)
// Store password for later VNC client connection
diff --git a/libs/lume/tests/Mocks/MockVM.swift b/libs/lume/tests/Mocks/MockVM.swift
index 1e24d08da9..75a68ddf3c 100644
--- a/libs/lume/tests/Mocks/MockVM.swift
+++ b/libs/lume/tests/Mocks/MockVM.swift
@@ -23,13 +23,13 @@ class MockVM: VM {
override func run(
noDisplay: Bool, sharedDirectories: [SharedDirectory], mount: Path?, vncPort: Int = 0,
- recoveryMode: Bool = false, usbMassStoragePaths: [Path]? = nil,
+ vncPassword: String? = nil, recoveryMode: Bool = false, usbMassStoragePaths: [Path]? = nil,
networkMode: NetworkMode? = nil, clipboard: Bool = false
) async throws {
mockIsRunning = true
try await super.run(
noDisplay: noDisplay, sharedDirectories: sharedDirectories, mount: mount,
- vncPort: vncPort, recoveryMode: recoveryMode,
+ vncPort: vncPort, vncPassword: vncPassword, recoveryMode: recoveryMode,
usbMassStoragePaths: usbMassStoragePaths,
networkMode: networkMode, clipboard: clipboard
)
diff --git a/libs/lume/tests/Mocks/MockVNCService.swift b/libs/lume/tests/Mocks/MockVNCService.swift
index 215053d0f6..1bc8f16d8d 100644
--- a/libs/lume/tests/Mocks/MockVNCService.swift
+++ b/libs/lume/tests/Mocks/MockVNCService.swift
@@ -29,7 +29,7 @@ final class MockVNCService: VNCService {
return nil
}
- func start(port: Int, virtualMachine: Any?) async throws {
+ func start(port: Int, password: String? = nil, virtualMachine: Any?) async throws {
isRunning = true
url = "vnc://localhost:\(port)"
_attachedVM = virtualMachine
diff --git a/libs/python/computer-server/computer_server/cli.py b/libs/python/computer-server/computer_server/cli.py
index 3ea0c76f0d..ca3ef52551 100644
--- a/libs/python/computer-server/computer_server/cli.py
+++ b/libs/python/computer-server/computer_server/cli.py
@@ -120,7 +120,9 @@ def main() -> None:
os.environ["CUA_BACKEND"] = "vnc"
if args.vnc_host:
os.environ["CUA_VNC_HOST"] = args.vnc_host
- os.environ["CUA_VNC_PORT"] = str(args.vnc_port)
+ # Only override env vars from CLI args when explicitly provided
+ if args.vnc_port != 5900 or "CUA_VNC_PORT" not in os.environ:
+ os.environ["CUA_VNC_PORT"] = str(args.vnc_port)
if args.vnc_password:
os.environ["CUA_VNC_PASSWORD"] = args.vnc_password
vnc_host = args.vnc_host or os.environ.get("CUA_VNC_HOST")
diff --git a/libs/python/computer-server/computer_server/handlers/vnc.py b/libs/python/computer-server/computer_server/handlers/vnc.py
index c691e01ab2..7285907b82 100644
--- a/libs/python/computer-server/computer_server/handlers/vnc.py
+++ b/libs/python/computer-server/computer_server/handlers/vnc.py
@@ -21,7 +21,6 @@ Usage:
import asyncio
import base64
import logging
-import threading
from io import BytesIO
from typing import Any, Dict, List, Optional, Tuple
@@ -81,7 +80,12 @@ def _translate_key(key: str) -> str:
class _VNCConnection:
- """Manages a vncdotool ThreadedVNCClientProxy lifecycle.
+ """Manages vncdotool client connections to a VNC server.
+
+ Uses fresh connections per operation to avoid a bug in vncdotool's
+ ThreadedVNCClientProxy where the Twisted deferred chain corrupts
+ coordinate state after operations like refreshScreen. Each public
+ method creates its own connection, executes, and disconnects.
All methods are synchronous (blocking) — the handler calls them via
asyncio.to_thread to avoid blocking the event loop.
@@ -91,164 +95,272 @@ class _VNCConnection:
self._host = host
self._port = port
self._password = password
- self._client = None
- self._lock = threading.Lock()
- # Track cursor position locally (vncdotool also tracks via client.x/y)
+ # Track cursor position locally
self._cursor_x: int = 0
self._cursor_y: int = 0
- def _ensure_connected(self):
- """Lazily connect to the VNC server. Returns the vncdotool client."""
- if self._client is not None:
- return self._client
- with self._lock:
- if self._client is not None:
- return self._client
- from vncdotool import api
-
- server_str = f"{self._host}::{self._port}"
- logger.info(f"VNC connecting to {self._host}:{self._port}")
- client = api.connect(
- server_str,
- password=self._password or None,
+ def _with_client(self, fn):
+ """Execute fn(client) with a fresh VNC connection via the global Twisted reactor.
+
+ Starts the reactor in a background thread on first use, then reuses it.
+ ``fn(client)`` receives a raw ``VNCDoToolClient`` whose methods return
+ Twisted Deferreds.
+ """
+ import threading
+
+ from twisted.internet import defer, reactor
+ from vncdotool.client import VNCDoToolFactory
+
+ result_holder: list = [None]
+ error_holder: list = [None]
+ done_event = threading.Event()
+
+ def _work():
+ factory = VNCDoToolFactory()
+ factory.password = self._password or None
+
+ @defer.inlineCallbacks
+ def _do():
+ client = None
+ try:
+ reactor.connectTCP(self._host, self._port, factory)
+ client = yield factory.deferred
+ res = yield defer.maybeDeferred(fn, client)
+ result_holder[0] = res
+ except Exception as e:
+ error_holder[0] = e
+ finally:
+ if client is not None and hasattr(client, "transport") and client.transport:
+ client.transport.loseConnection()
+ done_event.set()
+
+ _do()
+
+ # Ensure the reactor is running in a background thread
+ if not reactor.running:
+ t = threading.Thread(
+ target=reactor.run,
+ kwargs={"installSignalHandlers": False},
+ daemon=True,
)
- client.timeout = 30
- self._client = client
- logger.info("VNC connected")
- return client
+ t.start()
+
+ reactor.callFromThread(_work)
+ done_event.wait(timeout=30)
+ if not done_event.is_set():
+ raise TimeoutError("VNC operation timed out")
+ if error_holder[0] is not None:
+ raise error_holder[0]
+ return result_holder[0]
def disconnect(self):
- with self._lock:
- if self._client is not None:
- try:
- self._client.disconnect()
- except Exception:
- pass
- self._client = None
+ pass # No persistent connection to close
def _reset_on_error(self):
- """Disconnect so next call reconnects."""
- try:
- self.disconnect()
- except Exception:
- pass
+ pass # No persistent connection to reset
# -- Screenshot ---------------------------------------------------------
def capture_screenshot(self) -> bytes:
"""Capture the screen and return PNG bytes."""
- client = self._ensure_connected()
- # refreshScreen updates the internal screen image
- client.refreshScreen(incremental=False)
- screen = client.protocol.screen
- buf = BytesIO()
- screen.save(buf, format="PNG")
- return buf.getvalue()
+ from twisted.internet import defer
+
+ @defer.inlineCallbacks
+ def _do(client):
+ yield client.refreshScreen(incremental=False)
+ screen = client.screen
+ buf = BytesIO()
+ screen.save(buf, format="PNG")
+ defer.returnValue(buf.getvalue())
+
+ return self._with_client(_do)
# -- Mouse --------------------------------------------------------------
def mouse_move(self, x: int, y: int):
- client = self._ensure_connected()
- client.mouseMove(x, y)
+ self._with_client(lambda c: c.mouseMove(x, y))
self._cursor_x = x
self._cursor_y = y
def mouse_click(self, x: int, y: int, button: int = 1, clicks: int = 1):
- client = self._ensure_connected()
- client.mouseMove(x, y)
+ from twisted.internet import defer
+
+ @defer.inlineCallbacks
+ def _do(client):
+ yield client.mouseMove(x, y)
+ for _ in range(clicks):
+ yield client.mousePress(button)
+
+ self._with_client(_do)
self._cursor_x = x
self._cursor_y = y
- for _ in range(clicks):
- client.mousePress(button)
def mouse_down(self, x: int, y: int, button: int = 1):
- client = self._ensure_connected()
- client.mouseMove(x, y)
+ from twisted.internet import defer
+
+ @defer.inlineCallbacks
+ def _do(client):
+ yield client.mouseMove(x, y)
+ yield client.mouseDown(button)
+
+ self._with_client(_do)
self._cursor_x = x
self._cursor_y = y
- client.mouseDown(button)
def mouse_up(self, x: int, y: int, button: int = 1):
- client = self._ensure_connected()
- client.mouseMove(x, y)
+ from twisted.internet import defer
+
+ @defer.inlineCallbacks
+ def _do(client):
+ yield client.mouseMove(x, y)
+ yield client.mouseUp(button)
+
+ self._with_client(_do)
self._cursor_x = x
self._cursor_y = y
- client.mouseUp(button)
def mouse_drag(self, x: int, y: int, step: int = 1):
- client = self._ensure_connected()
- client.mouseDrag(x, y, step=step)
+ from twisted.internet import defer
+
+ sx, sy = self._cursor_x, self._cursor_y
+
+ @defer.inlineCallbacks
+ def _do(client):
+ # Move in increments from current position to target
+ dx, dy = x - sx, y - sy
+ steps = max(abs(dx), abs(dy)) // max(step, 1)
+ steps = max(steps, 1)
+ yield client.mouseDown(1)
+ for i in range(1, steps + 1):
+ ix = sx + dx * i // steps
+ iy = sy + dy * i // steps
+ yield client.mouseMove(ix, iy)
+ yield client.mouseUp(1)
+
+ self._with_client(_do)
self._cursor_x = x
self._cursor_y = y
+ def drag_to(
+ self, start_x: int, start_y: int, end_x: int, end_y: int, button: int = 1, step: int = 5
+ ):
+ """Drag from start to end on a single connection."""
+ from twisted.internet import defer
+
+ @defer.inlineCallbacks
+ def _do(client):
+ yield client.mouseMove(start_x, start_y)
+ yield client.mouseDown(button)
+ # Manual drag in increments instead of mouseDrag (avoids doPoll)
+ dx, dy = end_x - start_x, end_y - start_y
+ steps = max(abs(dx), abs(dy)) // max(step, 1)
+ steps = max(steps, 1)
+ for i in range(1, steps + 1):
+ ix = start_x + dx * i // steps
+ iy = start_y + dy * i // steps
+ yield client.mouseMove(ix, iy)
+ yield client.mouseUp(button)
+
+ self._with_client(_do)
+ self._cursor_x = end_x
+ self._cursor_y = end_y
+
+ def drag_path(self, path: List[Tuple[int, int]], button: int = 1):
+ """Drag along a path of points on a single connection."""
+ from twisted.internet import defer
+
+ @defer.inlineCallbacks
+ def _do(client):
+ yield client.mouseMove(path[0][0], path[0][1])
+ yield client.mouseDown(button)
+ for px, py in path[1:]:
+ yield client.mouseMove(px, py)
+ yield client.mouseUp(button)
+
+ self._with_client(_do)
+ self._cursor_x = path[-1][0]
+ self._cursor_y = path[-1][1]
+
def scroll(self, x: int, y: int):
- """Scroll. y>0 = down (button 5), y<0 = up (button 4)."""
- client = self._ensure_connected()
- if y > 0:
- for _ in range(y):
- client.mousePress(5)
- elif y < 0:
- for _ in range(abs(y)):
- client.mousePress(4)
- # Horizontal: button 6 (right), button 7 (left) — not universally supported
- if x > 0:
- for _ in range(x):
- client.mousePress(6)
- elif x < 0:
- for _ in range(abs(x)):
- client.mousePress(7)
+ """Scroll. y>0 = up, y<0 = down (matches macOS native handler convention).
+
+ Uses arrow key presses instead of mouse buttons 4/5 because Apple's
+ _VZVNCServer (used by Lume) does not translate RFB mouse buttons 4-7
+ into scroll wheel events.
+ """
+ from twisted.internet import defer
+
+ @defer.inlineCallbacks
+ def _do(client):
+ if y > 0:
+ for _ in range(y):
+ yield client.keyPress("up")
+ elif y < 0:
+ for _ in range(abs(y)):
+ yield client.keyPress("down")
+ if x > 0:
+ for _ in range(x):
+ yield client.keyPress("right")
+ elif x < 0:
+ for _ in range(abs(x)):
+ yield client.keyPress("left")
+
+ self._with_client(_do)
# -- Keyboard -----------------------------------------------------------
def key_press(self, key: str):
- client = self._ensure_connected()
- client.keyPress(_translate_key(key))
+ self._with_client(lambda c: c.keyPress(_translate_key(key)))
def key_down(self, key: str):
- client = self._ensure_connected()
- client.keyDown(_translate_key(key))
+ self._with_client(lambda c: c.keyDown(_translate_key(key)))
def key_up(self, key: str):
- client = self._ensure_connected()
- client.keyUp(_translate_key(key))
+ self._with_client(lambda c: c.keyUp(_translate_key(key)))
def type_text(self, text: str):
"""Type text character by character, handling shift for uppercase/symbols."""
- client = self._ensure_connected()
- for ch in text:
- if ch == " ":
- client.keyPress("space")
- elif ch == "\n":
- client.keyPress("enter")
- elif ch == "\t":
- client.keyPress("tab")
- else:
- # vncdotool's keyPress handles single characters directly
- client.keyPress(ch)
+ from twisted.internet import defer
+
+ @defer.inlineCallbacks
+ def _do(client):
+ for ch in text:
+ if ch == " ":
+ yield client.keyPress("space")
+ elif ch == "\n":
+ yield client.keyPress("enter")
+ elif ch == "\t":
+ yield client.keyPress("tab")
+ else:
+ yield client.keyPress(ch)
+
+ self._with_client(_do)
def hotkey(self, keys: List[str]):
"""Press a key combination (e.g. ['command', 'a'])."""
- client = self._ensure_connected()
translated = [_translate_key(k) for k in keys]
- # vncdotool supports "modifier-key" syntax: "alt-a", "ctrl-shift-c"
combo = "-".join(translated)
- client.keyPress(combo)
+ self._with_client(lambda c: c.keyPress(combo))
# -- Clipboard ----------------------------------------------------------
def paste_text(self, text: str):
- client = self._ensure_connected()
- client.paste(text)
+ self._with_client(lambda c: c.paste(text))
# -- Info ---------------------------------------------------------------
@property
def screen_size(self) -> Tuple[int, int]:
- client = self._ensure_connected()
- proto = client.protocol
- if proto is not None and proto.screen is not None:
- return proto.screen.size
- return (0, 0)
+ from twisted.internet import defer
+
+ @defer.inlineCallbacks
+ def _do(client):
+ yield client.refreshScreen(incremental=False)
+ if client.screen is not None:
+ defer.returnValue(client.screen.size)
+ defer.returnValue((0, 0))
+
+ return self._with_client(_do)
@property
def cursor_position(self) -> Tuple[int, int]:
@@ -365,14 +477,8 @@ class VNCAutomationHandler(BaseAutomationHandler):
try:
btn = _BUTTON_MAP.get(button, 1)
cx, cy = self._conn.cursor_position
-
- def _drag():
- self._conn.mouse_down(cx, cy, btn)
- step = max(1, int(1 / max(duration, 0.01) * 5))
- self._conn.mouse_drag(x, y, step=step)
- self._conn.mouse_up(x, y, btn)
-
- await asyncio.to_thread(_drag)
+ step = max(1, int(1 / max(duration, 0.01) * 5))
+ await asyncio.to_thread(self._conn.drag_to, cx, cy, x, y, btn, step)
return {"success": True}
except Exception as e:
return {"success": False, "error": str(e)}
@@ -384,14 +490,7 @@ class VNCAutomationHandler(BaseAutomationHandler):
if not path:
return {"success": False, "error": "Empty path"}
btn = _BUTTON_MAP.get(button, 1)
-
- def _drag_path():
- self._conn.mouse_down(path[0][0], path[0][1], btn)
- for px, py in path[1:]:
- self._conn.mouse_move(px, py)
- self._conn.mouse_up(path[-1][0], path[-1][1], btn)
-
- await asyncio.to_thread(_drag_path)
+ await asyncio.to_thread(self._conn.drag_path, path, btn)
return {"success": True}
except Exception as e:
return {"success": False, "error": str(e)}
@@ -443,10 +542,10 @@ class VNCAutomationHandler(BaseAutomationHandler):
return {"success": False, "error": str(e)}
async def scroll_down(self, clicks: int = 1) -> Dict[str, Any]:
- return await self.scroll(0, clicks)
+ return await self.scroll(0, -clicks)
async def scroll_up(self, clicks: int = 1) -> Dict[str, Any]:
- return await self.scroll(0, -clicks)
+ return await self.scroll(0, clicks)
# -- Screen info --------------------------------------------------------