Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
fix(cua-driver): refuse background drags in Electron windows and fix the AppKit drag cell (#5335)
Francesco Bonacci2dab1f2a6703
diff --git a/libs/cua-driver/rust/crates/cua-driver-e2e/tests/cross_platform_behavior_test.rs b/libs/cua-driver/rust/crates/cua-driver-e2e/tests/cross_platform_behavior_test.rsindex 9877e57314..aa94f1a687 100644--- a/libs/cua-driver/rust/crates/cua-driver-e2e/tests/cross_platform_behavior_test.rs+++ b/libs/cua-driver/rust/crates/cua-driver-e2e/tests/cross_platform_behavior_test.rs@@ -1657,7 +1657,9 @@ fn shared_case_with_native_hyprland(// Electron scroll is delivered in the background through// accessibility (reveal scroll), including inner scrollers.// The shared drag cell is a pointer-event drag, which the- // background window-routed drag delivers (CUA-1207).+ // background window-routed drag delivers (CUA-1207). Electron+ // loses the release, so the driver refuses there.+ ("electron", "drag", Targeting::Px) => vec![RefusalCode::BackgroundUnavailable],_ => Vec::new(),}} else if cfg!(target_os = "linux")diff --git a/libs/cua-driver/rust/crates/cua-driver-e2e/tests/harness_appkit_test.rs b/libs/cua-driver/rust/crates/cua-driver-e2e/tests/harness_appkit_test.rsindex d62d0d816c..1e09774aa7 100644--- a/libs/cua-driver/rust/crates/cua-driver-e2e/tests/harness_appkit_test.rs+++ b/libs/cua-driver/rust/crates/cua-driver-e2e/tests/harness_appkit_test.rs@@ -2210,7 +2210,13 @@ fn harness_appkit_slider_drag_px_background() {"AppKit background drag failed: {}",response.text());- assert_eq!(response.structured()["path"].as_str(), Some("cgevent"));+ assert!(+ response+ .text()+ .contains("background: routed to the window's pid"),+ "AppKit background drag was not routed to the window's pid: {}",+ response.text()+ );std::thread::sleep(Duration::from_millis(300));assert!(!snapshot_elements(driver, pid, wid)diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/drag.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/drag.rsindex 6c8157b9ba..ff00ff8d2c 100644--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/drag.rs+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/drag.rs@@ -311,6 +311,22 @@ impl Tool for DragTool {{return refusal;}+ // Electron/Chromium windows take the press and the moves of a+ // background drag but lose the release (the page's pointer+ // capture is dropped), so the drop never lands. Refuse instead+ // of reporting a delivery that did not happen.+ if crate::browser::ElectronJs::is_electron(pid) {+ return cua_driver_core::delivery::background_unavailable_result(+ ELECTRON_BACKGROUND_DRAG_UNAVAILABLE,+ "background_unavailable",+ "Electron/Chromium windows lose the release of a background drag on macOS",+ serde_json::json!({+ "effect": "refused",+ "event_kind": "mouse_drag",+ "requires": ["delivery_mode:foreground", "window_id"],+ }),+ );+ }}// Animate agent cursor along drag path (start → end).@@ -548,6 +564,12 @@ impl Tool for DragTool {/// back) drag-and-drop session in a window of the source app at this level.const DRAGGING_WINDOW_LEVEL: i32 = 500;+const ELECTRON_BACKGROUND_DRAG_UNAVAILABLE: &str = "Background drag is unavailable for this \+ Electron/Chromium window on macOS: it takes the press and the moves but loses the release, \+ so the drop does not land. Use the app's own controls (a button, menu item or keyboard \+ route) or, if the user allows the window to come forward, retry with \+ delivery_mode:\"foreground\".";+fn drag_image_windows(pid: i32) -> std::collections::HashSet<u32> {crate::windows::pid_on_screen_windows_any_layer(pid).into_iter()