Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

fix(cua-driver): refuse background drags in Electron windows and fix the AppKit drag cell (#5335)

Francesco Bonacci2dab1f2a6703
diff --git a/libs/cua-driver/rust/crates/cua-driver-e2e/tests/cross_platform_behavior_test.rs b/libs/cua-driver/rust/crates/cua-driver-e2e/tests/cross_platform_behavior_test.rs
index 9877e57314..aa94f1a687 100644
--- a/libs/cua-driver/rust/crates/cua-driver-e2e/tests/cross_platform_behavior_test.rs
+++ b/libs/cua-driver/rust/crates/cua-driver-e2e/tests/cross_platform_behavior_test.rs
@@ -1657,7 +1657,9 @@ fn shared_case_with_native_hyprland(
// Electron scroll is delivered in the background through
// accessibility (reveal scroll), including inner scrollers.
// The shared drag cell is a pointer-event drag, which the
- // background window-routed drag delivers (CUA-1207).
+ // background window-routed drag delivers (CUA-1207). Electron
+ // loses the release, so the driver refuses there.
+ ("electron", "drag", Targeting::Px) => vec![RefusalCode::BackgroundUnavailable],
_ => Vec::new(),
}
} else if cfg!(target_os = "linux")
diff --git a/libs/cua-driver/rust/crates/cua-driver-e2e/tests/harness_appkit_test.rs b/libs/cua-driver/rust/crates/cua-driver-e2e/tests/harness_appkit_test.rs
index d62d0d816c..1e09774aa7 100644
--- a/libs/cua-driver/rust/crates/cua-driver-e2e/tests/harness_appkit_test.rs
+++ b/libs/cua-driver/rust/crates/cua-driver-e2e/tests/harness_appkit_test.rs
@@ -2210,7 +2210,13 @@ fn harness_appkit_slider_drag_px_background() {
"AppKit background drag failed: {}",
response.text()
);
- assert_eq!(response.structured()["path"].as_str(), Some("cgevent"));
+ assert!(
+ response
+ .text()
+ .contains("background: routed to the window's pid"),
+ "AppKit background drag was not routed to the window's pid: {}",
+ response.text()
+ );
std::thread::sleep(Duration::from_millis(300));
assert!(
!snapshot_elements(driver, pid, wid)
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/drag.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/drag.rs
index 6c8157b9ba..ff00ff8d2c 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/drag.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/drag.rs
@@ -311,6 +311,22 @@ impl Tool for DragTool {
{
return refusal;
}
+ // Electron/Chromium windows take the press and the moves of a
+ // background drag but lose the release (the page's pointer
+ // capture is dropped), so the drop never lands. Refuse instead
+ // of reporting a delivery that did not happen.
+ if crate::browser::ElectronJs::is_electron(pid) {
+ return cua_driver_core::delivery::background_unavailable_result(
+ ELECTRON_BACKGROUND_DRAG_UNAVAILABLE,
+ "background_unavailable",
+ "Electron/Chromium windows lose the release of a background drag on macOS",
+ serde_json::json!({
+ "effect": "refused",
+ "event_kind": "mouse_drag",
+ "requires": ["delivery_mode:foreground", "window_id"],
+ }),
+ );
+ }
}
// Animate agent cursor along drag path (start → end).
@@ -548,6 +564,12 @@ impl Tool for DragTool {
/// back) drag-and-drop session in a window of the source app at this level.
const DRAGGING_WINDOW_LEVEL: i32 = 500;
+const ELECTRON_BACKGROUND_DRAG_UNAVAILABLE: &str = "Background drag is unavailable for this \
+ Electron/Chromium window on macOS: it takes the press and the moves but loses the release, \
+ so the drop does not land. Use the app's own controls (a button, menu item or keyboard \
+ route) or, if the user allows the window to come forward, retry with \
+ delivery_mode:\"foreground\".";
+
fn drag_image_windows(pid: i32) -> std::collections::HashSet<u32> {
crate::windows::pid_on_screen_windows_any_layer(pid)
.into_iter()