Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
fix(cua-driver): stop background actions from raising Chrome or re-activating the user's app (CUA-1309, CUA-1791) (#5334)
Francesco Bonacciffd65042bf0b
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/ax/bindings.rs b/libs/cua-driver/rust/crates/platform-macos/src/ax/bindings.rsindex 732ac0f353..91ba5def61 100644--- a/libs/cua-driver/rust/crates/platform-macos/src/ax/bindings.rs+++ b/libs/cua-driver/rust/crates/platform-macos/src/ax/bindings.rs@@ -551,6 +551,36 @@ pub fn focused_window_id_of_pid(pid: i32) -> Option<u32> {}}+/// Put `pid`'s focused window (else its main window) back on top of the+/// window order through the app's own `AXRaise`.+///+/// For the frontmost app this reorders without an activation: the app orders+/// its own window, so it never resigns or regains active status and its key+/// window does not change. `NSRunningApplication.activate` from the daemon,+/// the alternative, is only a request under cooperative activation; Chromium+/// keeps its window above it, and when it is honoured the user's app gets a+/// fresh `didBecomeActive`.+pub fn raise_focused_window_of_pid(pid: i32) -> bool {+ unsafe {+ let app = AXUIElementCreateApplication(pid);+ if app.is_null() {+ return false;+ }+ // A busy app must not stall the caller's order watch.+ AXUIElementSetMessagingTimeout(app, 0.25);+ let window = copy_element_attr(app, "AXFocusedWindow")+ .or_else(|| copy_element_attr(app, "AXMainWindow"));+ CFRelease(app as CFTypeRef);+ let Some(window) = window else {+ return false;+ };+ AXUIElementSetMessagingTimeout(window, 0.25);+ let raised = perform_action(window, "AXRaise") == kAXErrorSuccess;+ CFRelease(window as CFTypeRef);+ raised+ }+}+/// The CGWindowID of the window that holds `pid`'s focused UI element/// (`AXFocusedUIElement` → `AXWindow`). `None` when the app reports no focused/// element: a LibreOffice that has never been active names a focused windowdiff --git a/libs/cua-driver/rust/crates/platform-macos/src/key_focus_guard.rs b/libs/cua-driver/rust/crates/platform-macos/src/key_focus_guard.rsindex afabdee4a1..4d1fee14c7 100644--- a/libs/cua-driver/rust/crates/platform-macos/src/key_focus_guard.rs+++ b/libs/cua-driver/rust/crates/platform-macos/src/key_focus_guard.rs@@ -66,8 +66,9 @@ const POPUP_MENU_LAYER: i32 = 101;/// normal-level window of `front` in the on-screen order.fn target_above_front(target: i32, front: i32) -> bool {// Every layer, without the Space queries of `visible_windows`: this runs- // while the raise is on screen, so it has to be quick.- let windows = crate::windows::all_windows_any_layer();+ // while the raise is on screen, so it has to be quick. On-screen windows+ // only: the all-windows list did not show a Chromium raise.+ let windows = crate::windows::on_screen_windows_any_layer();target_above_front_in(&windows, target, front)}@@ -86,19 +87,28 @@ fn target_above_front_in(windows: &[crate::windows::WindowInfo], target: i32, fr}}-/// Put the user's front app back above a background target that raised its+/// Put the user's window back above a background target that raised its/// window (AppKit orders a popup menu's window group to the front, which-/// lifts its parent window). Re-activating the already-active front app with-/// all its windows restores the order without moving key status; the-/// target's menu stays open.-/// Put `front`'s windows back above `target` if the target's window was-/// lifted over them (see [`restore_front_if_raised`]).+/// lifts its parent window). The user's app raises its own focused window,+/// which restores the order without an activation or a key change; the+/// target's menu stays open. See [`put_front_window_back`].pub fn restore_order_now(target: i32, front: i32) -> bool {// NSWorkspace's frontmost reading can lag inside the daemon; the caller// has already checked WindowServer's front process.if !target_above_front(target, front) {return false;}+ put_front_window_back(front)+}++/// Order the user's window back above a raised target without activating+/// anything: the user's app raises its own focused window (`AXRaise`). Only+/// if that fails, re-activate the user's app with all its windows, which is a+/// request the system may ignore and, when honoured, re-activates the app.+fn put_front_window_back(front: i32) -> bool {+ if crate::ax::bindings::raise_focused_window_of_pid(front) {+ return true;+ }use objc2_app_kit::{NSApplicationActivationOptions, NSRunningApplication};unsafe {NSRunningApplication::runningApplicationWithProcessIdentifier(front).is_some_and(|app| {@@ -117,12 +127,7 @@ fn restore_front_if_raised(target: i32, front: i32) -> bool {if crate::apps::frontmost_pid() != Some(front) || !target_above_front(target, front) {return false;}- use objc2_app_kit::{NSApplicationActivationOptions, NSRunningApplication};- let restored = unsafe {- NSRunningApplication::runningApplicationWithProcessIdentifier(front).is_some_and(|app| {- app.activateWithOptions(NSApplicationActivationOptions::NSApplicationActivateAllWindows)- })- };+ let restored = put_front_window_back(front);RESTORED.fetch_add(1, Ordering::Relaxed);tracing::debug!(target,diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/click.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/click.rsindex 8dceb7c5f7..70105d483c 100644--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/click.rs+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/click.rs@@ -259,9 +259,18 @@ pub(super) async fn restore_user_focus_after_background_pixel(// The prologue leaves the user's window key, but the input can still make// the target self-activate, and re-activating the prior app does not by// itself make its window key again. Post the focus record to the user's- // window in both cases so their typing keeps landing there.+ // window only then, or when WindowServer's key focus left the user's app.+ // Posting it while their window is still key re-activates their app+ // (AppKit sends it didBecomeActive), which Bench v2 counted as a+ // disturbance after every background pixel click.if let (Some(previous_pid), Some(wid)) = (prior_front, window_id) {- if (focus_without_raise || reactivated) && apps::frontmost_pid() == Some(previous_pid) {+ let key_moved = user_focus_needs_restore(+ reactivated,+ focus_without_raise,+ crate::input::skylight::key_focus_pid(),+ previous_pid,+ );+ if key_moved && apps::frontmost_pid() == Some(previous_pid) {let _ = tokio::task::spawn_blocking(move || {crate::input::skylight::restore_focus_after_without_raise(previous_pid,@@ -275,6 +284,19 @@ pub(super) async fn restore_user_focus_after_background_pixel(}}+/// Whether the user's window needs the focus record back after a background+/// pixel action: their app was re-activated, or WindowServer's key focus+/// (`key_focus`) left it. Not when it never moved, since the record then makes+/// their app become active again.+fn user_focus_needs_restore(+ reactivated: bool,+ focus_without_raise: bool,+ key_focus: Option<i32>,+ previous_pid: i32,+) -> bool {+ reactivated || (focus_without_raise && key_focus.is_some_and(|pid| pid != previous_pid))+}+/// Poll until `pid` is the frontmost application or `budget` runs out.async fn wait_until_frontmost(pid: i32, budget: std::time::Duration) -> bool {let deadline = std::time::Instant::now() + budget;@@ -1958,6 +1980,17 @@ mod tests {);}+ /// Bench v2: re-posting the focus record to a user window that stayed key+ /// re-activated the user's app after every background pixel click.+ #[test]+ fn user_focus_record_only_when_focus_moved() {+ assert!(!user_focus_needs_restore(false, true, Some(7), 7));+ assert!(!user_focus_needs_restore(false, true, None, 7));+ assert!(!user_focus_needs_restore(false, false, Some(42), 7));+ assert!(user_focus_needs_restore(false, true, Some(42), 7));+ assert!(user_focus_needs_restore(true, false, Some(7), 7));+ }+/// The no-foreground contract must not depend on the private activation/// recipe reporting full success. If that recipe is unavailable or only/// partially succeeds but the target is nevertheless observed frontmost,diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/set_value.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/set_value.rsindex e16e87ceb4..df1b6b1aa9 100644--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/set_value.rs+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/set_value.rs@@ -222,7 +222,13 @@ impl Tool for SetValueTool {"set_value.AXValue",|| async move {tokio::task::spawn_blocking(move || {- set_value_blocking(element_guard.as_ptr(), element_index, pid, &value)+ set_value_blocking(+ element_guard.as_ptr(),+ element_index,+ pid,+ window_id,+ &value,+ )}).await},@@ -468,6 +474,7 @@ fn set_value_blocking(element_ptr: usize,element_index: usize,pid: i32,+ window_id: u32,value: &str,) -> anyhow::Result<SetValueOutcome> {let element = element_ptr as AXUIElementRef;@@ -477,14 +484,20 @@ fn set_value_blocking(if role == "AXPopUpButton" {let element_title = unsafe { copy_string_attr(element, "AXTitle") }.unwrap_or_default();// Menu-item selection, not an AXValue write — no read-back to report.- select_popup_option(element, element_index, pid, value, &element_title).map(|detail| {- SetValueOutcome {- detail,- verified: None,- changed: None,- not_applied: false,- document_edited: None,- }+ select_popup_option(+ element,+ element_index,+ pid,+ window_id,+ value,+ &element_title,+ )+ .map(|detail| SetValueOutcome {+ detail,+ verified: None,+ changed: None,+ not_applied: false,+ document_edited: None,})} else {// Default path: write AXValue directly. Numeric controls (AXSlider /@@ -852,13 +865,103 @@ fn dismiss_popup_menu(popup: AXUIElementRef, pid: i32) {}}+/// Whether a popup belongs to a Chromium renderer (Chrome-family browsers and+/// Electron apps), whose `<select>` menu is a native popup.+fn is_chromium_popup_owner(pid: i32) -> bool {+ let bundle_id = crate::apps::bundle_id_for_pid(pid).unwrap_or_default();+ super::type_text::is_chromium_identity(&bundle_id)+ || crate::browser::electron_js::ElectronJs::is_electron(pid)+}++/// How long the page may take to apply a type-ahead selection.+const TYPEAHEAD_SETTLE: std::time::Duration = std::time::Duration::from_millis(800);++/// Select a background Chromium `<select>` without opening its menu.+///+/// Chromium shows a `<select>` menu as a native popup. Opening it orders the+/// whole browser window above the user's window, and the window stays there+/// after the menu closes; re-activating the user's app does not reorder it+/// (Bench v2: WF-02 to WF-08, IR-05, IR-06, MA-02, CDB-G04). A focused closed+/// `<select>` takes the page's own type-ahead instead: typing an option's+/// label selects it and fires `input` and `change`, and the menu never opens.+///+/// Returns `None` when this route does not apply or did not land on `value`;+/// the caller then falls back to opening the menu.+fn select_chromium_popup_by_typeahead(+ popup: AXUIElementRef,+ element_index: usize,+ pid: i32,+ window_id: u32,+ value: &str,+ element_title: &str,+) -> Option<String> {+ let wanted = value.trim();+ if wanted.is_empty()+ || crate::apps::frontmost_pid() == Some(pid)+ || popup_menu_is_open(popup)+ || !is_chromium_popup_owner(pid)+ {+ return None;+ }+ let selected = || unsafe { copy_string_attr(popup, "AXValue") }.unwrap_or_default();+ let matches = |current: &str| current.trim().to_lowercase() == wanted.to_lowercase();+ if matches(&selected()) {+ return Some(format!(+ "✅ '{wanted}' is already selected in AXPopUpButton [{element_index}] \+ \"{element_title}\"; nothing was changed."+ ));+ }+ // Focus moves inside the page only: no activation, no window order change.+ unsafe { crate::ax::bindings::set_bool_attr_true(popup, "AXFocused") };+ std::thread::sleep(POPUP_POLL_INTERVAL);+ if unsafe { copy_bool_attr(popup, "AXFocused") } != Some(true) {+ return None;+ }+ // Keys go to the window's focused element, so the window must be the+ // app's key window (in-app records only; the app is not activated).+ let _scope = super::key_scope(pid, Some(window_id)).ok()?;+ crate::input::keyboard::type_text(pid, wanted).ok()?;+ let deadline = std::time::Instant::now() + TYPEAHEAD_SETTLE;+ loop {+ let current = selected();+ if matches(¤t) {+ return Some(format!(+ "✅ Selected '{current}' in AXPopUpButton [{element_index}] \+ \"{element_title}\" by typing its label into the focused control \+ (the menu was not opened, so the window order did not change)."+ ));+ }+ if std::time::Instant::now() >= deadline {+ tracing::debug!(+ pid,+ wanted,+ current,+ "type-ahead did not select the option; opening the menu"+ );+ return None;+ }+ std::thread::sleep(POPUP_POLL_INTERVAL);+ }+}+fn select_popup_option(element: AXUIElementRef,element_index: usize,pid: i32,+ window_id: u32,value: &str,element_title: &str,) -> anyhow::Result<String> {+ if let Some(detail) = select_chromium_popup_by_typeahead(+ element,+ element_index,+ pid,+ window_id,+ value,+ element_title,+ ) {+ return Ok(detail);+ }let mut options = popup_options(element);let mut opened_menu = false;let mut press_thread = None;diff --git a/libs/cua-driver/rust/crates/platform-macos/src/windows.rs b/libs/cua-driver/rust/crates/platform-macos/src/windows.rsindex 0b5e24b195..052e5f83f7 100644--- a/libs/cua-driver/rust/crates/platform-macos/src/windows.rs+++ b/libs/cua-driver/rust/crates/platform-macos/src/windows.rs@@ -119,6 +119,21 @@ pub(crate) fn pid_on_screen_windows_any_layer(pid: i32) -> Vec<WindowInfo> {.windows}+/// On-screen windows of every app on every layer, front to back, without the+/// Space snapshot. Use this, not [`all_windows_any_layer`], to compare two+/// apps' window order: the all-windows list (off-screen windows included) did+/// not follow a raise in Bench v2 (a Chromium window raised over the user's+/// window still read as behind it), while the on-screen list does.+pub(crate) fn on_screen_windows_any_layer() -> Vec<WindowInfo> {+ enumerate_windows(+ kCGWindowListOptionOnScreenOnly | kCGWindowListExcludeDesktopElements,+ LayerFilter::AnyLayer,+ None,+ false,+ )+ .windows+}+pub(crate) fn prefer_layer0(windows: &mut Vec<WindowInfo>) {if windows.iter().any(|w| w.layer == 0) {windows.retain(|w| w.layer == 0);