Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

fix(cua-driver): stop background actions from raising Chrome or re-activating the user's app (CUA-1309, CUA-1791) (#5334)

Francesco Bonacciffd65042bf0b
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/ax/bindings.rs b/libs/cua-driver/rust/crates/platform-macos/src/ax/bindings.rs
index 732ac0f353..91ba5def61 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/ax/bindings.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/ax/bindings.rs
@@ -551,6 +551,36 @@ pub fn focused_window_id_of_pid(pid: i32) -> Option<u32> {
}
}
+/// Put `pid`'s focused window (else its main window) back on top of the
+/// window order through the app's own `AXRaise`.
+///
+/// For the frontmost app this reorders without an activation: the app orders
+/// its own window, so it never resigns or regains active status and its key
+/// window does not change. `NSRunningApplication.activate` from the daemon,
+/// the alternative, is only a request under cooperative activation; Chromium
+/// keeps its window above it, and when it is honoured the user's app gets a
+/// fresh `didBecomeActive`.
+pub fn raise_focused_window_of_pid(pid: i32) -> bool {
+ unsafe {
+ let app = AXUIElementCreateApplication(pid);
+ if app.is_null() {
+ return false;
+ }
+ // A busy app must not stall the caller's order watch.
+ AXUIElementSetMessagingTimeout(app, 0.25);
+ let window = copy_element_attr(app, "AXFocusedWindow")
+ .or_else(|| copy_element_attr(app, "AXMainWindow"));
+ CFRelease(app as CFTypeRef);
+ let Some(window) = window else {
+ return false;
+ };
+ AXUIElementSetMessagingTimeout(window, 0.25);
+ let raised = perform_action(window, "AXRaise") == kAXErrorSuccess;
+ CFRelease(window as CFTypeRef);
+ raised
+ }
+}
+
/// The CGWindowID of the window that holds `pid`'s focused UI element
/// (`AXFocusedUIElement` → `AXWindow`). `None` when the app reports no focused
/// element: a LibreOffice that has never been active names a focused window
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/key_focus_guard.rs b/libs/cua-driver/rust/crates/platform-macos/src/key_focus_guard.rs
index afabdee4a1..4d1fee14c7 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/key_focus_guard.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/key_focus_guard.rs
@@ -66,8 +66,9 @@ const POPUP_MENU_LAYER: i32 = 101;
/// normal-level window of `front` in the on-screen order.
fn target_above_front(target: i32, front: i32) -> bool {
// Every layer, without the Space queries of `visible_windows`: this runs
- // while the raise is on screen, so it has to be quick.
- let windows = crate::windows::all_windows_any_layer();
+ // while the raise is on screen, so it has to be quick. On-screen windows
+ // only: the all-windows list did not show a Chromium raise.
+ let windows = crate::windows::on_screen_windows_any_layer();
target_above_front_in(&windows, target, front)
}
@@ -86,19 +87,28 @@ fn target_above_front_in(windows: &[crate::windows::WindowInfo], target: i32, fr
}
}
-/// Put the user's front app back above a background target that raised its
+/// Put the user's window back above a background target that raised its
/// window (AppKit orders a popup menu's window group to the front, which
-/// lifts its parent window). Re-activating the already-active front app with
-/// all its windows restores the order without moving key status; the
-/// target's menu stays open.
-/// Put `front`'s windows back above `target` if the target's window was
-/// lifted over them (see [`restore_front_if_raised`]).
+/// lifts its parent window). The user's app raises its own focused window,
+/// which restores the order without an activation or a key change; the
+/// target's menu stays open. See [`put_front_window_back`].
pub fn restore_order_now(target: i32, front: i32) -> bool {
// NSWorkspace's frontmost reading can lag inside the daemon; the caller
// has already checked WindowServer's front process.
if !target_above_front(target, front) {
return false;
}
+ put_front_window_back(front)
+}
+
+/// Order the user's window back above a raised target without activating
+/// anything: the user's app raises its own focused window (`AXRaise`). Only
+/// if that fails, re-activate the user's app with all its windows, which is a
+/// request the system may ignore and, when honoured, re-activates the app.
+fn put_front_window_back(front: i32) -> bool {
+ if crate::ax::bindings::raise_focused_window_of_pid(front) {
+ return true;
+ }
use objc2_app_kit::{NSApplicationActivationOptions, NSRunningApplication};
unsafe {
NSRunningApplication::runningApplicationWithProcessIdentifier(front).is_some_and(|app| {
@@ -117,12 +127,7 @@ fn restore_front_if_raised(target: i32, front: i32) -> bool {
if crate::apps::frontmost_pid() != Some(front) || !target_above_front(target, front) {
return false;
}
- use objc2_app_kit::{NSApplicationActivationOptions, NSRunningApplication};
- let restored = unsafe {
- NSRunningApplication::runningApplicationWithProcessIdentifier(front).is_some_and(|app| {
- app.activateWithOptions(NSApplicationActivationOptions::NSApplicationActivateAllWindows)
- })
- };
+ let restored = put_front_window_back(front);
RESTORED.fetch_add(1, Ordering::Relaxed);
tracing::debug!(
target,
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/click.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/click.rs
index 8dceb7c5f7..70105d483c 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/click.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/click.rs
@@ -259,9 +259,18 @@ pub(super) async fn restore_user_focus_after_background_pixel(
// The prologue leaves the user's window key, but the input can still make
// the target self-activate, and re-activating the prior app does not by
// itself make its window key again. Post the focus record to the user's
- // window in both cases so their typing keeps landing there.
+ // window only then, or when WindowServer's key focus left the user's app.
+ // Posting it while their window is still key re-activates their app
+ // (AppKit sends it didBecomeActive), which Bench v2 counted as a
+ // disturbance after every background pixel click.
if let (Some(previous_pid), Some(wid)) = (prior_front, window_id) {
- if (focus_without_raise || reactivated) && apps::frontmost_pid() == Some(previous_pid) {
+ let key_moved = user_focus_needs_restore(
+ reactivated,
+ focus_without_raise,
+ crate::input::skylight::key_focus_pid(),
+ previous_pid,
+ );
+ if key_moved && apps::frontmost_pid() == Some(previous_pid) {
let _ = tokio::task::spawn_blocking(move || {
crate::input::skylight::restore_focus_after_without_raise(
previous_pid,
@@ -275,6 +284,19 @@ pub(super) async fn restore_user_focus_after_background_pixel(
}
}
+/// Whether the user's window needs the focus record back after a background
+/// pixel action: their app was re-activated, or WindowServer's key focus
+/// (`key_focus`) left it. Not when it never moved, since the record then makes
+/// their app become active again.
+fn user_focus_needs_restore(
+ reactivated: bool,
+ focus_without_raise: bool,
+ key_focus: Option<i32>,
+ previous_pid: i32,
+) -> bool {
+ reactivated || (focus_without_raise && key_focus.is_some_and(|pid| pid != previous_pid))
+}
+
/// Poll until `pid` is the frontmost application or `budget` runs out.
async fn wait_until_frontmost(pid: i32, budget: std::time::Duration) -> bool {
let deadline = std::time::Instant::now() + budget;
@@ -1958,6 +1980,17 @@ mod tests {
);
}
+ /// Bench v2: re-posting the focus record to a user window that stayed key
+ /// re-activated the user's app after every background pixel click.
+ #[test]
+ fn user_focus_record_only_when_focus_moved() {
+ assert!(!user_focus_needs_restore(false, true, Some(7), 7));
+ assert!(!user_focus_needs_restore(false, true, None, 7));
+ assert!(!user_focus_needs_restore(false, false, Some(42), 7));
+ assert!(user_focus_needs_restore(false, true, Some(42), 7));
+ assert!(user_focus_needs_restore(true, false, Some(7), 7));
+ }
+
/// The no-foreground contract must not depend on the private activation
/// recipe reporting full success. If that recipe is unavailable or only
/// partially succeeds but the target is nevertheless observed frontmost,
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/set_value.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/set_value.rs
index e16e87ceb4..df1b6b1aa9 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/set_value.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/set_value.rs
@@ -222,7 +222,13 @@ impl Tool for SetValueTool {
"set_value.AXValue",
|| async move {
tokio::task::spawn_blocking(move || {
- set_value_blocking(element_guard.as_ptr(), element_index, pid, &value)
+ set_value_blocking(
+ element_guard.as_ptr(),
+ element_index,
+ pid,
+ window_id,
+ &value,
+ )
})
.await
},
@@ -468,6 +474,7 @@ fn set_value_blocking(
element_ptr: usize,
element_index: usize,
pid: i32,
+ window_id: u32,
value: &str,
) -> anyhow::Result<SetValueOutcome> {
let element = element_ptr as AXUIElementRef;
@@ -477,14 +484,20 @@ fn set_value_blocking(
if role == "AXPopUpButton" {
let element_title = unsafe { copy_string_attr(element, "AXTitle") }.unwrap_or_default();
// Menu-item selection, not an AXValue write — no read-back to report.
- select_popup_option(element, element_index, pid, value, &element_title).map(|detail| {
- SetValueOutcome {
- detail,
- verified: None,
- changed: None,
- not_applied: false,
- document_edited: None,
- }
+ select_popup_option(
+ element,
+ element_index,
+ pid,
+ window_id,
+ value,
+ &element_title,
+ )
+ .map(|detail| SetValueOutcome {
+ detail,
+ verified: None,
+ changed: None,
+ not_applied: false,
+ document_edited: None,
})
} else {
// Default path: write AXValue directly. Numeric controls (AXSlider /
@@ -852,13 +865,103 @@ fn dismiss_popup_menu(popup: AXUIElementRef, pid: i32) {
}
}
+/// Whether a popup belongs to a Chromium renderer (Chrome-family browsers and
+/// Electron apps), whose `<select>` menu is a native popup.
+fn is_chromium_popup_owner(pid: i32) -> bool {
+ let bundle_id = crate::apps::bundle_id_for_pid(pid).unwrap_or_default();
+ super::type_text::is_chromium_identity(&bundle_id)
+ || crate::browser::electron_js::ElectronJs::is_electron(pid)
+}
+
+/// How long the page may take to apply a type-ahead selection.
+const TYPEAHEAD_SETTLE: std::time::Duration = std::time::Duration::from_millis(800);
+
+/// Select a background Chromium `<select>` without opening its menu.
+///
+/// Chromium shows a `<select>` menu as a native popup. Opening it orders the
+/// whole browser window above the user's window, and the window stays there
+/// after the menu closes; re-activating the user's app does not reorder it
+/// (Bench v2: WF-02 to WF-08, IR-05, IR-06, MA-02, CDB-G04). A focused closed
+/// `<select>` takes the page's own type-ahead instead: typing an option's
+/// label selects it and fires `input` and `change`, and the menu never opens.
+///
+/// Returns `None` when this route does not apply or did not land on `value`;
+/// the caller then falls back to opening the menu.
+fn select_chromium_popup_by_typeahead(
+ popup: AXUIElementRef,
+ element_index: usize,
+ pid: i32,
+ window_id: u32,
+ value: &str,
+ element_title: &str,
+) -> Option<String> {
+ let wanted = value.trim();
+ if wanted.is_empty()
+ || crate::apps::frontmost_pid() == Some(pid)
+ || popup_menu_is_open(popup)
+ || !is_chromium_popup_owner(pid)
+ {
+ return None;
+ }
+ let selected = || unsafe { copy_string_attr(popup, "AXValue") }.unwrap_or_default();
+ let matches = |current: &str| current.trim().to_lowercase() == wanted.to_lowercase();
+ if matches(&selected()) {
+ return Some(format!(
+ "✅ '{wanted}' is already selected in AXPopUpButton [{element_index}] \
+ \"{element_title}\"; nothing was changed."
+ ));
+ }
+ // Focus moves inside the page only: no activation, no window order change.
+ unsafe { crate::ax::bindings::set_bool_attr_true(popup, "AXFocused") };
+ std::thread::sleep(POPUP_POLL_INTERVAL);
+ if unsafe { copy_bool_attr(popup, "AXFocused") } != Some(true) {
+ return None;
+ }
+ // Keys go to the window's focused element, so the window must be the
+ // app's key window (in-app records only; the app is not activated).
+ let _scope = super::key_scope(pid, Some(window_id)).ok()?;
+ crate::input::keyboard::type_text(pid, wanted).ok()?;
+ let deadline = std::time::Instant::now() + TYPEAHEAD_SETTLE;
+ loop {
+ let current = selected();
+ if matches(&current) {
+ return Some(format!(
+ "✅ Selected '{current}' in AXPopUpButton [{element_index}] \
+ \"{element_title}\" by typing its label into the focused control \
+ (the menu was not opened, so the window order did not change)."
+ ));
+ }
+ if std::time::Instant::now() >= deadline {
+ tracing::debug!(
+ pid,
+ wanted,
+ current,
+ "type-ahead did not select the option; opening the menu"
+ );
+ return None;
+ }
+ std::thread::sleep(POPUP_POLL_INTERVAL);
+ }
+}
+
fn select_popup_option(
element: AXUIElementRef,
element_index: usize,
pid: i32,
+ window_id: u32,
value: &str,
element_title: &str,
) -> anyhow::Result<String> {
+ if let Some(detail) = select_chromium_popup_by_typeahead(
+ element,
+ element_index,
+ pid,
+ window_id,
+ value,
+ element_title,
+ ) {
+ return Ok(detail);
+ }
let mut options = popup_options(element);
let mut opened_menu = false;
let mut press_thread = None;
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/windows.rs b/libs/cua-driver/rust/crates/platform-macos/src/windows.rs
index 0b5e24b195..052e5f83f7 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/windows.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/windows.rs
@@ -119,6 +119,21 @@ pub(crate) fn pid_on_screen_windows_any_layer(pid: i32) -> Vec<WindowInfo> {
.windows
}
+/// On-screen windows of every app on every layer, front to back, without the
+/// Space snapshot. Use this, not [`all_windows_any_layer`], to compare two
+/// apps' window order: the all-windows list (off-screen windows included) did
+/// not follow a raise in Bench v2 (a Chromium window raised over the user's
+/// window still read as behind it), while the on-screen list does.
+pub(crate) fn on_screen_windows_any_layer() -> Vec<WindowInfo> {
+ enumerate_windows(
+ kCGWindowListOptionOnScreenOnly | kCGWindowListExcludeDesktopElements,
+ LayerFilter::AnyLayer,
+ None,
+ false,
+ )
+ .windows
+}
+
pub(crate) fn prefer_layer0(windows: &mut Vec<WindowInfo>) {
if windows.iter().any(|w| w.layer == 0) {
windows.retain(|w| w.layer == 0);