Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

fix(cua-driver): deliver window-only foreground hotkeys through the HID activation guard

injaneitya4b5fec16a25
diff --git a/libs/cua-driver/rust/crates/cua-driver-e2e/tests/README.md b/libs/cua-driver/rust/crates/cua-driver-e2e/tests/README.md
index 18e422129d..d5e2edc270 100644
--- a/libs/cua-driver/rust/crates/cua-driver-e2e/tests/README.md
+++ b/libs/cua-driver/rust/crates/cua-driver-e2e/tests/README.md
@@ -146,7 +146,9 @@ desktop state outside the repo-local fixtures:
- `installed_app_launch_macos_test.rs`: macOS Calculator/TextEdit launch focus
checks. Requires a logged-in GUI session and usable System Events scripting.
- `installed_app_textedit_macos_test.rs`: real TextEdit background AX write and
- verification. Requires a logged-in GUI session and Accessibility permission.
+ verification, plus the window-only foreground chord delivered while Finder
+ holds the front or while a sibling document holds the keyboard focus.
+ Requires a logged-in GUI session and Accessibility permission.
- `standalone_browser_behavior_test.rs`: installed Chrome and Edge browser-tool
targeting, mutation, frame, tab, ambiguity, and isolated-profile rows. The
harness owns a fresh browser profile and creates adversarial tabs and windows
diff --git a/libs/cua-driver/rust/crates/cua-driver-e2e/tests/installed_app_textedit_macos_test.rs b/libs/cua-driver/rust/crates/cua-driver-e2e/tests/installed_app_textedit_macos_test.rs
index 94ff8d7445..f8fc449efe 100644
--- a/libs/cua-driver/rust/crates/cua-driver-e2e/tests/installed_app_textedit_macos_test.rs
+++ b/libs/cua-driver/rust/crates/cua-driver-e2e/tests/installed_app_textedit_macos_test.rs
@@ -1,13 +1,232 @@
-//! TextEdit background-delivery integration check for macOS.
+//! TextEdit delivery integration checks for macOS.
//!
//! Covers the accessibility route and confirmed value read-back on a real
-//! Cocoa app, then reads the editor back through a fresh snapshot.
+//! Cocoa app, then reads the editor back through a fresh snapshot, and the
+//! window-only foreground chord while another app holds the front or while a
+//! sibling document of the same app holds the keyboard focus.
//!
-//! The schema contract lives in `protocol_schema_test.rs`. This installed-app
-//! check runs in the canonical logged-in macOS desktop lane.
+//! The schema contract lives in `protocol_schema_test.rs`. These installed-app
+//! checks run in the canonical logged-in macOS desktop lane.
#![cfg(target_os = "macos")]
+use std::process::Command;
+use std::time::{Duration, Instant};
+
+use cua_driver_testkit::e2e::{
+ execute_case, recording_evidence, CaseSpec, Delivery, DriverRoute, Observation, OracleKind,
+ Scope, Targeting,
+};
+use cua_driver_testkit::{Driver, McpDriver};
+
+const FINDER_BUNDLE: &str = "com.apple.finder";
+const TEXTEDIT_BUNDLE: &str = "com.apple.TextEdit";
+
+struct TextEditDocument {
+ pid: i64,
+ window_id: u64,
+ element_token: String,
+}
+
+/// TextEdit may restore an earlier unsaved document, so read-back oracles need
+/// text that no earlier run could have left behind.
+fn unique_marker(prefix: &str) -> String {
+ format!(
+ "{prefix}{}",
+ std::time::SystemTime::now()
+ .duration_since(std::time::UNIX_EPOCH)
+ .expect("system clock after the Unix epoch")
+ .as_nanos()
+ )
+}
+
+/// Launch TextEdit and locate the on-screen window that exposes the editor.
+///
+/// TextEdit can retain an off-screen restoration window alongside the visible
+/// blank document, and the document window can appear a moment after launch.
+/// WindowServer does not guarantee their order, so prefer visible windows and
+/// select the one that actually exposes the editor instead of assuming
+/// `windows[0]` is the document.
+fn launch_textedit_document(driver: &mut McpDriver) -> TextEditDocument {
+ let launch = driver.call(
+ "launch_app",
+ serde_json::json!({ "bundle_id": TEXTEDIT_BUNDLE }),
+ );
+ assert!(
+ !launch.is_error(),
+ "could not launch TextEdit: {}",
+ launch.text()
+ );
+ let pid = launch.structured()["pid"].as_i64().expect("TextEdit pid");
+ let mut windows = launch.structured()["windows"]
+ .as_array()
+ .cloned()
+ .unwrap_or_default();
+ let deadline = Instant::now() + Duration::from_secs(5);
+ loop {
+ windows.sort_by_key(|window| !window["is_on_screen"].as_bool().unwrap_or(false));
+ let editor = windows
+ .iter()
+ .filter_map(|window| window["window_id"].as_u64())
+ .find_map(|window_id| {
+ let state = driver.call(
+ "get_window_state",
+ serde_json::json!({
+ "pid": pid,
+ "window_id": window_id,
+ "capture_mode": "ax"
+ }),
+ );
+ state.structured()["elements"]
+ .as_array()
+ .and_then(|elements| {
+ elements
+ .iter()
+ .find(|element| element["role"] == "AXTextArea")
+ .and_then(|element| element["element_token"].as_str())
+ })
+ .map(|element_token| (window_id, element_token.to_owned()))
+ });
+ if let Some((window_id, element_token)) = editor {
+ return TextEditDocument {
+ pid,
+ window_id,
+ element_token,
+ };
+ }
+ assert!(
+ Instant::now() < deadline,
+ "TextEdit opened no window containing an AXTextArea: {windows:?}"
+ );
+ std::thread::sleep(Duration::from_millis(250));
+ let listed = driver.call("list_windows", serde_json::json!({}));
+ windows = listed.structured()["windows"]
+ .as_array()
+ .map(|listed| {
+ listed
+ .iter()
+ .filter(|window| window["pid"].as_i64() == Some(pid))
+ .cloned()
+ .collect()
+ })
+ .unwrap_or_default();
+ }
+}
+
+/// The driver's own read-back is not the oracle. Take fresh snapshots until the
+/// editor holds the expected text, and return the editor's token from that
+/// snapshot since each snapshot retires the earlier tokens.
+fn wait_for_editor_text(
+ driver: &mut McpDriver,
+ document: &TextEditDocument,
+ accept: impl Fn(&str) -> bool,
+ expectation: &str,
+) -> String {
+ let deadline = Instant::now() + Duration::from_secs(5);
+ loop {
+ let state = driver.call(
+ "get_window_state",
+ serde_json::json!({
+ "pid": document.pid,
+ "window_id": document.window_id,
+ "capture_mode": "ax"
+ }),
+ );
+ let landed = state.structured()["elements"]
+ .as_array()
+ .and_then(|elements| {
+ elements.iter().find(|element| {
+ element["role"] == "AXTextArea"
+ && element["value"].as_str().is_some_and(&accept)
+ })
+ })
+ .and_then(|element| element["element_token"].as_str());
+ if let Some(element_token) = landed {
+ return element_token.to_owned();
+ }
+ assert!(
+ Instant::now() < deadline,
+ "TextEdit's AXTextArea never {expectation}:\n{}",
+ state.text()
+ );
+ std::thread::sleep(Duration::from_millis(200));
+ }
+}
+
+fn osascript(script: &str) -> String {
+ let output = Command::new("osascript")
+ .args(["-e", script])
+ .output()
+ .expect("run osascript");
+ assert!(
+ output.status.success(),
+ "osascript failed: {}",
+ String::from_utf8_lossy(&output.stderr).trim()
+ );
+ String::from_utf8_lossy(&output.stdout).trim().to_owned()
+}
+
+fn frontmost_bundle_id() -> String {
+ osascript(
+ r#"tell application "System Events" to bundle identifier of first application process whose frontmost is true"#,
+ )
+}
+
+fn wait_for_frontmost(bundle_id: &str) -> bool {
+ let deadline = Instant::now() + Duration::from_secs(2);
+ while Instant::now() < deadline {
+ if frontmost_bundle_id() == bundle_id {
+ return true;
+ }
+ std::thread::sleep(Duration::from_millis(100));
+ }
+ false
+}
+
+fn ensure_frontmost(bundle_id: &str) {
+ for _ in 0..3 {
+ osascript(&format!(r#"tell application id "{bundle_id}" to activate"#));
+ if wait_for_frontmost(bundle_id) {
+ return;
+ }
+ std::thread::sleep(Duration::from_millis(500));
+ }
+ panic!("could not make {bundle_id} frontmost for the TextEdit foreground check");
+}
+
+fn focused_textedit_window_title() -> String {
+ osascript(
+ r#"tell application "System Events" to tell process "TextEdit" to get name of value of attribute "AXFocusedWindow""#,
+ )
+}
+
+/// Open a second TextEdit document. TextEdit focuses the new document, so the
+/// first one becomes an unfocused sibling of a frontmost app.
+fn open_sibling_document(text: &str) {
+ // Unsaved document names need not match their AX window titles (for
+ // example, TextEdit can append an edited suffix). Use a stable file name.
+ let sibling_title = format!("{text}.txt");
+ let path = std::env::temp_dir().join(&sibling_title);
+ std::fs::write(&path, text).expect("write sibling document");
+ let opened = Command::new("open")
+ .args(["-b", TEXTEDIT_BUNDLE])
+ .arg(&path)
+ .status()
+ .expect("open sibling document");
+ assert!(opened.success(), "could not open sibling document");
+ ensure_frontmost(TEXTEDIT_BUNDLE);
+ let deadline = Instant::now() + Duration::from_secs(5);
+ while focused_textedit_window_title() != sibling_title {
+ assert!(
+ Instant::now() < deadline,
+ "the sibling document never took the keyboard focus; focused window is {:?}",
+ focused_textedit_window_title()
+ );
+ std::thread::sleep(Duration::from_millis(200));
+ }
+ std::fs::remove_file(path).expect("remove sibling document fixture");
+}
+
// ── End-to-end ladder behavior (interactive; needs a GUI session) ────────────
/// On a NATIVE Cocoa field (TextEdit), `delivery_mode:"background"` lands via the
@@ -16,24 +235,11 @@
#[test]
#[ignore]
fn background_type_on_native_cocoa_is_ax_verified() {
- use cua_driver_testkit::e2e::{
- execute_case, recording_evidence, CaseSpec, Delivery, DriverRoute, Observation, OracleKind,
- Scope, Targeting,
- };
use cua_driver_testkit::observer::TargetWindow;
use cua_driver_testkit::sentinel::run_with_background_oracles;
- use cua_driver_testkit::{Driver, McpDriver};
let cell_id = "macos-textedit-type-text-ax-background";
- // TextEdit may restore an earlier unsaved document, so the read-back
- // oracle needs text that no earlier run could have left behind.
- let marker = format!(
- "ladder{}",
- std::time::SystemTime::now()
- .duration_since(std::time::UNIX_EPOCH)
- .expect("system clock after the Unix epoch")
- .as_nanos()
- );
+ let marker = unique_marker("ladder");
let case = CaseSpec::delivered(
cell_id,
"textedit",
@@ -55,64 +261,20 @@ fn background_type_on_native_cocoa_is_ax_verified() {
let mut driver = McpDriver::spawn_macos_daemon_proxy_named(cell_id)
.expect("start installed macOS daemon proxy");
*evidence = recording_evidence(driver.recording_dir());
-
- // Launch TextEdit and open a blank document.
- let launch = driver.call(
- "launch_app",
- serde_json::json!({ "bundle_id": "com.apple.TextEdit" }),
- );
- assert!(
- !launch.is_error(),
- "could not launch TextEdit: {}",
- launch.text()
- );
- let pid = launch.structured()["pid"].as_i64().expect("TextEdit pid");
- let mut windows = launch.structured()["windows"]
- .as_array()
- .cloned()
- .unwrap_or_default();
- assert!(!windows.is_empty(), "TextEdit opened no window");
-
- // TextEdit can retain an off-screen restoration window alongside the
- // visible blank document. WindowServer does not guarantee their order,
- // so prefer visible windows and select the one that actually exposes
- // the editor instead of assuming `windows[0]` is the document.
- windows.sort_by_key(|window| !window["is_on_screen"].as_bool().unwrap_or(false));
- let (wid, element_token) = windows
- .iter()
- .filter_map(|window| window["window_id"].as_u64())
- .find_map(|window_id| {
- let state = driver.call(
- "get_window_state",
- serde_json::json!({
- "pid": pid,
- "window_id": window_id,
- "capture_mode": "ax"
- }),
- );
- state.structured()["elements"]
- .as_array()
- .and_then(|elements| {
- elements
- .iter()
- .find(|element| element["role"] == "AXTextArea")
- .and_then(|element| element["element_token"].as_str())
- })
- .map(|element_token| (window_id, element_token.to_owned()))
- })
- .expect("TextEdit opened no window containing an AXTextArea");
+ let document = launch_textedit_document(&mut driver);
let (typed, mut passed) = run_with_background_oracles(
&mut driver,
TargetWindow {
- pid: pid as u32,
- native_id: wid,
+ pid: document.pid as u32,
+ native_id: document.window_id,
},
|driver| {
driver.call(
"type_text",
serde_json::json!({
- "pid": pid, "window_id": wid, "element_token": element_token,
+ "pid": document.pid, "window_id": document.window_id,
+ "element_token": document.element_token,
"text": marker, "delivery_mode": "background"
}),
)
@@ -138,35 +300,182 @@ fn background_type_on_native_cocoa_is_ax_verified() {
"confirmed actions must expose publishable evidence: {}",
typed.text()
);
- // The driver's own read-back is not the oracle. Take a fresh snapshot
- // and require the editor to hold the typed text.
- let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5);
- loop {
- let state = driver.call(
- "get_window_state",
- serde_json::json!({ "pid": pid, "window_id": wid, "capture_mode": "ax" }),
- );
- let landed = state.structured()["elements"]
- .as_array()
- .is_some_and(|elements| {
- elements.iter().any(|element| {
- element["role"] == "AXTextArea"
- && element["value"]
- .as_str()
- .is_some_and(|value| value.contains(&marker))
- })
- });
- if landed {
- break;
- }
- assert!(
- std::time::Instant::now() < deadline,
- "TextEdit's AXTextArea never held {marker:?} after a confirmed write:\n{}",
- state.text()
- );
- std::thread::sleep(std::time::Duration::from_millis(200));
- }
+ wait_for_editor_text(
+ &mut driver,
+ &document,
+ |value| value.contains(&marker),
+ &format!("held {marker:?} after a confirmed write"),
+ );
passed.push(OracleKind::AxState);
Observation::delivered(passed, Default::default())
});
}
+
+/// A window-only `delivery_mode:"foreground"` chord reaches the TextEdit
+/// document while Finder holds the front, and Finder gets the front back. The
+/// selection is proven by the next AX insert replacing the whole document
+/// rather than appending to it.
+#[test]
+#[ignore]
+fn foreground_window_hotkey_selects_all_behind_another_app() {
+ let cell_id = "macos-textedit-hotkey-window-foreground";
+ let seed = unique_marker("seed");
+ let replacement = unique_marker("replaced");
+ let case = CaseSpec::delivered(
+ cell_id,
+ "textedit",
+ "appkit",
+ "hotkey",
+ Targeting::NotApplicable,
+ Delivery::Foreground,
+ Scope::Window,
+ DriverRoute::MacosCgEventHid,
+ vec![OracleKind::AxState, OracleKind::Focus],
+ );
+ execute_case(case, |evidence| {
+ let mut driver = McpDriver::spawn_macos_daemon_proxy_named(cell_id)
+ .expect("start installed macOS daemon proxy");
+ *evidence = recording_evidence(driver.recording_dir());
+ let document = launch_textedit_document(&mut driver);
+
+ let seeded = driver.call(
+ "type_text",
+ serde_json::json!({
+ "pid": document.pid, "window_id": document.window_id,
+ "element_token": document.element_token,
+ "text": seed, "delivery_mode": "background"
+ }),
+ );
+ assert!(!seeded.is_error(), "seeding failed: {}", seeded.text());
+ let element_token = wait_for_editor_text(
+ &mut driver,
+ &document,
+ |value| value.contains(&seed),
+ &format!("held the seed {seed:?}"),
+ );
+
+ ensure_frontmost(FINDER_BUNDLE);
+ driver.start_behavior_recording();
+ let hotkey = driver.call(
+ "hotkey",
+ serde_json::json!({
+ "pid": document.pid, "window_id": document.window_id,
+ "keys": ["cmd", "a"], "delivery_mode": "foreground"
+ }),
+ );
+ assert!(
+ !hotkey.is_error(),
+ "foreground hotkey failed: {}",
+ hotkey.text()
+ );
+ assert!(
+ wait_for_frontmost(FINDER_BUNDLE),
+ "Finder did not get the front back after the hotkey; frontmost is {}",
+ frontmost_bundle_id()
+ );
+
+ let typed = driver.call(
+ "type_text",
+ serde_json::json!({
+ "pid": document.pid, "window_id": document.window_id,
+ "element_token": element_token,
+ "text": replacement, "delivery_mode": "background"
+ }),
+ );
+ assert!(!typed.is_error(), "type_text errored: {}", typed.text());
+ wait_for_editor_text(
+ &mut driver,
+ &document,
+ |value| value.contains(&replacement) && !value.contains(&seed),
+ &format!("replaced {seed:?} with {replacement:?}: cmd+a did not select the document"),
+ );
+ Observation::delivered(
+ vec![OracleKind::AxState, OracleKind::Focus],
+ Default::default(),
+ )
+ });
+}
+
+/// A window-only `delivery_mode:"foreground"` chord addressed at a document
+/// of the frontmost app reaches that document even while a sibling document
+/// holds the keyboard focus.
+#[test]
+#[ignore]
+fn foreground_window_hotkey_reaches_an_unfocused_sibling_of_the_frontmost_app() {
+ let cell_id = "macos-textedit-hotkey-window-foreground-sibling";
+ let seed = unique_marker("seed");
+ let replacement = unique_marker("replaced");
+ let case = CaseSpec::delivered(
+ cell_id,
+ "textedit",
+ "appkit",
+ "hotkey",
+ Targeting::NotApplicable,
+ Delivery::Foreground,
+ Scope::Window,
+ DriverRoute::MacosCgEventHid,
+ vec![OracleKind::AxState, OracleKind::Focus],
+ );
+ execute_case(case, |evidence| {
+ let mut driver = McpDriver::spawn_macos_daemon_proxy_named(cell_id)
+ .expect("start installed macOS daemon proxy");
+ *evidence = recording_evidence(driver.recording_dir());
+ let document = launch_textedit_document(&mut driver);
+
+ let seeded = driver.call(
+ "type_text",
+ serde_json::json!({
+ "pid": document.pid, "window_id": document.window_id,
+ "element_token": document.element_token,
+ "text": seed, "delivery_mode": "background"
+ }),
+ );
+ assert!(!seeded.is_error(), "seeding failed: {}", seeded.text());
+ let element_token = wait_for_editor_text(
+ &mut driver,
+ &document,
+ |value| value.contains(&seed),
+ &format!("held the seed {seed:?}"),
+ );
+
+ open_sibling_document(&unique_marker("sibling"));
+ driver.start_behavior_recording();
+ let hotkey = driver.call(
+ "hotkey",
+ serde_json::json!({
+ "pid": document.pid, "window_id": document.window_id,
+ "keys": ["cmd", "a"], "delivery_mode": "foreground"
+ }),
+ );
+ assert!(
+ !hotkey.is_error(),
+ "foreground hotkey failed: {}",
+ hotkey.text()
+ );
+ assert!(
+ wait_for_frontmost(TEXTEDIT_BUNDLE),
+ "TextEdit lost the front after a chord addressed at its own window; frontmost is {}",
+ frontmost_bundle_id()
+ );
+
+ let typed = driver.call(
+ "type_text",
+ serde_json::json!({
+ "pid": document.pid, "window_id": document.window_id,
+ "element_token": element_token,
+ "text": replacement, "delivery_mode": "background"
+ }),
+ );
+ assert!(!typed.is_error(), "type_text errored: {}", typed.text());
+ wait_for_editor_text(
+ &mut driver,
+ &document,
+ |value| value.contains(&replacement) && !value.contains(&seed),
+ &format!("replaced {seed:?} with {replacement:?}: cmd+a did not select the document"),
+ );
+ Observation::delivered(
+ vec![OracleKind::AxState, OracleKind::Focus],
+ Default::default(),
+ )
+ });
+}
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/ax/bindings.rs b/libs/cua-driver/rust/crates/platform-macos/src/ax/bindings.rs
index 91ba5def61..ca39f141a3 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/ax/bindings.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/ax/bindings.rs
@@ -581,6 +581,37 @@ pub fn raise_focused_window_of_pid(pid: i32) -> bool {
}
}
+/// Raise exactly the window of `pid` whose CGWindowID is `window_id`, and ask
+/// the app to make it main and focused. Within an already frontmost app this
+/// is what moves the key window between siblings; a front-process change on
+/// its own leaves AppKit's key window where it was. No other window of the
+/// process is addressed.
+pub fn raise_exact_window_of_pid(pid: i32, window_id: u32) -> bool {
+ unsafe {
+ let app = AXUIElementCreateApplication(pid);
+ if app.is_null() {
+ return false;
+ }
+ let mut target = None;
+ for window in copy_ax_windows(app) {
+ if target.is_none() && ax_get_window_id(window) == Some(window_id) {
+ target = Some(window);
+ } else {
+ CFRelease(window as CFTypeRef);
+ }
+ }
+ CFRelease(app as CFTypeRef);
+ let Some(target) = target else {
+ return false;
+ };
+ let raised = perform_action(target, "AXRaise") == kAXErrorSuccess;
+ let main = set_bool_attr_true(target, "AXMain") == kAXErrorSuccess;
+ let focused = set_bool_attr_true(target, "AXFocused") == kAXErrorSuccess;
+ CFRelease(target as CFTypeRef);
+ raised || main || focused
+ }
+}
+
/// The CGWindowID of the window that holds `pid`'s focused UI element
/// (`AXFocusedUIElement` → `AXWindow`). `None` when the app reports no focused
/// element: a LibreOffice that has never been active names a focused window
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/input/skylight.rs b/libs/cua-driver/rust/crates/platform-macos/src/input/skylight.rs
index 70e87b39bd..136e0c977f 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/input/skylight.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/input/skylight.rs
@@ -1286,20 +1286,14 @@ fn foreground_turn() -> ForegroundTurn {
/// seen the background rungs fail (clicks) or the field is unverifiable +
/// focus-sensitive (Catalyst typing).
///
-/// ## Why this does not delegate to [`with_menu_shortcut_activation`]
+/// ## Why the body waits for the activation
///
-/// It used to. That helper posts `set_front` and calls `action` immediately,
-/// which is correct for its own purpose: NSMenu key dispatch only needs the key
-/// event *enqueued* in the target's run-loop queue, so first-responder identity
-/// is irrelevant and the sub-millisecond front → act → restore is a feature.
-///
-/// Input delivery has the opposite requirement. `set_front` is asynchronous, so
-/// running the body straight away means the body's `AXFocused` write races
-/// AppKit's own activation. AppKit wins: when activation completes it installs
-/// the window's remembered first responder and clobbers the write. The
-/// keystrokes then land wherever the app chose — for a Catalyst app such as
-/// WhatsApp, the message list rather than the composer, which silently scrolls
-/// the transcript instead of typing.
+/// `set_front` is asynchronous, so running the body straight away means the
+/// body's `AXFocused` write races AppKit's own activation. AppKit wins: when
+/// activation completes it installs the window's remembered first responder
+/// and clobbers the write. The keystrokes then land wherever the app chose —
+/// for a Catalyst app such as WhatsApp, the message list rather than the
+/// composer, which silently scrolls the transcript instead of typing.
///
/// So this waits for the activation to be observable before running `body`, the
/// same ordering [`with_foreground_hid_activation`] already relies on. The wait
@@ -1435,12 +1429,11 @@ fn focus_failure_hint_from(
/// Activate an exact target window for a global HID keyboard action.
///
-/// Unlike [`with_menu_shortcut_activation`], this helper must not run `action`
-/// when the private foreground SPI is unavailable: a global HID event has no
-/// pid addressing and would otherwise land in whichever application is
-/// currently frontmost. The short settles keep the target frontmost until
-/// WindowServer has routed both sides of the key chord, then restore the prior
-/// process even when the action fails.
+/// This helper must not run `action` when the private foreground SPI is
+/// unavailable: a global HID event has no pid addressing and would otherwise
+/// land in whichever application is currently frontmost. The short settles
+/// keep the target frontmost until WindowServer has routed both sides of the
+/// key chord, then restore the prior process even when the action fails.
pub fn with_foreground_hid_activation(
target_pid: libc::pid_t,
target_wid: u32,
@@ -1476,11 +1469,13 @@ pub fn with_foreground_hid_activation(
}
make_exact_window_key(target_pid, target_wid);
+ crate::ax::bindings::raise_exact_window_of_pid(target_pid, target_wid);
// A slow app (LibreOffice) can take longer than one wait to install its
// key window: ask once more before giving up.
if !await_window_focused(target_pid, target_wid) {
unsafe { set_front(target_psn.as_ptr() as *const c_void, target_wid, 0x400) };
make_exact_window_key(target_pid, target_wid);
+ crate::ax::bindings::raise_exact_window_of_pid(target_pid, target_wid);
}
if !await_window_focused(target_pid, target_wid) {
if prev_ok {
@@ -1514,58 +1509,6 @@ fn preserves_exact_existing_focus(
&& focused_window_id == Some(target_window_id)
}
-/// Activate `target_pid`'s window `target_wid` for NSMenu key dispatch, run `action`,
-/// then immediately restore the prior frontmost process.
-///
-/// The entire activate → action → restore sequence is < 1 ms — a 5 ms UX monitor
-/// never observes the intermediate frontmost state. NSMenu still fires because the
-/// key event is already enqueued in the target's run-loop queue before we restore.
-///
-/// Returns `Ok(true)` when activation succeeded, `Ok(false)` when SPIs unavailable.
-pub fn with_menu_shortcut_activation(
- target_pid: libc::pid_t,
- target_wid: u32,
- action: impl FnOnce() -> anyhow::Result<()>,
-) -> anyhow::Result<bool> {
- let _turn = foreground_turn();
- let set_front = match set_front_process_fn() {
- Some(f) => f,
- None => {
- // SPIs unavailable — run action anyway without activation.
- action()?;
- return Ok(false);
- }
- };
-
- // Capture prior frontmost PSN.
- let mut prev_psn = [0u8; 8];
- let prev_ok = get_front_process_fn()
- .map(|f| unsafe { f(prev_psn.as_mut_ptr() as *mut c_void) } == 0)
- .unwrap_or(false);
-
- // Resolve target PSN.
- let mut target_psn = [0u8; 8];
- let target_ok = get_process_psn_for_window(target_wid, target_pid, &mut target_psn);
- if !target_ok {
- action()?;
- return Ok(false);
- }
-
- // Make target WindowServer-frontmost (kCPSNoWindows = 0x400).
- unsafe { set_front(target_psn.as_ptr() as *const c_void, target_wid, 0x400) };
-
- // Run action then restore — even if action fails.
- let result = action();
-
- // Restore prior frontmost (windowID=0, options=0x400).
- if prev_ok {
- unsafe { set_front(prev_psn.as_ptr() as *const c_void, 0, 0x400) };
- }
-
- result?;
- Ok(true)
-}
-
#[cfg(test)]
mod tests {
use super::{
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/bring_to_front.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/bring_to_front.rs
index 222de0c88d..8ba36c09a8 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/bring_to_front.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/bring_to_front.rs
@@ -9,7 +9,6 @@
use std::time::{Duration, Instant};
use async_trait::async_trait;
-use core_foundation::base::{CFRelease, CFTypeRef};
use core_graphics::display::CGDisplay;
use cua_driver_core::{
protocol::ToolResult,
@@ -18,10 +17,7 @@ use cua_driver_core::{
use objc2_app_kit::{NSApplicationActivationOptions, NSRunningApplication};
use serde_json::{json, Value};
-use crate::ax::bindings::{
- ax_get_window_id, copy_ax_windows, perform_action, set_bool_attr_true,
- AXUIElementCreateApplication,
-};
+use crate::ax::bindings::raise_exact_window_of_pid;
use crate::windows::{WindowBounds, WindowInfo};
pub struct BringToFrontTool;
@@ -252,7 +248,7 @@ pub(crate) fn activate_exact_window_blocking(pid: i32, window_id: u32) -> bool {
unsafe {
app.activateWithOptions(NSApplicationActivationOptions::NSApplicationActivateAllWindows)
};
- raise_exact_ax_window(pid, window_id);
+ raise_exact_window_of_pid(pid, window_id);
wait_for_exact_window(pid, window_id).exact_postcondition(pid, window_id)
}
@@ -268,34 +264,6 @@ pub(crate) fn reactivate_application(pid: i32) -> bool {
unsafe { app.activateWithOptions(NSApplicationActivationOptions::empty()) }
}
-/// Best-effort completion of the one exact-window request. This addresses only
-/// the requested AX window; it never orders every window owned by the process.
-fn raise_exact_ax_window(pid: i32, window_id: u32) -> bool {
- unsafe {
- let app = AXUIElementCreateApplication(pid);
- if app.is_null() {
- return false;
- }
- let mut target = None;
- for window in copy_ax_windows(app) {
- if target.is_none() && ax_get_window_id(window) == Some(window_id) {
- target = Some(window);
- } else {
- CFRelease(window as CFTypeRef);
- }
- }
- CFRelease(app as CFTypeRef);
- let Some(target) = target else {
- return false;
- };
- let raised = perform_action(target, "AXRaise") == 0;
- let main = set_bool_attr_true(target, "AXMain") == 0;
- let focused = set_bool_attr_true(target, "AXFocused") == 0;
- CFRelease(target as CFTypeRef);
- raised || main || focused
- }
-}
-
fn exact_result(
pid: i32,
window_id: u32,
@@ -470,7 +438,7 @@ impl Tool for BringToFrontTool {
NSApplicationActivationOptions::NSApplicationActivateAllWindows,
)
};
- let ax_window_requested = raise_exact_ax_window(pid, window_id);
+ let ax_window_requested = raise_exact_window_of_pid(pid, window_id);
let path = match (
skylight_process_accepted,
skylight_exact_accepted,
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/hotkey.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/hotkey.rs
index fa30e286b7..2bf28e0582 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/hotkey.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/hotkey.rs
@@ -572,13 +572,7 @@ impl Tool for HotkeyTool {
)?;
Ok(())
}
- // Screen Sharing is an input forwarder: modifier flags
- // on a PID-routed base-key event are not relayed to the
- // guest. Emit the physical modifier down/base/up
- // sequence through the guarded foreground HID path.
- (true, false, Some(wid), None)
- if crate::input::keyboard::is_screen_sharing_pid(pid) =>
- {
+ (true, false, Some(wid), None) => {
crate::input::skylight::with_foreground_hid_activation(
pid as libc::pid_t,
wid,
@@ -586,16 +580,6 @@ impl Tool for HotkeyTool {
)?;
Ok(())
}
- // foreground rung: briefly front the window so NSMenu key
- // equivalents dispatch, then restore prior frontmost.
- (true, false, Some(wid), None) => {
- crate::input::skylight::with_menu_shortcut_activation(
- pid as libc::pid_t,
- wid,
- || crate::input::keyboard::hotkey_no_auth(pid, &key, &m),
- )?;
- Ok(())
- }
// background (default): auth-envelope post to the pid, no
// raise — even when window_id was supplied for targeting.
(false, false, _, Some(ptr)) => {
diff --git a/scripts/ci/macos/run-rust-e2e.sh b/scripts/ci/macos/run-rust-e2e.sh
index dc6d8cda90..c2e9f0928e 100755
--- a/scripts/ci/macos/run-rust-e2e.sh
+++ b/scripts/ci/macos/run-rust-e2e.sh
@@ -414,6 +414,12 @@ EOF
run_test installed-app-textedit cargo test -p cua-driver-e2e --test installed_app_textedit_macos_test -- \
--ignored --exact background_type_on_native_cocoa_is_ax_verified \
--nocapture --test-threads=1
+ run_test installed-app-textedit-hotkey cargo test -p cua-driver-e2e --test installed_app_textedit_macos_test -- \
+ --ignored --exact foreground_window_hotkey_selects_all_behind_another_app \
+ --nocapture --test-threads=1
+ run_test installed-app-textedit-hotkey-sibling cargo test -p cua-driver-e2e --test installed_app_textedit_macos_test -- \
+ --ignored --exact foreground_window_hotkey_reaches_an_unfocused_sibling_of_the_frontmost_app \
+ --nocapture --test-threads=1
# These rows assert through System Events and CoreGraphics oracles rather
# than typed matrix results, so they must not leave unowned trajectories.
run_test bring-to-front env -u CUA_E2E_RECORDINGS_ROOT \