Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
fix(cua-driver): deliver window-only foreground hotkeys through the HID activation guard
injaneitya4b5fec16a25
diff --git a/libs/cua-driver/rust/crates/cua-driver-e2e/tests/README.md b/libs/cua-driver/rust/crates/cua-driver-e2e/tests/README.mdindex 18e422129d..d5e2edc270 100644--- a/libs/cua-driver/rust/crates/cua-driver-e2e/tests/README.md+++ b/libs/cua-driver/rust/crates/cua-driver-e2e/tests/README.md@@ -146,7 +146,9 @@ desktop state outside the repo-local fixtures:- `installed_app_launch_macos_test.rs`: macOS Calculator/TextEdit launch focuschecks. Requires a logged-in GUI session and usable System Events scripting.- `installed_app_textedit_macos_test.rs`: real TextEdit background AX write and- verification. Requires a logged-in GUI session and Accessibility permission.+ verification, plus the window-only foreground chord delivered while Finder+ holds the front or while a sibling document holds the keyboard focus.+ Requires a logged-in GUI session and Accessibility permission.- `standalone_browser_behavior_test.rs`: installed Chrome and Edge browser-tooltargeting, mutation, frame, tab, ambiguity, and isolated-profile rows. Theharness owns a fresh browser profile and creates adversarial tabs and windowsdiff --git a/libs/cua-driver/rust/crates/cua-driver-e2e/tests/installed_app_textedit_macos_test.rs b/libs/cua-driver/rust/crates/cua-driver-e2e/tests/installed_app_textedit_macos_test.rsindex 94ff8d7445..f8fc449efe 100644--- a/libs/cua-driver/rust/crates/cua-driver-e2e/tests/installed_app_textedit_macos_test.rs+++ b/libs/cua-driver/rust/crates/cua-driver-e2e/tests/installed_app_textedit_macos_test.rs@@ -1,13 +1,232 @@-//! TextEdit background-delivery integration check for macOS.+//! TextEdit delivery integration checks for macOS.//!//! Covers the accessibility route and confirmed value read-back on a real-//! Cocoa app, then reads the editor back through a fresh snapshot.+//! Cocoa app, then reads the editor back through a fresh snapshot, and the+//! window-only foreground chord while another app holds the front or while a+//! sibling document of the same app holds the keyboard focus.//!-//! The schema contract lives in `protocol_schema_test.rs`. This installed-app-//! check runs in the canonical logged-in macOS desktop lane.+//! The schema contract lives in `protocol_schema_test.rs`. These installed-app+//! checks run in the canonical logged-in macOS desktop lane.#![cfg(target_os = "macos")]+use std::process::Command;+use std::time::{Duration, Instant};++use cua_driver_testkit::e2e::{+ execute_case, recording_evidence, CaseSpec, Delivery, DriverRoute, Observation, OracleKind,+ Scope, Targeting,+};+use cua_driver_testkit::{Driver, McpDriver};++const FINDER_BUNDLE: &str = "com.apple.finder";+const TEXTEDIT_BUNDLE: &str = "com.apple.TextEdit";++struct TextEditDocument {+ pid: i64,+ window_id: u64,+ element_token: String,+}++/// TextEdit may restore an earlier unsaved document, so read-back oracles need+/// text that no earlier run could have left behind.+fn unique_marker(prefix: &str) -> String {+ format!(+ "{prefix}{}",+ std::time::SystemTime::now()+ .duration_since(std::time::UNIX_EPOCH)+ .expect("system clock after the Unix epoch")+ .as_nanos()+ )+}++/// Launch TextEdit and locate the on-screen window that exposes the editor.+///+/// TextEdit can retain an off-screen restoration window alongside the visible+/// blank document, and the document window can appear a moment after launch.+/// WindowServer does not guarantee their order, so prefer visible windows and+/// select the one that actually exposes the editor instead of assuming+/// `windows[0]` is the document.+fn launch_textedit_document(driver: &mut McpDriver) -> TextEditDocument {+ let launch = driver.call(+ "launch_app",+ serde_json::json!({ "bundle_id": TEXTEDIT_BUNDLE }),+ );+ assert!(+ !launch.is_error(),+ "could not launch TextEdit: {}",+ launch.text()+ );+ let pid = launch.structured()["pid"].as_i64().expect("TextEdit pid");+ let mut windows = launch.structured()["windows"]+ .as_array()+ .cloned()+ .unwrap_or_default();+ let deadline = Instant::now() + Duration::from_secs(5);+ loop {+ windows.sort_by_key(|window| !window["is_on_screen"].as_bool().unwrap_or(false));+ let editor = windows+ .iter()+ .filter_map(|window| window["window_id"].as_u64())+ .find_map(|window_id| {+ let state = driver.call(+ "get_window_state",+ serde_json::json!({+ "pid": pid,+ "window_id": window_id,+ "capture_mode": "ax"+ }),+ );+ state.structured()["elements"]+ .as_array()+ .and_then(|elements| {+ elements+ .iter()+ .find(|element| element["role"] == "AXTextArea")+ .and_then(|element| element["element_token"].as_str())+ })+ .map(|element_token| (window_id, element_token.to_owned()))+ });+ if let Some((window_id, element_token)) = editor {+ return TextEditDocument {+ pid,+ window_id,+ element_token,+ };+ }+ assert!(+ Instant::now() < deadline,+ "TextEdit opened no window containing an AXTextArea: {windows:?}"+ );+ std::thread::sleep(Duration::from_millis(250));+ let listed = driver.call("list_windows", serde_json::json!({}));+ windows = listed.structured()["windows"]+ .as_array()+ .map(|listed| {+ listed+ .iter()+ .filter(|window| window["pid"].as_i64() == Some(pid))+ .cloned()+ .collect()+ })+ .unwrap_or_default();+ }+}++/// The driver's own read-back is not the oracle. Take fresh snapshots until the+/// editor holds the expected text, and return the editor's token from that+/// snapshot since each snapshot retires the earlier tokens.+fn wait_for_editor_text(+ driver: &mut McpDriver,+ document: &TextEditDocument,+ accept: impl Fn(&str) -> bool,+ expectation: &str,+) -> String {+ let deadline = Instant::now() + Duration::from_secs(5);+ loop {+ let state = driver.call(+ "get_window_state",+ serde_json::json!({+ "pid": document.pid,+ "window_id": document.window_id,+ "capture_mode": "ax"+ }),+ );+ let landed = state.structured()["elements"]+ .as_array()+ .and_then(|elements| {+ elements.iter().find(|element| {+ element["role"] == "AXTextArea"+ && element["value"].as_str().is_some_and(&accept)+ })+ })+ .and_then(|element| element["element_token"].as_str());+ if let Some(element_token) = landed {+ return element_token.to_owned();+ }+ assert!(+ Instant::now() < deadline,+ "TextEdit's AXTextArea never {expectation}:\n{}",+ state.text()+ );+ std::thread::sleep(Duration::from_millis(200));+ }+}++fn osascript(script: &str) -> String {+ let output = Command::new("osascript")+ .args(["-e", script])+ .output()+ .expect("run osascript");+ assert!(+ output.status.success(),+ "osascript failed: {}",+ String::from_utf8_lossy(&output.stderr).trim()+ );+ String::from_utf8_lossy(&output.stdout).trim().to_owned()+}++fn frontmost_bundle_id() -> String {+ osascript(+ r#"tell application "System Events" to bundle identifier of first application process whose frontmost is true"#,+ )+}++fn wait_for_frontmost(bundle_id: &str) -> bool {+ let deadline = Instant::now() + Duration::from_secs(2);+ while Instant::now() < deadline {+ if frontmost_bundle_id() == bundle_id {+ return true;+ }+ std::thread::sleep(Duration::from_millis(100));+ }+ false+}++fn ensure_frontmost(bundle_id: &str) {+ for _ in 0..3 {+ osascript(&format!(r#"tell application id "{bundle_id}" to activate"#));+ if wait_for_frontmost(bundle_id) {+ return;+ }+ std::thread::sleep(Duration::from_millis(500));+ }+ panic!("could not make {bundle_id} frontmost for the TextEdit foreground check");+}++fn focused_textedit_window_title() -> String {+ osascript(+ r#"tell application "System Events" to tell process "TextEdit" to get name of value of attribute "AXFocusedWindow""#,+ )+}++/// Open a second TextEdit document. TextEdit focuses the new document, so the+/// first one becomes an unfocused sibling of a frontmost app.+fn open_sibling_document(text: &str) {+ // Unsaved document names need not match their AX window titles (for+ // example, TextEdit can append an edited suffix). Use a stable file name.+ let sibling_title = format!("{text}.txt");+ let path = std::env::temp_dir().join(&sibling_title);+ std::fs::write(&path, text).expect("write sibling document");+ let opened = Command::new("open")+ .args(["-b", TEXTEDIT_BUNDLE])+ .arg(&path)+ .status()+ .expect("open sibling document");+ assert!(opened.success(), "could not open sibling document");+ ensure_frontmost(TEXTEDIT_BUNDLE);+ let deadline = Instant::now() + Duration::from_secs(5);+ while focused_textedit_window_title() != sibling_title {+ assert!(+ Instant::now() < deadline,+ "the sibling document never took the keyboard focus; focused window is {:?}",+ focused_textedit_window_title()+ );+ std::thread::sleep(Duration::from_millis(200));+ }+ std::fs::remove_file(path).expect("remove sibling document fixture");+}+// ── End-to-end ladder behavior (interactive; needs a GUI session) ────────────/// On a NATIVE Cocoa field (TextEdit), `delivery_mode:"background"` lands via the@@ -16,24 +235,11 @@#[test]#[ignore]fn background_type_on_native_cocoa_is_ax_verified() {- use cua_driver_testkit::e2e::{- execute_case, recording_evidence, CaseSpec, Delivery, DriverRoute, Observation, OracleKind,- Scope, Targeting,- };use cua_driver_testkit::observer::TargetWindow;use cua_driver_testkit::sentinel::run_with_background_oracles;- use cua_driver_testkit::{Driver, McpDriver};let cell_id = "macos-textedit-type-text-ax-background";- // TextEdit may restore an earlier unsaved document, so the read-back- // oracle needs text that no earlier run could have left behind.- let marker = format!(- "ladder{}",- std::time::SystemTime::now()- .duration_since(std::time::UNIX_EPOCH)- .expect("system clock after the Unix epoch")- .as_nanos()- );+ let marker = unique_marker("ladder");let case = CaseSpec::delivered(cell_id,"textedit",@@ -55,64 +261,20 @@ fn background_type_on_native_cocoa_is_ax_verified() {let mut driver = McpDriver::spawn_macos_daemon_proxy_named(cell_id).expect("start installed macOS daemon proxy");*evidence = recording_evidence(driver.recording_dir());-- // Launch TextEdit and open a blank document.- let launch = driver.call(- "launch_app",- serde_json::json!({ "bundle_id": "com.apple.TextEdit" }),- );- assert!(- !launch.is_error(),- "could not launch TextEdit: {}",- launch.text()- );- let pid = launch.structured()["pid"].as_i64().expect("TextEdit pid");- let mut windows = launch.structured()["windows"]- .as_array()- .cloned()- .unwrap_or_default();- assert!(!windows.is_empty(), "TextEdit opened no window");-- // TextEdit can retain an off-screen restoration window alongside the- // visible blank document. WindowServer does not guarantee their order,- // so prefer visible windows and select the one that actually exposes- // the editor instead of assuming `windows[0]` is the document.- windows.sort_by_key(|window| !window["is_on_screen"].as_bool().unwrap_or(false));- let (wid, element_token) = windows- .iter()- .filter_map(|window| window["window_id"].as_u64())- .find_map(|window_id| {- let state = driver.call(- "get_window_state",- serde_json::json!({- "pid": pid,- "window_id": window_id,- "capture_mode": "ax"- }),- );- state.structured()["elements"]- .as_array()- .and_then(|elements| {- elements- .iter()- .find(|element| element["role"] == "AXTextArea")- .and_then(|element| element["element_token"].as_str())- })- .map(|element_token| (window_id, element_token.to_owned()))- })- .expect("TextEdit opened no window containing an AXTextArea");+ let document = launch_textedit_document(&mut driver);let (typed, mut passed) = run_with_background_oracles(&mut driver,TargetWindow {- pid: pid as u32,- native_id: wid,+ pid: document.pid as u32,+ native_id: document.window_id,},|driver| {driver.call("type_text",serde_json::json!({- "pid": pid, "window_id": wid, "element_token": element_token,+ "pid": document.pid, "window_id": document.window_id,+ "element_token": document.element_token,"text": marker, "delivery_mode": "background"}),)@@ -138,35 +300,182 @@ fn background_type_on_native_cocoa_is_ax_verified() {"confirmed actions must expose publishable evidence: {}",typed.text());- // The driver's own read-back is not the oracle. Take a fresh snapshot- // and require the editor to hold the typed text.- let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5);- loop {- let state = driver.call(- "get_window_state",- serde_json::json!({ "pid": pid, "window_id": wid, "capture_mode": "ax" }),- );- let landed = state.structured()["elements"]- .as_array()- .is_some_and(|elements| {- elements.iter().any(|element| {- element["role"] == "AXTextArea"- && element["value"]- .as_str()- .is_some_and(|value| value.contains(&marker))- })- });- if landed {- break;- }- assert!(- std::time::Instant::now() < deadline,- "TextEdit's AXTextArea never held {marker:?} after a confirmed write:\n{}",- state.text()- );- std::thread::sleep(std::time::Duration::from_millis(200));- }+ wait_for_editor_text(+ &mut driver,+ &document,+ |value| value.contains(&marker),+ &format!("held {marker:?} after a confirmed write"),+ );passed.push(OracleKind::AxState);Observation::delivered(passed, Default::default())});}++/// A window-only `delivery_mode:"foreground"` chord reaches the TextEdit+/// document while Finder holds the front, and Finder gets the front back. The+/// selection is proven by the next AX insert replacing the whole document+/// rather than appending to it.+#[test]+#[ignore]+fn foreground_window_hotkey_selects_all_behind_another_app() {+ let cell_id = "macos-textedit-hotkey-window-foreground";+ let seed = unique_marker("seed");+ let replacement = unique_marker("replaced");+ let case = CaseSpec::delivered(+ cell_id,+ "textedit",+ "appkit",+ "hotkey",+ Targeting::NotApplicable,+ Delivery::Foreground,+ Scope::Window,+ DriverRoute::MacosCgEventHid,+ vec![OracleKind::AxState, OracleKind::Focus],+ );+ execute_case(case, |evidence| {+ let mut driver = McpDriver::spawn_macos_daemon_proxy_named(cell_id)+ .expect("start installed macOS daemon proxy");+ *evidence = recording_evidence(driver.recording_dir());+ let document = launch_textedit_document(&mut driver);++ let seeded = driver.call(+ "type_text",+ serde_json::json!({+ "pid": document.pid, "window_id": document.window_id,+ "element_token": document.element_token,+ "text": seed, "delivery_mode": "background"+ }),+ );+ assert!(!seeded.is_error(), "seeding failed: {}", seeded.text());+ let element_token = wait_for_editor_text(+ &mut driver,+ &document,+ |value| value.contains(&seed),+ &format!("held the seed {seed:?}"),+ );++ ensure_frontmost(FINDER_BUNDLE);+ driver.start_behavior_recording();+ let hotkey = driver.call(+ "hotkey",+ serde_json::json!({+ "pid": document.pid, "window_id": document.window_id,+ "keys": ["cmd", "a"], "delivery_mode": "foreground"+ }),+ );+ assert!(+ !hotkey.is_error(),+ "foreground hotkey failed: {}",+ hotkey.text()+ );+ assert!(+ wait_for_frontmost(FINDER_BUNDLE),+ "Finder did not get the front back after the hotkey; frontmost is {}",+ frontmost_bundle_id()+ );++ let typed = driver.call(+ "type_text",+ serde_json::json!({+ "pid": document.pid, "window_id": document.window_id,+ "element_token": element_token,+ "text": replacement, "delivery_mode": "background"+ }),+ );+ assert!(!typed.is_error(), "type_text errored: {}", typed.text());+ wait_for_editor_text(+ &mut driver,+ &document,+ |value| value.contains(&replacement) && !value.contains(&seed),+ &format!("replaced {seed:?} with {replacement:?}: cmd+a did not select the document"),+ );+ Observation::delivered(+ vec![OracleKind::AxState, OracleKind::Focus],+ Default::default(),+ )+ });+}++/// A window-only `delivery_mode:"foreground"` chord addressed at a document+/// of the frontmost app reaches that document even while a sibling document+/// holds the keyboard focus.+#[test]+#[ignore]+fn foreground_window_hotkey_reaches_an_unfocused_sibling_of_the_frontmost_app() {+ let cell_id = "macos-textedit-hotkey-window-foreground-sibling";+ let seed = unique_marker("seed");+ let replacement = unique_marker("replaced");+ let case = CaseSpec::delivered(+ cell_id,+ "textedit",+ "appkit",+ "hotkey",+ Targeting::NotApplicable,+ Delivery::Foreground,+ Scope::Window,+ DriverRoute::MacosCgEventHid,+ vec![OracleKind::AxState, OracleKind::Focus],+ );+ execute_case(case, |evidence| {+ let mut driver = McpDriver::spawn_macos_daemon_proxy_named(cell_id)+ .expect("start installed macOS daemon proxy");+ *evidence = recording_evidence(driver.recording_dir());+ let document = launch_textedit_document(&mut driver);++ let seeded = driver.call(+ "type_text",+ serde_json::json!({+ "pid": document.pid, "window_id": document.window_id,+ "element_token": document.element_token,+ "text": seed, "delivery_mode": "background"+ }),+ );+ assert!(!seeded.is_error(), "seeding failed: {}", seeded.text());+ let element_token = wait_for_editor_text(+ &mut driver,+ &document,+ |value| value.contains(&seed),+ &format!("held the seed {seed:?}"),+ );++ open_sibling_document(&unique_marker("sibling"));+ driver.start_behavior_recording();+ let hotkey = driver.call(+ "hotkey",+ serde_json::json!({+ "pid": document.pid, "window_id": document.window_id,+ "keys": ["cmd", "a"], "delivery_mode": "foreground"+ }),+ );+ assert!(+ !hotkey.is_error(),+ "foreground hotkey failed: {}",+ hotkey.text()+ );+ assert!(+ wait_for_frontmost(TEXTEDIT_BUNDLE),+ "TextEdit lost the front after a chord addressed at its own window; frontmost is {}",+ frontmost_bundle_id()+ );++ let typed = driver.call(+ "type_text",+ serde_json::json!({+ "pid": document.pid, "window_id": document.window_id,+ "element_token": element_token,+ "text": replacement, "delivery_mode": "background"+ }),+ );+ assert!(!typed.is_error(), "type_text errored: {}", typed.text());+ wait_for_editor_text(+ &mut driver,+ &document,+ |value| value.contains(&replacement) && !value.contains(&seed),+ &format!("replaced {seed:?} with {replacement:?}: cmd+a did not select the document"),+ );+ Observation::delivered(+ vec![OracleKind::AxState, OracleKind::Focus],+ Default::default(),+ )+ });+}diff --git a/libs/cua-driver/rust/crates/platform-macos/src/ax/bindings.rs b/libs/cua-driver/rust/crates/platform-macos/src/ax/bindings.rsindex 91ba5def61..ca39f141a3 100644--- a/libs/cua-driver/rust/crates/platform-macos/src/ax/bindings.rs+++ b/libs/cua-driver/rust/crates/platform-macos/src/ax/bindings.rs@@ -581,6 +581,37 @@ pub fn raise_focused_window_of_pid(pid: i32) -> bool {}}+/// Raise exactly the window of `pid` whose CGWindowID is `window_id`, and ask+/// the app to make it main and focused. Within an already frontmost app this+/// is what moves the key window between siblings; a front-process change on+/// its own leaves AppKit's key window where it was. No other window of the+/// process is addressed.+pub fn raise_exact_window_of_pid(pid: i32, window_id: u32) -> bool {+ unsafe {+ let app = AXUIElementCreateApplication(pid);+ if app.is_null() {+ return false;+ }+ let mut target = None;+ for window in copy_ax_windows(app) {+ if target.is_none() && ax_get_window_id(window) == Some(window_id) {+ target = Some(window);+ } else {+ CFRelease(window as CFTypeRef);+ }+ }+ CFRelease(app as CFTypeRef);+ let Some(target) = target else {+ return false;+ };+ let raised = perform_action(target, "AXRaise") == kAXErrorSuccess;+ let main = set_bool_attr_true(target, "AXMain") == kAXErrorSuccess;+ let focused = set_bool_attr_true(target, "AXFocused") == kAXErrorSuccess;+ CFRelease(target as CFTypeRef);+ raised || main || focused+ }+}+/// The CGWindowID of the window that holds `pid`'s focused UI element/// (`AXFocusedUIElement` → `AXWindow`). `None` when the app reports no focused/// element: a LibreOffice that has never been active names a focused windowdiff --git a/libs/cua-driver/rust/crates/platform-macos/src/input/skylight.rs b/libs/cua-driver/rust/crates/platform-macos/src/input/skylight.rsindex 70e87b39bd..136e0c977f 100644--- a/libs/cua-driver/rust/crates/platform-macos/src/input/skylight.rs+++ b/libs/cua-driver/rust/crates/platform-macos/src/input/skylight.rs@@ -1286,20 +1286,14 @@ fn foreground_turn() -> ForegroundTurn {/// seen the background rungs fail (clicks) or the field is unverifiable +/// focus-sensitive (Catalyst typing).///-/// ## Why this does not delegate to [`with_menu_shortcut_activation`]+/// ## Why the body waits for the activation///-/// It used to. That helper posts `set_front` and calls `action` immediately,-/// which is correct for its own purpose: NSMenu key dispatch only needs the key-/// event *enqueued* in the target's run-loop queue, so first-responder identity-/// is irrelevant and the sub-millisecond front → act → restore is a feature.-///-/// Input delivery has the opposite requirement. `set_front` is asynchronous, so-/// running the body straight away means the body's `AXFocused` write races-/// AppKit's own activation. AppKit wins: when activation completes it installs-/// the window's remembered first responder and clobbers the write. The-/// keystrokes then land wherever the app chose — for a Catalyst app such as-/// WhatsApp, the message list rather than the composer, which silently scrolls-/// the transcript instead of typing.+/// `set_front` is asynchronous, so running the body straight away means the+/// body's `AXFocused` write races AppKit's own activation. AppKit wins: when+/// activation completes it installs the window's remembered first responder+/// and clobbers the write. The keystrokes then land wherever the app chose —+/// for a Catalyst app such as WhatsApp, the message list rather than the+/// composer, which silently scrolls the transcript instead of typing.////// So this waits for the activation to be observable before running `body`, the/// same ordering [`with_foreground_hid_activation`] already relies on. The wait@@ -1435,12 +1429,11 @@ fn focus_failure_hint_from(/// Activate an exact target window for a global HID keyboard action.///-/// Unlike [`with_menu_shortcut_activation`], this helper must not run `action`-/// when the private foreground SPI is unavailable: a global HID event has no-/// pid addressing and would otherwise land in whichever application is-/// currently frontmost. The short settles keep the target frontmost until-/// WindowServer has routed both sides of the key chord, then restore the prior-/// process even when the action fails.+/// This helper must not run `action` when the private foreground SPI is+/// unavailable: a global HID event has no pid addressing and would otherwise+/// land in whichever application is currently frontmost. The short settles+/// keep the target frontmost until WindowServer has routed both sides of the+/// key chord, then restore the prior process even when the action fails.pub fn with_foreground_hid_activation(target_pid: libc::pid_t,target_wid: u32,@@ -1476,11 +1469,13 @@ pub fn with_foreground_hid_activation(}make_exact_window_key(target_pid, target_wid);+ crate::ax::bindings::raise_exact_window_of_pid(target_pid, target_wid);// A slow app (LibreOffice) can take longer than one wait to install its// key window: ask once more before giving up.if !await_window_focused(target_pid, target_wid) {unsafe { set_front(target_psn.as_ptr() as *const c_void, target_wid, 0x400) };make_exact_window_key(target_pid, target_wid);+ crate::ax::bindings::raise_exact_window_of_pid(target_pid, target_wid);}if !await_window_focused(target_pid, target_wid) {if prev_ok {@@ -1514,58 +1509,6 @@ fn preserves_exact_existing_focus(&& focused_window_id == Some(target_window_id)}-/// Activate `target_pid`'s window `target_wid` for NSMenu key dispatch, run `action`,-/// then immediately restore the prior frontmost process.-///-/// The entire activate → action → restore sequence is < 1 ms — a 5 ms UX monitor-/// never observes the intermediate frontmost state. NSMenu still fires because the-/// key event is already enqueued in the target's run-loop queue before we restore.-///-/// Returns `Ok(true)` when activation succeeded, `Ok(false)` when SPIs unavailable.-pub fn with_menu_shortcut_activation(- target_pid: libc::pid_t,- target_wid: u32,- action: impl FnOnce() -> anyhow::Result<()>,-) -> anyhow::Result<bool> {- let _turn = foreground_turn();- let set_front = match set_front_process_fn() {- Some(f) => f,- None => {- // SPIs unavailable — run action anyway without activation.- action()?;- return Ok(false);- }- };-- // Capture prior frontmost PSN.- let mut prev_psn = [0u8; 8];- let prev_ok = get_front_process_fn()- .map(|f| unsafe { f(prev_psn.as_mut_ptr() as *mut c_void) } == 0)- .unwrap_or(false);-- // Resolve target PSN.- let mut target_psn = [0u8; 8];- let target_ok = get_process_psn_for_window(target_wid, target_pid, &mut target_psn);- if !target_ok {- action()?;- return Ok(false);- }-- // Make target WindowServer-frontmost (kCPSNoWindows = 0x400).- unsafe { set_front(target_psn.as_ptr() as *const c_void, target_wid, 0x400) };-- // Run action then restore — even if action fails.- let result = action();-- // Restore prior frontmost (windowID=0, options=0x400).- if prev_ok {- unsafe { set_front(prev_psn.as_ptr() as *const c_void, 0, 0x400) };- }-- result?;- Ok(true)-}-#[cfg(test)]mod tests {use super::{diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/bring_to_front.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/bring_to_front.rsindex 222de0c88d..8ba36c09a8 100644--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/bring_to_front.rs+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/bring_to_front.rs@@ -9,7 +9,6 @@use std::time::{Duration, Instant};use async_trait::async_trait;-use core_foundation::base::{CFRelease, CFTypeRef};use core_graphics::display::CGDisplay;use cua_driver_core::{protocol::ToolResult,@@ -18,10 +17,7 @@ use cua_driver_core::{use objc2_app_kit::{NSApplicationActivationOptions, NSRunningApplication};use serde_json::{json, Value};-use crate::ax::bindings::{- ax_get_window_id, copy_ax_windows, perform_action, set_bool_attr_true,- AXUIElementCreateApplication,-};+use crate::ax::bindings::raise_exact_window_of_pid;use crate::windows::{WindowBounds, WindowInfo};pub struct BringToFrontTool;@@ -252,7 +248,7 @@ pub(crate) fn activate_exact_window_blocking(pid: i32, window_id: u32) -> bool {unsafe {app.activateWithOptions(NSApplicationActivationOptions::NSApplicationActivateAllWindows)};- raise_exact_ax_window(pid, window_id);+ raise_exact_window_of_pid(pid, window_id);wait_for_exact_window(pid, window_id).exact_postcondition(pid, window_id)}@@ -268,34 +264,6 @@ pub(crate) fn reactivate_application(pid: i32) -> bool {unsafe { app.activateWithOptions(NSApplicationActivationOptions::empty()) }}-/// Best-effort completion of the one exact-window request. This addresses only-/// the requested AX window; it never orders every window owned by the process.-fn raise_exact_ax_window(pid: i32, window_id: u32) -> bool {- unsafe {- let app = AXUIElementCreateApplication(pid);- if app.is_null() {- return false;- }- let mut target = None;- for window in copy_ax_windows(app) {- if target.is_none() && ax_get_window_id(window) == Some(window_id) {- target = Some(window);- } else {- CFRelease(window as CFTypeRef);- }- }- CFRelease(app as CFTypeRef);- let Some(target) = target else {- return false;- };- let raised = perform_action(target, "AXRaise") == 0;- let main = set_bool_attr_true(target, "AXMain") == 0;- let focused = set_bool_attr_true(target, "AXFocused") == 0;- CFRelease(target as CFTypeRef);- raised || main || focused- }-}-fn exact_result(pid: i32,window_id: u32,@@ -470,7 +438,7 @@ impl Tool for BringToFrontTool {NSApplicationActivationOptions::NSApplicationActivateAllWindows,)};- let ax_window_requested = raise_exact_ax_window(pid, window_id);+ let ax_window_requested = raise_exact_window_of_pid(pid, window_id);let path = match (skylight_process_accepted,skylight_exact_accepted,diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/hotkey.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/hotkey.rsindex fa30e286b7..2bf28e0582 100644--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/hotkey.rs+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/hotkey.rs@@ -572,13 +572,7 @@ impl Tool for HotkeyTool {)?;Ok(())}- // Screen Sharing is an input forwarder: modifier flags- // on a PID-routed base-key event are not relayed to the- // guest. Emit the physical modifier down/base/up- // sequence through the guarded foreground HID path.- (true, false, Some(wid), None)- if crate::input::keyboard::is_screen_sharing_pid(pid) =>- {+ (true, false, Some(wid), None) => {crate::input::skylight::with_foreground_hid_activation(pid as libc::pid_t,wid,@@ -586,16 +580,6 @@ impl Tool for HotkeyTool {)?;Ok(())}- // foreground rung: briefly front the window so NSMenu key- // equivalents dispatch, then restore prior frontmost.- (true, false, Some(wid), None) => {- crate::input::skylight::with_menu_shortcut_activation(- pid as libc::pid_t,- wid,- || crate::input::keyboard::hotkey_no_auth(pid, &key, &m),- )?;- Ok(())- }// background (default): auth-envelope post to the pid, no// raise — even when window_id was supplied for targeting.(false, false, _, Some(ptr)) => {diff --git a/scripts/ci/macos/run-rust-e2e.sh b/scripts/ci/macos/run-rust-e2e.shindex dc6d8cda90..c2e9f0928e 100755--- a/scripts/ci/macos/run-rust-e2e.sh+++ b/scripts/ci/macos/run-rust-e2e.sh@@ -414,6 +414,12 @@ EOFrun_test installed-app-textedit cargo test -p cua-driver-e2e --test installed_app_textedit_macos_test -- \--ignored --exact background_type_on_native_cocoa_is_ax_verified \--nocapture --test-threads=1+ run_test installed-app-textedit-hotkey cargo test -p cua-driver-e2e --test installed_app_textedit_macos_test -- \+ --ignored --exact foreground_window_hotkey_selects_all_behind_another_app \+ --nocapture --test-threads=1+ run_test installed-app-textedit-hotkey-sibling cargo test -p cua-driver-e2e --test installed_app_textedit_macos_test -- \+ --ignored --exact foreground_window_hotkey_reaches_an_unfocused_sibling_of_the_frontmost_app \+ --nocapture --test-threads=1# These rows assert through System Events and CoreGraphics oracles rather# than typed matrix results, so they must not leave unowned trajectories.run_test bring-to-front env -u CUA_E2E_RECORDINGS_ROOT \