Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
fix(cua-driver): close the menu a failed macOS invoke_menu path opened (#4484)
Ed-Key154ca5690350
diff --git a/libs/cua-driver/rust/Skills/cua-driver/MACOS.md b/libs/cua-driver/rust/Skills/cua-driver/MACOS.mdindex 2896a0cd5f..3a2486cec4 100644--- a/libs/cua-driver/rust/Skills/cua-driver/MACOS.md+++ b/libs/cua-driver/rust/Skills/cua-driver/MACOS.md@@ -443,6 +443,10 @@ application, resolves each immediate child from live AX state, uses onlyon a best-effort basis.It refuses missing, duplicate, disabled, or non-actionable segments and neverfalls back to pixels.+When a path fails after the tool already opened a menu, it cancels that menu+and the refusal says whether any menu window it opened is still on screen. If+the refusal says the menu may still be open, press `escape` on the window+before other input.```bashcua-driver invoke_menu \diff --git a/libs/cua-driver/rust/crates/cua-driver-e2e/tests/harness_appkit_test.rs b/libs/cua-driver/rust/crates/cua-driver-e2e/tests/harness_appkit_test.rsindex 8e60131993..b1bbf38958 100644--- a/libs/cua-driver/rust/crates/cua-driver-e2e/tests/harness_appkit_test.rs+++ b/libs/cua-driver/rust/crates/cua-driver-e2e/tests/harness_appkit_test.rs@@ -949,6 +949,74 @@ fn harness_appkit_invoke_menu_live_path() {);}+/// A path that fails after opening a menu closes it again: the next command+/// through the same menu, with no other activation in between, still runs.+#[test]+#[ignore]+fn harness_appkit_invoke_menu_failed_path_leaves_no_menu_open() {+ run_case(+ native_foreground_case(+ "appkit",+ "invoke_menu_failed_path",+ Targeting::Ax,+ DriverRoute::MacosAxAction,+ ),+ |pid, wid, driver| {+ let refused = driver.call(+ "invoke_menu",+ serde_json::json!({+ "pid": pid,+ "window_id": wid,+ "path": ["Window", "Arrange", "Missing"]+ }),+ );+ assert!(refused.is_error(), "missing menu path was accepted");+ assert_eq!(+ refused.structured()["refusal"]["code"],+ "menu_path_unavailable",+ "{}",+ refused.raw+ );++ let invoked = driver.call(+ "invoke_menu",+ serde_json::json!({+ "pid": pid,+ "window_id": wid,+ "path": ["Window", "Arrange", "Left"]+ }),+ );+ assert!(+ !invoked.is_error(),+ "invoke_menu failed: {}",+ invoked.text()+ );+ let deadline = std::time::Instant::now() + Duration::from_secs(3);+ while !snapshot_elements(driver, pid, wid)+ .tree_text()+ .contains("menu_action=window_arrange_left")+ {+ assert!(+ std::time::Instant::now() < deadline,+ "menu command after a failed path did not reach fixture; refusal: {}; command: {}",+ refused.raw,+ invoked.raw+ );+ std::thread::sleep(Duration::from_millis(100));+ }+ assert!(+ refused.structured()["refusal"]["message"]+ .as_str()+ .is_some_and(|message| message.contains("path segment 2 was not found")+ && message.contains("No menu window this call opened is still on screen")),+ "{}",+ refused.raw+ );+ Observation::delivered(vec![OracleKind::FixtureState], Evidence::default())+ },+ );+}+/// text_input: type_text into the NSTextField, verify the mirror label/// shows the typed string. Exercises the AX type_text path/// (AXSetAttribute on AXValue, or CGEvent fallback).diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/invoke_menu.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/invoke_menu.rsindex 8b9c7ad5d4..ebc09104c6 100644--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/invoke_menu.rs+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/invoke_menu.rs@@ -131,6 +131,54 @@ unsafe fn resolve_exact_prefix(}}+/// Close the menu a failed path opened from the menu bar item `top` with+/// AXCancel on that item's menu (which also ends its submenus), and read the+/// result back from WindowServer's window list. Menu windows in+/// `menus_before` were on screen before this call and do not count.+unsafe fn close_opened_menu(+ app: AXUIElementRef,+ pid: i32,+ top: &str,+ menus_before: &[u32],+) -> Option<bool> {+ // An action that reported an error can still open its menu a moment+ // later; give it the same settle time as a hop before looking.+ std::thread::sleep(Duration::from_millis(80));+ if crate::windows::new_menu_windows(pid, menus_before)? == 0 {+ return Some(true);+ }+ let item = copy_element_attr(app, "AXMenuBar").and_then(|bar| {+ set_messaging_timeout(bar);+ let item = resolve_exact_prefix(bar, std::slice::from_ref(&top.to_owned())).ok();+ CFRelease(bar as CFTypeRef);+ item+ });+ let Some(item) = item else {+ return Some(false);+ };+ set_messaging_timeout(item);+ for child in copy_children(item) {+ set_messaging_timeout(child);+ if copy_string_attr(child, "AXRole").as_deref() == Some("AXMenu") {+ let _ = perform_action(child, "AXCancel");+ }+ CFRelease(child as CFTypeRef);+ }+ CFRelease(item as CFTypeRef);+ crate::windows::wait_for_new_menus_closed(pid, menus_before)+}++/// The refusal for a path that failed after pressing a menu item, saying+/// whether the menu it opened was seen to close.+fn failure_after_press(error: String, top: &str, closed: Option<bool>) -> String {+ match closed {+ Some(true) => format!("{error}. No menu window this call opened is still on screen."),+ Some(false) | None => format!(+ "{error}. The {top} menu this call opened may still be open: press escape on the window before other input."+ ),+ }+}+fn choose_action(actions: &[String], final_segment: bool) -> Option<&'static str> {let supports = |name: &str| actions.iter().any(|action| action == name);let order: &[&str] = if final_segment {@@ -148,6 +196,12 @@ unsafe fn invoke_path(pid: i32, path: &[String]) -> Result<(), String> {}set_messaging_timeout(app);+ // Whether this call pressed a menu item (a press that reports an error+ // can still open its menu), so a failure must close what it opened.+ let mut pressed = false;+ // Unreadable now counts every menu window later as this call's, which can+ // only make the closed claim more cautious.+ let menus_before = crate::windows::menu_window_ids(pid).unwrap_or_default();let result = (|| {for depth in 0..path.len() {// Resolve from the live app root for every hop. Opening a menu can@@ -175,6 +229,7 @@ unsafe fn invoke_path(pid: i32, path: &[String]) -> Result<(), String> {return Err(error);}};+ pressed = true;let error = perform_action(target, action);CFRelease(target as CFTypeRef);if error != kAXErrorSuccess {@@ -189,6 +244,17 @@ unsafe fn invoke_path(pid: i32, path: &[String]) -> Result<(), String> {Ok(())})();+ // A failure after a hop opened a menu: close it, so the app is not left+ // tracking a menu (where the next menu command reports success and does+ // nothing).+ let result = result.map_err(|error| {+ if !pressed {+ return error;+ }+ let closed = close_opened_menu(app, pid, &path[0], &menus_before);+ failure_after_press(error, &path[0], closed)+ });+CFRelease(app as CFTypeRef);result}@@ -498,6 +564,19 @@ mod tests {assert_eq!(choose_action(&["AXShowMenu".into()], true), None);}+ #[test]+ fn a_failed_path_says_whether_its_menu_was_closed() {+ let error = || "invoke_menu: path segment 1 was not found".to_owned();+ assert_eq!(+ failure_after_press(error(), "File", Some(true)),+ "invoke_menu: path segment 1 was not found. No menu window this call opened is still on screen."+ );+ for unconfirmed in [Some(false), None] {+ assert!(failure_after_press(error(), "File", unconfirmed)+ .ends_with("The File menu this call opened may still be open: press escape on the window before other input."));+ }+ }+#[test]fn menu_focus_requires_the_exact_frontmost_app_and_window() {assert!(exact_window_is_ready(Some(7), 7, Some(42), 42));diff --git a/libs/cua-driver/rust/crates/platform-macos/src/windows.rs b/libs/cua-driver/rust/crates/platform-macos/src/windows.rsindex 8101a4936c..3f179defff 100644--- a/libs/cua-driver/rust/crates/platform-macos/src/windows.rs+++ b/libs/cua-driver/rust/crates/platform-macos/src/windows.rs@@ -425,6 +425,54 @@ pub fn resolve_main_window_id(pid: i32) -> anyhow::Result<u32> {Ok(largest.unwrap().window_id)}+/// kCGPopUpMenuWindowLevel: the WindowServer layer of an open menu's window.+const MENU_WINDOW_LAYER: i32 = 101;++/// The ids of the menu windows `pid` has on screen; `None` when+/// WindowServer's window list could not be read (unknown, never "no menu").+pub(crate) fn menu_window_ids(pid: i32) -> Option<Vec<u32>> {+ let windows = all_windows_any_layer();+ (!windows.is_empty()).then(|| {+ windows+ .iter()+ .filter(|w| w.pid == pid && w.is_on_screen && w.layer == MENU_WINDOW_LAYER)+ .map(|w| w.window_id)+ .collect()+ })+}++/// How many menu windows of `pid` are on screen that were not in `before`;+/// `None` when unknown.+pub(crate) fn new_menu_windows(pid: i32, before: &[u32]) -> Option<usize> {+ menu_window_ids(pid).map(|ids| ids.iter().filter(|id| !before.contains(id)).count())+}++/// `Some(true)` once no menu window of `pid` outside `before` is on screen+/// (polled for up to 400 ms), `Some(false)` when one still is, `None` when+/// that could not be read.+pub(crate) fn wait_for_new_menus_closed(pid: i32, before: &[u32]) -> Option<bool> {+ poll_until_no_menu(+ || new_menu_windows(pid, before),+ std::time::Duration::from_millis(400),+ )+}++fn poll_until_no_menu(+ mut open_menus: impl FnMut() -> Option<usize>,+ timeout: std::time::Duration,+) -> Option<bool> {+ let deadline = std::time::Instant::now() + timeout;+ loop {+ if open_menus()? == 0 {+ return Some(true);+ }+ if std::time::Instant::now() >= deadline {+ return Some(false);+ }+ std::thread::sleep(std::time::Duration::from_millis(40));+ }+}+#[cfg(test)]mod tests {use super::*;@@ -480,6 +528,19 @@ mod tests {}}+ #[test]+ fn menu_close_polling_keeps_an_unreadable_window_list_unknown() {+ let timeout = std::time::Duration::from_millis(100);+ assert_eq!(poll_until_no_menu(|| Some(0), timeout), Some(true));+ assert_eq!(poll_until_no_menu(|| None, timeout), None);+ assert_eq!(poll_until_no_menu(|| Some(1), timeout), Some(false));+ let mut reads = [Some(1), Some(0)].into_iter();+ assert_eq!(+ poll_until_no_menu(|| reads.next().flatten(), timeout),+ Some(true)+ );+ }+#[test]fn owner_resolves_same_pid() {let windows = vec![window(42, 800, "TextEdit")];diff --git a/libs/cua/skills/cua-driver/MACOS.md b/libs/cua/skills/cua-driver/MACOS.mdindex 2896a0cd5f..3a2486cec4 100644--- a/libs/cua/skills/cua-driver/MACOS.md+++ b/libs/cua/skills/cua-driver/MACOS.md@@ -443,6 +443,10 @@ application, resolves each immediate child from live AX state, uses onlyon a best-effort basis.It refuses missing, duplicate, disabled, or non-actionable segments and neverfalls back to pixels.+When a path fails after the tool already opened a menu, it cancels that menu+and the refusal says whether any menu window it opened is still on screen. If+the refusal says the menu may still be open, press `escape` on the window+before other input.```bashcua-driver invoke_menu \diff --git a/scripts/ci/macos/run-rust-e2e.sh b/scripts/ci/macos/run-rust-e2e.shindex 1c45a6713e..1d975344c2 100755--- a/scripts/ci/macos/run-rust-e2e.sh+++ b/scripts/ci/macos/run-rust-e2e.sh@@ -356,6 +356,7 @@ if [[ "${SUITE}" == native || "${SUITE}" == all ]]; thenharness_appkit_stale_element_token_fails_closed \snapshot_publication::harness_appkit_pending_snapshot_cannot_retarget_token \harness_appkit_invoke_menu_live_path \+ harness_appkit_invoke_menu_failed_path_leaves_no_menu_open \harness_appkit_text_input \harness_appkit_element_foreground_press_key_commits_edit \harness_appkit_foreground_press_key_chord_carries_its_modifiers \