Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

fix(cua-driver): close the menu a failed macOS invoke_menu path opened (#4484)

Ed-Key154ca5690350
diff --git a/libs/cua-driver/rust/Skills/cua-driver/MACOS.md b/libs/cua-driver/rust/Skills/cua-driver/MACOS.md
index 2896a0cd5f..3a2486cec4 100644
--- a/libs/cua-driver/rust/Skills/cua-driver/MACOS.md
+++ b/libs/cua-driver/rust/Skills/cua-driver/MACOS.md
@@ -443,6 +443,10 @@ application, resolves each immediate child from live AX state, uses only
on a best-effort basis.
It refuses missing, duplicate, disabled, or non-actionable segments and never
falls back to pixels.
+When a path fails after the tool already opened a menu, it cancels that menu
+and the refusal says whether any menu window it opened is still on screen. If
+the refusal says the menu may still be open, press `escape` on the window
+before other input.
```bash
cua-driver invoke_menu \
diff --git a/libs/cua-driver/rust/crates/cua-driver-e2e/tests/harness_appkit_test.rs b/libs/cua-driver/rust/crates/cua-driver-e2e/tests/harness_appkit_test.rs
index 8e60131993..b1bbf38958 100644
--- a/libs/cua-driver/rust/crates/cua-driver-e2e/tests/harness_appkit_test.rs
+++ b/libs/cua-driver/rust/crates/cua-driver-e2e/tests/harness_appkit_test.rs
@@ -949,6 +949,74 @@ fn harness_appkit_invoke_menu_live_path() {
);
}
+/// A path that fails after opening a menu closes it again: the next command
+/// through the same menu, with no other activation in between, still runs.
+#[test]
+#[ignore]
+fn harness_appkit_invoke_menu_failed_path_leaves_no_menu_open() {
+ run_case(
+ native_foreground_case(
+ "appkit",
+ "invoke_menu_failed_path",
+ Targeting::Ax,
+ DriverRoute::MacosAxAction,
+ ),
+ |pid, wid, driver| {
+ let refused = driver.call(
+ "invoke_menu",
+ serde_json::json!({
+ "pid": pid,
+ "window_id": wid,
+ "path": ["Window", "Arrange", "Missing"]
+ }),
+ );
+ assert!(refused.is_error(), "missing menu path was accepted");
+ assert_eq!(
+ refused.structured()["refusal"]["code"],
+ "menu_path_unavailable",
+ "{}",
+ refused.raw
+ );
+
+ let invoked = driver.call(
+ "invoke_menu",
+ serde_json::json!({
+ "pid": pid,
+ "window_id": wid,
+ "path": ["Window", "Arrange", "Left"]
+ }),
+ );
+ assert!(
+ !invoked.is_error(),
+ "invoke_menu failed: {}",
+ invoked.text()
+ );
+ let deadline = std::time::Instant::now() + Duration::from_secs(3);
+ while !snapshot_elements(driver, pid, wid)
+ .tree_text()
+ .contains("menu_action=window_arrange_left")
+ {
+ assert!(
+ std::time::Instant::now() < deadline,
+ "menu command after a failed path did not reach fixture; refusal: {}; command: {}",
+ refused.raw,
+ invoked.raw
+ );
+ std::thread::sleep(Duration::from_millis(100));
+ }
+ assert!(
+ refused.structured()["refusal"]["message"]
+ .as_str()
+ .is_some_and(|message| message.contains("path segment 2 was not found")
+ && message.contains("No menu window this call opened is still on screen")),
+ "{}",
+ refused.raw
+ );
+ Observation::delivered(vec![OracleKind::FixtureState], Evidence::default())
+ },
+ );
+}
+
/// text_input: type_text into the NSTextField, verify the mirror label
/// shows the typed string. Exercises the AX type_text path
/// (AXSetAttribute on AXValue, or CGEvent fallback).
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/invoke_menu.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/invoke_menu.rs
index 8b9c7ad5d4..ebc09104c6 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/invoke_menu.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/invoke_menu.rs
@@ -131,6 +131,54 @@ unsafe fn resolve_exact_prefix(
}
}
+/// Close the menu a failed path opened from the menu bar item `top` with
+/// AXCancel on that item's menu (which also ends its submenus), and read the
+/// result back from WindowServer's window list. Menu windows in
+/// `menus_before` were on screen before this call and do not count.
+unsafe fn close_opened_menu(
+ app: AXUIElementRef,
+ pid: i32,
+ top: &str,
+ menus_before: &[u32],
+) -> Option<bool> {
+ // An action that reported an error can still open its menu a moment
+ // later; give it the same settle time as a hop before looking.
+ std::thread::sleep(Duration::from_millis(80));
+ if crate::windows::new_menu_windows(pid, menus_before)? == 0 {
+ return Some(true);
+ }
+ let item = copy_element_attr(app, "AXMenuBar").and_then(|bar| {
+ set_messaging_timeout(bar);
+ let item = resolve_exact_prefix(bar, std::slice::from_ref(&top.to_owned())).ok();
+ CFRelease(bar as CFTypeRef);
+ item
+ });
+ let Some(item) = item else {
+ return Some(false);
+ };
+ set_messaging_timeout(item);
+ for child in copy_children(item) {
+ set_messaging_timeout(child);
+ if copy_string_attr(child, "AXRole").as_deref() == Some("AXMenu") {
+ let _ = perform_action(child, "AXCancel");
+ }
+ CFRelease(child as CFTypeRef);
+ }
+ CFRelease(item as CFTypeRef);
+ crate::windows::wait_for_new_menus_closed(pid, menus_before)
+}
+
+/// The refusal for a path that failed after pressing a menu item, saying
+/// whether the menu it opened was seen to close.
+fn failure_after_press(error: String, top: &str, closed: Option<bool>) -> String {
+ match closed {
+ Some(true) => format!("{error}. No menu window this call opened is still on screen."),
+ Some(false) | None => format!(
+ "{error}. The {top} menu this call opened may still be open: press escape on the window before other input."
+ ),
+ }
+}
+
fn choose_action(actions: &[String], final_segment: bool) -> Option<&'static str> {
let supports = |name: &str| actions.iter().any(|action| action == name);
let order: &[&str] = if final_segment {
@@ -148,6 +196,12 @@ unsafe fn invoke_path(pid: i32, path: &[String]) -> Result<(), String> {
}
set_messaging_timeout(app);
+ // Whether this call pressed a menu item (a press that reports an error
+ // can still open its menu), so a failure must close what it opened.
+ let mut pressed = false;
+ // Unreadable now counts every menu window later as this call's, which can
+ // only make the closed claim more cautious.
+ let menus_before = crate::windows::menu_window_ids(pid).unwrap_or_default();
let result = (|| {
for depth in 0..path.len() {
// Resolve from the live app root for every hop. Opening a menu can
@@ -175,6 +229,7 @@ unsafe fn invoke_path(pid: i32, path: &[String]) -> Result<(), String> {
return Err(error);
}
};
+ pressed = true;
let error = perform_action(target, action);
CFRelease(target as CFTypeRef);
if error != kAXErrorSuccess {
@@ -189,6 +244,17 @@ unsafe fn invoke_path(pid: i32, path: &[String]) -> Result<(), String> {
Ok(())
})();
+ // A failure after a hop opened a menu: close it, so the app is not left
+ // tracking a menu (where the next menu command reports success and does
+ // nothing).
+ let result = result.map_err(|error| {
+ if !pressed {
+ return error;
+ }
+ let closed = close_opened_menu(app, pid, &path[0], &menus_before);
+ failure_after_press(error, &path[0], closed)
+ });
+
CFRelease(app as CFTypeRef);
result
}
@@ -498,6 +564,19 @@ mod tests {
assert_eq!(choose_action(&["AXShowMenu".into()], true), None);
}
+ #[test]
+ fn a_failed_path_says_whether_its_menu_was_closed() {
+ let error = || "invoke_menu: path segment 1 was not found".to_owned();
+ assert_eq!(
+ failure_after_press(error(), "File", Some(true)),
+ "invoke_menu: path segment 1 was not found. No menu window this call opened is still on screen."
+ );
+ for unconfirmed in [Some(false), None] {
+ assert!(failure_after_press(error(), "File", unconfirmed)
+ .ends_with("The File menu this call opened may still be open: press escape on the window before other input."));
+ }
+ }
+
#[test]
fn menu_focus_requires_the_exact_frontmost_app_and_window() {
assert!(exact_window_is_ready(Some(7), 7, Some(42), 42));
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/windows.rs b/libs/cua-driver/rust/crates/platform-macos/src/windows.rs
index 8101a4936c..3f179defff 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/windows.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/windows.rs
@@ -425,6 +425,54 @@ pub fn resolve_main_window_id(pid: i32) -> anyhow::Result<u32> {
Ok(largest.unwrap().window_id)
}
+/// kCGPopUpMenuWindowLevel: the WindowServer layer of an open menu's window.
+const MENU_WINDOW_LAYER: i32 = 101;
+
+/// The ids of the menu windows `pid` has on screen; `None` when
+/// WindowServer's window list could not be read (unknown, never "no menu").
+pub(crate) fn menu_window_ids(pid: i32) -> Option<Vec<u32>> {
+ let windows = all_windows_any_layer();
+ (!windows.is_empty()).then(|| {
+ windows
+ .iter()
+ .filter(|w| w.pid == pid && w.is_on_screen && w.layer == MENU_WINDOW_LAYER)
+ .map(|w| w.window_id)
+ .collect()
+ })
+}
+
+/// How many menu windows of `pid` are on screen that were not in `before`;
+/// `None` when unknown.
+pub(crate) fn new_menu_windows(pid: i32, before: &[u32]) -> Option<usize> {
+ menu_window_ids(pid).map(|ids| ids.iter().filter(|id| !before.contains(id)).count())
+}
+
+/// `Some(true)` once no menu window of `pid` outside `before` is on screen
+/// (polled for up to 400 ms), `Some(false)` when one still is, `None` when
+/// that could not be read.
+pub(crate) fn wait_for_new_menus_closed(pid: i32, before: &[u32]) -> Option<bool> {
+ poll_until_no_menu(
+ || new_menu_windows(pid, before),
+ std::time::Duration::from_millis(400),
+ )
+}
+
+fn poll_until_no_menu(
+ mut open_menus: impl FnMut() -> Option<usize>,
+ timeout: std::time::Duration,
+) -> Option<bool> {
+ let deadline = std::time::Instant::now() + timeout;
+ loop {
+ if open_menus()? == 0 {
+ return Some(true);
+ }
+ if std::time::Instant::now() >= deadline {
+ return Some(false);
+ }
+ std::thread::sleep(std::time::Duration::from_millis(40));
+ }
+}
+
#[cfg(test)]
mod tests {
use super::*;
@@ -480,6 +528,19 @@ mod tests {
}
}
+ #[test]
+ fn menu_close_polling_keeps_an_unreadable_window_list_unknown() {
+ let timeout = std::time::Duration::from_millis(100);
+ assert_eq!(poll_until_no_menu(|| Some(0), timeout), Some(true));
+ assert_eq!(poll_until_no_menu(|| None, timeout), None);
+ assert_eq!(poll_until_no_menu(|| Some(1), timeout), Some(false));
+ let mut reads = [Some(1), Some(0)].into_iter();
+ assert_eq!(
+ poll_until_no_menu(|| reads.next().flatten(), timeout),
+ Some(true)
+ );
+ }
+
#[test]
fn owner_resolves_same_pid() {
let windows = vec![window(42, 800, "TextEdit")];
diff --git a/libs/cua/skills/cua-driver/MACOS.md b/libs/cua/skills/cua-driver/MACOS.md
index 2896a0cd5f..3a2486cec4 100644
--- a/libs/cua/skills/cua-driver/MACOS.md
+++ b/libs/cua/skills/cua-driver/MACOS.md
@@ -443,6 +443,10 @@ application, resolves each immediate child from live AX state, uses only
on a best-effort basis.
It refuses missing, duplicate, disabled, or non-actionable segments and never
falls back to pixels.
+When a path fails after the tool already opened a menu, it cancels that menu
+and the refusal says whether any menu window it opened is still on screen. If
+the refusal says the menu may still be open, press `escape` on the window
+before other input.
```bash
cua-driver invoke_menu \
diff --git a/scripts/ci/macos/run-rust-e2e.sh b/scripts/ci/macos/run-rust-e2e.sh
index 1c45a6713e..1d975344c2 100755
--- a/scripts/ci/macos/run-rust-e2e.sh
+++ b/scripts/ci/macos/run-rust-e2e.sh
@@ -356,6 +356,7 @@ if [[ "${SUITE}" == native || "${SUITE}" == all ]]; then
harness_appkit_stale_element_token_fails_closed \
snapshot_publication::harness_appkit_pending_snapshot_cannot_retarget_token \
harness_appkit_invoke_menu_live_path \
+ harness_appkit_invoke_menu_failed_path_leaves_no_menu_open \
harness_appkit_text_input \
harness_appkit_element_foreground_press_key_commits_edit \
harness_appkit_foreground_press_key_chord_carries_its_modifiers \