Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
fix(cua-driver): background menu/format controls and page reads on another Space (CUA-1768, CUA-1769) (#5307)
Francesco Bonaccie08eb41e10de
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/ax/exact_target.rs b/libs/cua-driver/rust/crates/platform-macos/src/ax/exact_target.rsindex 1b6060632f..74485384b3 100644--- a/libs/cua-driver/rust/crates/platform-macos/src/ax/exact_target.rs+++ b/libs/cua-driver/rust/crates/platform-macos/src/ax/exact_target.rs@@ -77,6 +77,63 @@ pub unsafe fn element_window_id(element: AXUIElementRef) -> Option<u32> {resolved}+/// Decide an addressed element's ancestry from what was read about it.+fn element_ancestry(window: Option<u32>, in_app_menu_bar: bool, requested: u32) -> ElementAncestry {+ match window {+ Some(id) if id == requested => ElementAncestry::ProvenDescendant,+ Some(_) => ElementAncestry::OutsideTargetWindow,+ None if in_app_menu_bar => ElementAncestry::ProvenDescendant,+ None => ElementAncestry::Unproven,+ }+}++/// Whether `element` sits under the application's own `AXMenuBar`: a bounded+/// `AXParent` ascent that must end at the same menu bar element the+/// application reports.+///+/// # Safety+///+/// `app` and `element` must be valid `AXUIElementRef`s.+unsafe fn element_in_app_menu_bar(app: AXUIElementRef, element: AXUIElementRef) -> bool {+ let Some(menu_bar) = copy_element_attr(app, "AXMenuBar") else {+ return false;+ };+ let mut current = element;+ let mut owned = false;+ let mut found = false;+ for _ in 0..MAX_ANCESTRY_DEPTH {+ if core_foundation::base::CFEqual(current as CFTypeRef, menu_bar as CFTypeRef) != 0 {+ found = true;+ break;+ }+ if matches!(+ copy_string_attr(current, "AXRole").as_deref(),+ Some("AXWindow") | Some("AXSheet") | Some("AXApplication") | None+ ) {+ break;+ }+ let parent = copy_element_attr(current, "AXParent");+ if owned {+ CFRelease(current as CFTypeRef);+ }+ match parent {+ Some(parent) => {+ current = parent;+ owned = true;+ }+ None => {+ owned = false;+ break;+ }+ }+ }+ if owned {+ CFRelease(current as CFTypeRef);+ }+ CFRelease(menu_bar as CFTypeRef);+ found+}+/// Map a window-role AX element to the CGWindowID of the top-level window it/// belongs to: an `AXWindow` maps to itself, an `AXSheet` to the window it is/// attached to (see [`element_window_id`]).@@ -258,10 +315,13 @@ pub fn gather_background_facts(super::enablement::ensure_chromium_ax_enabled(pid, app);let records = ax_window_records(app, pid, window_id);let app_hidden = copy_bool_attr(app, "AXHidden");- let element = element_ptr.map(|ptr| match element_window_id(ptr as AXUIElementRef) {- Some(id) if id == window_id => ElementAncestry::ProvenDescendant,- Some(_) => ElementAncestry::OutsideTargetWindow,- None => ElementAncestry::Unproven,+ let element = element_ptr.map(|ptr| {+ let element = ptr as AXUIElementRef;+ let window = element_window_id(element);+ // A menu-bar item has no window ancestor: the menu bar hangs+ // off the application. It still belongs to this app.+ let in_menu_bar = window.is_none() && element_in_app_menu_bar(app, element);+ element_ancestry(window, in_menu_bar, window_id)});let focused_window_id = copy_element_attr(app, "AXFocusedWindow").and_then(|window| {let window_id = ax_get_window_id(window);@@ -318,6 +378,16 @@ mod tests {)}+ #[test]+ fn menu_bar_item_is_proven_only_through_the_app_menu_bar() {+ use super::element_ancestry;+ use cua_driver_core::background_input::ElementAncestry::*;+ assert_eq!(element_ancestry(None, true, 7), ProvenDescendant);+ assert_eq!(element_ancestry(None, false, 7), Unproven);+ assert_eq!(element_ancestry(Some(7), false, 7), ProvenDescendant);+ assert_eq!(element_ancestry(Some(8), true, 7), OutsideTargetWindow);+ }+#[test]fn window_maps_to_itself() {assert_eq!(resolve(&[("AXWindow", None, Some(167))], 0), Some(167));diff --git a/libs/cua-driver/rust/crates/platform-macos/src/browser/ax_page_reader.rs b/libs/cua-driver/rust/crates/platform-macos/src/browser/ax_page_reader.rsindex 8c35cfac7f..bb017e0418 100644--- a/libs/cua-driver/rust/crates/platform-macos/src/browser/ax_page_reader.rs+++ b/libs/cua-driver/rust/crates/platform-macos/src/browser/ax_page_reader.rs@@ -29,6 +29,33 @@ const SKIP_ROLES: &[&str] = &[const TEXT_ROLES: &[&str] = &["AXStaticText", "AXHeading", "AXWebArea"];impl AXPageReader {+ /// The part of a window-scoped AX tree that is page content: every+ /// `AXWebArea` row and the rows nested under it. A window-scoped walk also+ /// carries the app's menu bar and the browser's own toolbar, which are not+ /// the page; reading them as page text answers a `get_text` or `query_dom`+ /// with "Apple, About This Mac, ...". `None` when the tree has no web area+ /// (the page's accessibility tree is not built, or the window is not a+ /// page).+ pub fn web_area_markdown(tree_markdown: &str) -> Option<String> {+ let mut out: Vec<&str> = Vec::new();+ let mut area_indent: Option<usize> = None;+ for line in tree_markdown.lines() {+ let indent = line.len() - line.trim_start().len();+ if let Some(base) = area_indent {+ if indent > base {+ out.push(line);+ continue;+ }+ area_indent = None;+ }+ if parse_ax_line(line).is_some_and(|el| el.role == "AXWebArea") {+ area_indent = Some(indent);+ out.push(line);+ }+ }+ (!out.is_empty()).then(|| out.join("\n"))+ }+/// Extract readable text from an AX tree markdown string.pub fn extract_text(tree_markdown: &str) -> String {let mut lines_out: Vec<String> = Vec::new();@@ -220,3 +247,40 @@ fn css_selector_to_ax_roles(selector: &str) -> Vec<&'static str> {_ => vec![],}}++#[cfg(test)]+mod web_area_tests {+ use super::AXPageReader;++ const TREE: &str = "\+- [0] AXMenuBar+ - [1] AXMenuBarItem \"Apple\"+ - [2] AXMenuItem \"About This Mac\"+- [3] AXWindow \"Orders - Chrome\"+ - [4] AXToolbar+ - [5] AXTextField \"Address and search bar\"+ - [6] AXWebArea \"Orders\"+ - [7] AXHeading \"Your loans\"+ - [8] AXStaticText \"Hedge trimmer, due 3 Nov\"+ - [9] AXButton \"Back\"+";++ #[test]+ fn web_area_markdown_keeps_only_page_rows() {+ let page = AXPageReader::web_area_markdown(TREE).expect("web area");+ assert!(page.contains("Hedge trimmer"));+ assert!(page.contains("Your loans"));+ assert!(!page.contains("About This Mac"));+ assert!(!page.contains("Address and search bar"));+ assert!(!page.contains("\"Back\""));+ let text = AXPageReader::extract_text(&page);+ assert!(text.contains("Hedge trimmer, due 3 Nov"));+ assert!(!text.contains("Apple"));+ }++ #[test]+ fn a_tree_without_a_web_area_has_no_page_content() {+ let tree = "- [0] AXMenuBar\n - [1] AXMenuBarItem \"Apple\"\n- [2] AXWindow \"x\"\n";+ assert_eq!(AXPageReader::web_area_markdown(tree), None);+ }+}diff --git a/libs/cua-driver/rust/crates/platform-macos/src/browser/cdp_client.rs b/libs/cua-driver/rust/crates/platform-macos/src/browser/cdp_client.rsindex 17e0521b55..9bcde9898d 100644--- a/libs/cua-driver/rust/crates/platform-macos/src/browser/cdp_client.rs+++ b/libs/cua-driver/rust/crates/platform-macos/src/browser/cdp_client.rs@@ -150,6 +150,11 @@ impl CdpSessionCache {Ok(arc)}+ /// Whether a connection for `port` is already open (and so approved).+ pub async fn has_session(&self, port: u16) -> bool {+ self.sessions.lock().await.contains_key(&port)+ }+async fn evict(&self, port: u16) {self.sessions.lock().await.remove(&port);}diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/click.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/click.rsindex 161b13f5a1..dc8c4745cb 100644--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/click.rs+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/click.rs@@ -1425,6 +1425,31 @@ impl Tool for ClickTool {////// `element` is the cached AX element pointer and its snapshot index; `window`/// is the target (pid, window_id).+/// Whether a background AXPress needs its window to be the app's key window.+///+/// A toolbar or format-bar control in a native app sends its action to the+/// first responder through the key window (TextEdit's Bold checkbox applies+/// the trait to the text view that way). With the app behind the user's+/// window nothing is key, so the press flips the control but the document+/// does not change. Browsers and Electron shells run page content in their own+/// process and take no part in this.+fn press_needs_key_window(action: &str, front_pid: Option<i32>, pid: i32, native: bool) -> bool {+ action == "AXPress" && native && front_pid != Some(pid)+}++/// Make the target window AppKit-key without raising it, activating the app+/// or taking key status from the user's window (the focus record only, as the+/// background pixel click does), then let AppKit settle.+fn prime_key_window_for_background_press(pid: i32, window_id: u32, action: &str) {+ let native = !super::type_text::is_browser_pid(pid)+ && !crate::browser::electron_js::ElectronJs::is_electron(pid);+ if press_needs_key_window(action, apps::frontmost_pid(), pid, native)+ && crate::input::skylight::activate_without_raise_with(pid, window_id, false)+ {+ std::thread::sleep(std::time::Duration::from_millis(80));+ }+}+fn perform_ax_click(element: (usize, usize),window: (i32, u32),@@ -1575,6 +1600,9 @@ fn perform_ax_click(let toggle_before = is_toggle_press(ax_action, &role, &advertised).then(read_value).flatten();+ if !foreground {+ prime_key_window_for_background_press(pid, window_id, ax_action);+ }let err = unsafe { crate::ax::bindings::perform_action(element, ax_action) };if err != crate::ax::bindings::kAXErrorSuccess {// Checked before the row-selection fallback, which writes AXSelected@@ -1830,6 +1858,20 @@ mod tests {/// Surface 5: schema must advertise the new `button` field with the three/// canonical values and default to "left". Hermes / Codex / Claude Code/// consumers branch on this enum being present.+ #[test]+ fn background_press_needs_the_key_window_only_in_a_native_background_app() {+ assert!(press_needs_key_window("AXPress", Some(1), 2, true));+ assert!(+ !press_needs_key_window("AXPress", Some(2), 2, true),+ "frontmost"+ );+ assert!(+ !press_needs_key_window("AXPress", Some(1), 2, false),+ "browser or Electron"+ );+ assert!(!press_needs_key_window("AXShowMenu", Some(1), 2, true));+ }+#[test]fn schema_advertises_button_enum() {let d = def();diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/invoke_menu.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/invoke_menu.rsindex 352fe4c3dc..d913e6670a 100644--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/invoke_menu.rs+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/invoke_menu.rs@@ -577,6 +577,11 @@ unsafe fn item_is_save(item: AXUIElementRef) -> bool {== 0.0}+/// How long a disabled item gets to enable after its window becomes key, and+/// how often it is re-read.+const ENABLE_WAIT: Duration = Duration::from_millis(600);+const ENABLE_POLL: Duration = Duration::from_millis(40);+/// Resolve the whole path from the app's live menu bar without opening any/// level, then perform the final item's action. Used for a background app:/// opening a menu-bar menu would take key focus and draw over the user's@@ -611,7 +616,13 @@ unsafe fn invoke_path_in_background(if copy_bool_attr(target, "AXEnabled") == Some(false)&& crate::input::skylight::activate_without_raise_with(pid, window_id, false){- std::thread::sleep(Duration::from_millis(120));+ // AppKit revalidates the item once the window is key; wait for it+ // instead of reading the stale flag after a fixed pause.+ let mut waited = Duration::ZERO;+ while waited < ENABLE_WAIT && copy_bool_attr(target, "AXEnabled") == Some(false) {+ std::thread::sleep(ENABLE_POLL);+ waited += ENABLE_POLL;+ }}if copy_bool_attr(target, "AXEnabled") == Some(false) {return Err(BackgroundPressError::Disabled {diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/page.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/page.rsindex 07be33d842..4fe5804934 100644--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/page.rs+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/page.rs@@ -25,6 +25,17 @@ impl MacOsPageBackend {Self { state }}+ /// Evaluate `js` through a CDP session the driver already holds for this+ /// browser. Never opens a new connection: a new one makes Chrome show its+ /// "Allow remote debugging?" prompt and pull itself to the front.+ async fn attached_cdp_evaluate(&self, pid: i32, js: &str) -> Option<String> {+ let port = CdpClient::find_port_for_pid(pid).await?;+ if !self.state.cdp_sessions.has_session(port).await {+ return None;+ }+ self.state.cdp_sessions.evaluate(js, port, None).await.ok()+ }+/// Resolve `bundle_id` for `pid` via the running-apps list.async fn bundle_id_for(pid: i32) -> String {tokio::task::spawn_blocking(move || {@@ -50,13 +61,22 @@ impl PageBackend for MacOsPageBackend {.unwrap_or(false);if use_ax_fallback {- return ax_text_fallback(pid, window_id).await;+ return ax_text_fallback(pid, window_id, false).await;}- // Try JS path first; on error, fall back to AX walk.- match execute_js("document.body.innerText", &bundle_id, pid, window_id).await {+ // Try JS path first; on error, use a CDP session the driver already+ // holds, then the window's web area.+ let js = "document.body.innerText";+ match execute_js(js, &bundle_id, pid, window_id).await {Ok(result) => Ok(result),- Err(_) => ax_text_fallback(pid, window_id).await,+ Err(error) => {+ if let Some(result) = self.attached_cdp_evaluate(pid, js).await {+ return Ok(result);+ }+ ax_text_fallback(pid, window_id, BrowserJs::supports(&bundle_id))+ .await+ .map_err(|ax| anyhow::anyhow!("{ax} (JavaScript read failed first: {error})"))+ }}}@@ -75,15 +95,25 @@ impl PageBackend for MacOsPageBackend {.unwrap_or(false);if use_ax_fallback {- let results = ax_query_fallback(pid, window_id, css_selector).await?;+ let results = ax_query_fallback(pid, window_id, css_selector, false).await?;return Ok(format_ax_elements(&results));}let js = build_query_selector_js(css_selector, attributes);match execute_js(&js, &bundle_id, pid, window_id).await {Ok(result) => Ok(result),- Err(_) => {- let results = ax_query_fallback(pid, window_id, css_selector).await?;+ Err(error) => {+ if let Some(result) = self.attached_cdp_evaluate(pid, &js).await {+ return Ok(result);+ }+ let results = ax_query_fallback(+ pid,+ window_id,+ css_selector,+ BrowserJs::supports(&bundle_id),+ )+ .await+ .map_err(|ax| anyhow::anyhow!("{ax} (JavaScript read failed first: {error})"))?;Ok(format_ax_elements(&results))}}@@ -307,15 +337,37 @@ async fn execute_js(js: &str, bundle_id: &str, pid: i32, window_id: u64) -> anyhanyhow::bail!("Unsupported browser: bundle_id={bundle_id}");}+/// The tree rows that are page content. A browser window's walk also holds+/// the app's menu bar and the browser toolbar, so a browser read is scoped to+/// the window's `AXWebArea` and fails when there is none, instead of+/// returning the menu bar as if it were the page. Other web-view apps keep+/// the whole tree when they expose no web area.+fn page_rows(tree_markdown: String, require_web_area: bool) -> anyhow::Result<String> {+ match AXPageReader::web_area_markdown(&tree_markdown) {+ Some(page) => Ok(page),+ None if require_web_area => anyhow::bail!(+ "This window's page content is not readable: its accessibility tree has no \+ AXWebArea (only the browser's own controls and the menu bar). Read it again in \+ a moment, use get_browser_state, or attach CDP with browser_prepare."+ ),+ None => Ok(tree_markdown),+ }+}+/// Extract page text via the AX tree.-async fn ax_text_fallback(pid: i32, window_id: u64) -> anyhow::Result<String> {+async fn ax_text_fallback(+ pid: i32,+ window_id: u64,+ require_web_area: bool,+) -> anyhow::Result<String> {let window_id = u32::try_from(window_id).map_err(|_| anyhow::anyhow!("macOS window_id {window_id} is out of u32 range"))?;let result =tokio::task::spawn_blocking(move || crate::ax::tree::walk_tree(pid, Some(window_id), None)).await.map_err(|e| anyhow::anyhow!("AX walk task failed: {e}"))?;- Ok(AXPageReader::extract_text(&result.tree_markdown))+ let rows = page_rows(result.tree_markdown, require_web_area)?;+ Ok(AXPageReader::extract_text(&rows))}/// Query AX tree by CSS selector.@@ -323,6 +375,7 @@ async fn ax_query_fallback(pid: i32,window_id: u64,selector: &str,+ require_web_area: bool,) -> anyhow::Result<Vec<crate::browser::ax_page_reader::AXElement>> {let window_id = u32::try_from(window_id).map_err(|_| anyhow::anyhow!("macOS window_id {window_id} is out of u32 range"))?;@@ -331,7 +384,8 @@ async fn ax_query_fallback(tokio::task::spawn_blocking(move || crate::ax::tree::walk_tree(pid, Some(window_id), None)).await.map_err(|e| anyhow::anyhow!("AX walk task failed: {e}"))?;- Ok(AXPageReader::query(&sel, &result.tree_markdown))+ let rows = page_rows(result.tree_markdown, require_web_area)?;+ Ok(AXPageReader::query(&sel, &rows))}fn format_ax_elements(elements: &[crate::browser::ax_page_reader::AXElement]) -> String {@@ -421,3 +475,31 @@ fn required_finite(value: &serde_json::Value, key: &str, raw: &str) -> anyhow::R)})}++#[cfg(test)]+mod tests {+ use super::page_rows;++ const MENU_ONLY: &str = "- [0] AXMenuBar\n - [1] AXMenuBarItem \"Apple\"\n";++ #[test]+ fn browser_read_without_a_web_area_fails_instead_of_returning_the_menu_bar() {+ let error = page_rows(MENU_ONLY.to_owned(), true)+ .unwrap_err()+ .to_string();+ assert!(error.contains("AXWebArea"), "{error}");+ }++ #[test]+ fn browser_read_is_scoped_to_the_web_area() {+ let tree = format!("{MENU_ONLY}- [2] AXWindow \"w\"\n - [3] AXWebArea \"p\"\n - [4] AXStaticText \"hello\"\n");+ let page = page_rows(tree, true).unwrap();+ assert!(page.contains("hello"));+ assert!(!page.contains("Apple"));+ }++ #[test]+ fn web_view_apps_keep_the_whole_tree_without_a_web_area() {+ assert_eq!(page_rows(MENU_ONLY.to_owned(), false).unwrap(), MENU_ONLY);+ }+}