Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

fix(cua-driver): background menu/format controls and page reads on another Space (CUA-1768, CUA-1769) (#5307)

Francesco Bonaccie08eb41e10de
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/ax/exact_target.rs b/libs/cua-driver/rust/crates/platform-macos/src/ax/exact_target.rs
index 1b6060632f..74485384b3 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/ax/exact_target.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/ax/exact_target.rs
@@ -77,6 +77,63 @@ pub unsafe fn element_window_id(element: AXUIElementRef) -> Option<u32> {
resolved
}
+/// Decide an addressed element's ancestry from what was read about it.
+fn element_ancestry(window: Option<u32>, in_app_menu_bar: bool, requested: u32) -> ElementAncestry {
+ match window {
+ Some(id) if id == requested => ElementAncestry::ProvenDescendant,
+ Some(_) => ElementAncestry::OutsideTargetWindow,
+ None if in_app_menu_bar => ElementAncestry::ProvenDescendant,
+ None => ElementAncestry::Unproven,
+ }
+}
+
+/// Whether `element` sits under the application's own `AXMenuBar`: a bounded
+/// `AXParent` ascent that must end at the same menu bar element the
+/// application reports.
+///
+/// # Safety
+///
+/// `app` and `element` must be valid `AXUIElementRef`s.
+unsafe fn element_in_app_menu_bar(app: AXUIElementRef, element: AXUIElementRef) -> bool {
+ let Some(menu_bar) = copy_element_attr(app, "AXMenuBar") else {
+ return false;
+ };
+ let mut current = element;
+ let mut owned = false;
+ let mut found = false;
+ for _ in 0..MAX_ANCESTRY_DEPTH {
+ if core_foundation::base::CFEqual(current as CFTypeRef, menu_bar as CFTypeRef) != 0 {
+ found = true;
+ break;
+ }
+ if matches!(
+ copy_string_attr(current, "AXRole").as_deref(),
+ Some("AXWindow") | Some("AXSheet") | Some("AXApplication") | None
+ ) {
+ break;
+ }
+ let parent = copy_element_attr(current, "AXParent");
+ if owned {
+ CFRelease(current as CFTypeRef);
+ }
+ match parent {
+ Some(parent) => {
+ current = parent;
+ owned = true;
+ }
+ None => {
+ owned = false;
+ break;
+ }
+ }
+ }
+ if owned {
+ CFRelease(current as CFTypeRef);
+ }
+ CFRelease(menu_bar as CFTypeRef);
+ found
+}
+
/// Map a window-role AX element to the CGWindowID of the top-level window it
/// belongs to: an `AXWindow` maps to itself, an `AXSheet` to the window it is
/// attached to (see [`element_window_id`]).
@@ -258,10 +315,13 @@ pub fn gather_background_facts(
super::enablement::ensure_chromium_ax_enabled(pid, app);
let records = ax_window_records(app, pid, window_id);
let app_hidden = copy_bool_attr(app, "AXHidden");
- let element = element_ptr.map(|ptr| match element_window_id(ptr as AXUIElementRef) {
- Some(id) if id == window_id => ElementAncestry::ProvenDescendant,
- Some(_) => ElementAncestry::OutsideTargetWindow,
- None => ElementAncestry::Unproven,
+ let element = element_ptr.map(|ptr| {
+ let element = ptr as AXUIElementRef;
+ let window = element_window_id(element);
+ // A menu-bar item has no window ancestor: the menu bar hangs
+ // off the application. It still belongs to this app.
+ let in_menu_bar = window.is_none() && element_in_app_menu_bar(app, element);
+ element_ancestry(window, in_menu_bar, window_id)
});
let focused_window_id = copy_element_attr(app, "AXFocusedWindow").and_then(|window| {
let window_id = ax_get_window_id(window);
@@ -318,6 +378,16 @@ mod tests {
)
}
+ #[test]
+ fn menu_bar_item_is_proven_only_through_the_app_menu_bar() {
+ use super::element_ancestry;
+ use cua_driver_core::background_input::ElementAncestry::*;
+ assert_eq!(element_ancestry(None, true, 7), ProvenDescendant);
+ assert_eq!(element_ancestry(None, false, 7), Unproven);
+ assert_eq!(element_ancestry(Some(7), false, 7), ProvenDescendant);
+ assert_eq!(element_ancestry(Some(8), true, 7), OutsideTargetWindow);
+ }
+
#[test]
fn window_maps_to_itself() {
assert_eq!(resolve(&[("AXWindow", None, Some(167))], 0), Some(167));
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/browser/ax_page_reader.rs b/libs/cua-driver/rust/crates/platform-macos/src/browser/ax_page_reader.rs
index 8c35cfac7f..bb017e0418 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/browser/ax_page_reader.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/browser/ax_page_reader.rs
@@ -29,6 +29,33 @@ const SKIP_ROLES: &[&str] = &[
const TEXT_ROLES: &[&str] = &["AXStaticText", "AXHeading", "AXWebArea"];
impl AXPageReader {
+ /// The part of a window-scoped AX tree that is page content: every
+ /// `AXWebArea` row and the rows nested under it. A window-scoped walk also
+ /// carries the app's menu bar and the browser's own toolbar, which are not
+ /// the page; reading them as page text answers a `get_text` or `query_dom`
+ /// with "Apple, About This Mac, ...". `None` when the tree has no web area
+ /// (the page's accessibility tree is not built, or the window is not a
+ /// page).
+ pub fn web_area_markdown(tree_markdown: &str) -> Option<String> {
+ let mut out: Vec<&str> = Vec::new();
+ let mut area_indent: Option<usize> = None;
+ for line in tree_markdown.lines() {
+ let indent = line.len() - line.trim_start().len();
+ if let Some(base) = area_indent {
+ if indent > base {
+ out.push(line);
+ continue;
+ }
+ area_indent = None;
+ }
+ if parse_ax_line(line).is_some_and(|el| el.role == "AXWebArea") {
+ area_indent = Some(indent);
+ out.push(line);
+ }
+ }
+ (!out.is_empty()).then(|| out.join("\n"))
+ }
+
/// Extract readable text from an AX tree markdown string.
pub fn extract_text(tree_markdown: &str) -> String {
let mut lines_out: Vec<String> = Vec::new();
@@ -220,3 +247,40 @@ fn css_selector_to_ax_roles(selector: &str) -> Vec<&'static str> {
_ => vec![],
}
}
+
+#[cfg(test)]
+mod web_area_tests {
+ use super::AXPageReader;
+
+ const TREE: &str = "\
+- [0] AXMenuBar
+ - [1] AXMenuBarItem \"Apple\"
+ - [2] AXMenuItem \"About This Mac\"
+- [3] AXWindow \"Orders - Chrome\"
+ - [4] AXToolbar
+ - [5] AXTextField \"Address and search bar\"
+ - [6] AXWebArea \"Orders\"
+ - [7] AXHeading \"Your loans\"
+ - [8] AXStaticText \"Hedge trimmer, due 3 Nov\"
+ - [9] AXButton \"Back\"
+";
+
+ #[test]
+ fn web_area_markdown_keeps_only_page_rows() {
+ let page = AXPageReader::web_area_markdown(TREE).expect("web area");
+ assert!(page.contains("Hedge trimmer"));
+ assert!(page.contains("Your loans"));
+ assert!(!page.contains("About This Mac"));
+ assert!(!page.contains("Address and search bar"));
+ assert!(!page.contains("\"Back\""));
+ let text = AXPageReader::extract_text(&page);
+ assert!(text.contains("Hedge trimmer, due 3 Nov"));
+ assert!(!text.contains("Apple"));
+ }
+
+ #[test]
+ fn a_tree_without_a_web_area_has_no_page_content() {
+ let tree = "- [0] AXMenuBar\n - [1] AXMenuBarItem \"Apple\"\n- [2] AXWindow \"x\"\n";
+ assert_eq!(AXPageReader::web_area_markdown(tree), None);
+ }
+}
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/browser/cdp_client.rs b/libs/cua-driver/rust/crates/platform-macos/src/browser/cdp_client.rs
index 17e0521b55..9bcde9898d 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/browser/cdp_client.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/browser/cdp_client.rs
@@ -150,6 +150,11 @@ impl CdpSessionCache {
Ok(arc)
}
+ /// Whether a connection for `port` is already open (and so approved).
+ pub async fn has_session(&self, port: u16) -> bool {
+ self.sessions.lock().await.contains_key(&port)
+ }
+
async fn evict(&self, port: u16) {
self.sessions.lock().await.remove(&port);
}
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/click.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/click.rs
index 161b13f5a1..dc8c4745cb 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/click.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/click.rs
@@ -1425,6 +1425,31 @@ impl Tool for ClickTool {
///
/// `element` is the cached AX element pointer and its snapshot index; `window`
/// is the target (pid, window_id).
+/// Whether a background AXPress needs its window to be the app's key window.
+///
+/// A toolbar or format-bar control in a native app sends its action to the
+/// first responder through the key window (TextEdit's Bold checkbox applies
+/// the trait to the text view that way). With the app behind the user's
+/// window nothing is key, so the press flips the control but the document
+/// does not change. Browsers and Electron shells run page content in their own
+/// process and take no part in this.
+fn press_needs_key_window(action: &str, front_pid: Option<i32>, pid: i32, native: bool) -> bool {
+ action == "AXPress" && native && front_pid != Some(pid)
+}
+
+/// Make the target window AppKit-key without raising it, activating the app
+/// or taking key status from the user's window (the focus record only, as the
+/// background pixel click does), then let AppKit settle.
+fn prime_key_window_for_background_press(pid: i32, window_id: u32, action: &str) {
+ let native = !super::type_text::is_browser_pid(pid)
+ && !crate::browser::electron_js::ElectronJs::is_electron(pid);
+ if press_needs_key_window(action, apps::frontmost_pid(), pid, native)
+ && crate::input::skylight::activate_without_raise_with(pid, window_id, false)
+ {
+ std::thread::sleep(std::time::Duration::from_millis(80));
+ }
+}
+
fn perform_ax_click(
element: (usize, usize),
window: (i32, u32),
@@ -1575,6 +1600,9 @@ fn perform_ax_click(
let toggle_before = is_toggle_press(ax_action, &role, &advertised)
.then(read_value)
.flatten();
+ if !foreground {
+ prime_key_window_for_background_press(pid, window_id, ax_action);
+ }
let err = unsafe { crate::ax::bindings::perform_action(element, ax_action) };
if err != crate::ax::bindings::kAXErrorSuccess {
// Checked before the row-selection fallback, which writes AXSelected
@@ -1830,6 +1858,20 @@ mod tests {
/// Surface 5: schema must advertise the new `button` field with the three
/// canonical values and default to "left". Hermes / Codex / Claude Code
/// consumers branch on this enum being present.
+ #[test]
+ fn background_press_needs_the_key_window_only_in_a_native_background_app() {
+ assert!(press_needs_key_window("AXPress", Some(1), 2, true));
+ assert!(
+ !press_needs_key_window("AXPress", Some(2), 2, true),
+ "frontmost"
+ );
+ assert!(
+ !press_needs_key_window("AXPress", Some(1), 2, false),
+ "browser or Electron"
+ );
+ assert!(!press_needs_key_window("AXShowMenu", Some(1), 2, true));
+ }
+
#[test]
fn schema_advertises_button_enum() {
let d = def();
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/invoke_menu.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/invoke_menu.rs
index 352fe4c3dc..d913e6670a 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/invoke_menu.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/invoke_menu.rs
@@ -577,6 +577,11 @@ unsafe fn item_is_save(item: AXUIElementRef) -> bool {
== 0.0
}
+/// How long a disabled item gets to enable after its window becomes key, and
+/// how often it is re-read.
+const ENABLE_WAIT: Duration = Duration::from_millis(600);
+const ENABLE_POLL: Duration = Duration::from_millis(40);
+
/// Resolve the whole path from the app's live menu bar without opening any
/// level, then perform the final item's action. Used for a background app:
/// opening a menu-bar menu would take key focus and draw over the user's
@@ -611,7 +616,13 @@ unsafe fn invoke_path_in_background(
if copy_bool_attr(target, "AXEnabled") == Some(false)
&& crate::input::skylight::activate_without_raise_with(pid, window_id, false)
{
- std::thread::sleep(Duration::from_millis(120));
+ // AppKit revalidates the item once the window is key; wait for it
+ // instead of reading the stale flag after a fixed pause.
+ let mut waited = Duration::ZERO;
+ while waited < ENABLE_WAIT && copy_bool_attr(target, "AXEnabled") == Some(false) {
+ std::thread::sleep(ENABLE_POLL);
+ waited += ENABLE_POLL;
+ }
}
if copy_bool_attr(target, "AXEnabled") == Some(false) {
return Err(BackgroundPressError::Disabled {
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/page.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/page.rs
index 07be33d842..4fe5804934 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/page.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/page.rs
@@ -25,6 +25,17 @@ impl MacOsPageBackend {
Self { state }
}
+ /// Evaluate `js` through a CDP session the driver already holds for this
+ /// browser. Never opens a new connection: a new one makes Chrome show its
+ /// "Allow remote debugging?" prompt and pull itself to the front.
+ async fn attached_cdp_evaluate(&self, pid: i32, js: &str) -> Option<String> {
+ let port = CdpClient::find_port_for_pid(pid).await?;
+ if !self.state.cdp_sessions.has_session(port).await {
+ return None;
+ }
+ self.state.cdp_sessions.evaluate(js, port, None).await.ok()
+ }
+
/// Resolve `bundle_id` for `pid` via the running-apps list.
async fn bundle_id_for(pid: i32) -> String {
tokio::task::spawn_blocking(move || {
@@ -50,13 +61,22 @@ impl PageBackend for MacOsPageBackend {
.unwrap_or(false);
if use_ax_fallback {
- return ax_text_fallback(pid, window_id).await;
+ return ax_text_fallback(pid, window_id, false).await;
}
- // Try JS path first; on error, fall back to AX walk.
- match execute_js("document.body.innerText", &bundle_id, pid, window_id).await {
+ // Try JS path first; on error, use a CDP session the driver already
+ // holds, then the window's web area.
+ let js = "document.body.innerText";
+ match execute_js(js, &bundle_id, pid, window_id).await {
Ok(result) => Ok(result),
- Err(_) => ax_text_fallback(pid, window_id).await,
+ Err(error) => {
+ if let Some(result) = self.attached_cdp_evaluate(pid, js).await {
+ return Ok(result);
+ }
+ ax_text_fallback(pid, window_id, BrowserJs::supports(&bundle_id))
+ .await
+ .map_err(|ax| anyhow::anyhow!("{ax} (JavaScript read failed first: {error})"))
+ }
}
}
@@ -75,15 +95,25 @@ impl PageBackend for MacOsPageBackend {
.unwrap_or(false);
if use_ax_fallback {
- let results = ax_query_fallback(pid, window_id, css_selector).await?;
+ let results = ax_query_fallback(pid, window_id, css_selector, false).await?;
return Ok(format_ax_elements(&results));
}
let js = build_query_selector_js(css_selector, attributes);
match execute_js(&js, &bundle_id, pid, window_id).await {
Ok(result) => Ok(result),
- Err(_) => {
- let results = ax_query_fallback(pid, window_id, css_selector).await?;
+ Err(error) => {
+ if let Some(result) = self.attached_cdp_evaluate(pid, &js).await {
+ return Ok(result);
+ }
+ let results = ax_query_fallback(
+ pid,
+ window_id,
+ css_selector,
+ BrowserJs::supports(&bundle_id),
+ )
+ .await
+ .map_err(|ax| anyhow::anyhow!("{ax} (JavaScript read failed first: {error})"))?;
Ok(format_ax_elements(&results))
}
}
@@ -307,15 +337,37 @@ async fn execute_js(js: &str, bundle_id: &str, pid: i32, window_id: u64) -> anyh
anyhow::bail!("Unsupported browser: bundle_id={bundle_id}");
}
+/// The tree rows that are page content. A browser window's walk also holds
+/// the app's menu bar and the browser toolbar, so a browser read is scoped to
+/// the window's `AXWebArea` and fails when there is none, instead of
+/// returning the menu bar as if it were the page. Other web-view apps keep
+/// the whole tree when they expose no web area.
+fn page_rows(tree_markdown: String, require_web_area: bool) -> anyhow::Result<String> {
+ match AXPageReader::web_area_markdown(&tree_markdown) {
+ Some(page) => Ok(page),
+ None if require_web_area => anyhow::bail!(
+ "This window's page content is not readable: its accessibility tree has no \
+ AXWebArea (only the browser's own controls and the menu bar). Read it again in \
+ a moment, use get_browser_state, or attach CDP with browser_prepare."
+ ),
+ None => Ok(tree_markdown),
+ }
+}
+
/// Extract page text via the AX tree.
-async fn ax_text_fallback(pid: i32, window_id: u64) -> anyhow::Result<String> {
+async fn ax_text_fallback(
+ pid: i32,
+ window_id: u64,
+ require_web_area: bool,
+) -> anyhow::Result<String> {
let window_id = u32::try_from(window_id)
.map_err(|_| anyhow::anyhow!("macOS window_id {window_id} is out of u32 range"))?;
let result =
tokio::task::spawn_blocking(move || crate::ax::tree::walk_tree(pid, Some(window_id), None))
.await
.map_err(|e| anyhow::anyhow!("AX walk task failed: {e}"))?;
- Ok(AXPageReader::extract_text(&result.tree_markdown))
+ let rows = page_rows(result.tree_markdown, require_web_area)?;
+ Ok(AXPageReader::extract_text(&rows))
}
/// Query AX tree by CSS selector.
@@ -323,6 +375,7 @@ async fn ax_query_fallback(
pid: i32,
window_id: u64,
selector: &str,
+ require_web_area: bool,
) -> anyhow::Result<Vec<crate::browser::ax_page_reader::AXElement>> {
let window_id = u32::try_from(window_id)
.map_err(|_| anyhow::anyhow!("macOS window_id {window_id} is out of u32 range"))?;
@@ -331,7 +384,8 @@ async fn ax_query_fallback(
tokio::task::spawn_blocking(move || crate::ax::tree::walk_tree(pid, Some(window_id), None))
.await
.map_err(|e| anyhow::anyhow!("AX walk task failed: {e}"))?;
- Ok(AXPageReader::query(&sel, &result.tree_markdown))
+ let rows = page_rows(result.tree_markdown, require_web_area)?;
+ Ok(AXPageReader::query(&sel, &rows))
}
fn format_ax_elements(elements: &[crate::browser::ax_page_reader::AXElement]) -> String {
@@ -421,3 +475,31 @@ fn required_finite(value: &serde_json::Value, key: &str, raw: &str) -> anyhow::R
)
})
}
+
+#[cfg(test)]
+mod tests {
+ use super::page_rows;
+
+ const MENU_ONLY: &str = "- [0] AXMenuBar\n - [1] AXMenuBarItem \"Apple\"\n";
+
+ #[test]
+ fn browser_read_without_a_web_area_fails_instead_of_returning_the_menu_bar() {
+ let error = page_rows(MENU_ONLY.to_owned(), true)
+ .unwrap_err()
+ .to_string();
+ assert!(error.contains("AXWebArea"), "{error}");
+ }
+
+ #[test]
+ fn browser_read_is_scoped_to_the_web_area() {
+ let tree = format!("{MENU_ONLY}- [2] AXWindow \"w\"\n - [3] AXWebArea \"p\"\n - [4] AXStaticText \"hello\"\n");
+ let page = page_rows(tree, true).unwrap();
+ assert!(page.contains("hello"));
+ assert!(!page.contains("Apple"));
+ }
+
+ #[test]
+ fn web_view_apps_keep_the_whole_tree_without_a_web_area() {
+ assert_eq!(page_rows(MENU_ONLY.to_owned(), false).unwrap(), MENU_ONLY);
+ }
+}