Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
fix(cua-driver): resolve sheet elements to their parent window (#4507)
ddupont030d5a0c05e0
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/ax/exact_target.rs b/libs/cua-driver/rust/crates/platform-macos/src/ax/exact_target.rsindex 79fe5b577b..3e7bda2227 100644--- a/libs/cua-driver/rust/crates/platform-macos/src/ax/exact_target.rs+++ b/libs/cua-driver/rust/crates/platform-macos/src/ax/exact_target.rs@@ -16,23 +16,33 @@ use super::bindings::{ax_get_window_id, copy_ax_windows_including, copy_bool_attr, copy_element_attr,copy_string_attr, focused_element_of_pid, AXUIElementCreateApplication, AXUIElementRef,};+use super::snapshot::RetainedElement;use crate::windows::{all_windows, resolve_window_owner, WindowOwner};/// Bounded `AXParent` ascent used when an element does not expose `AXWindow`.const MAX_ANCESTRY_DEPTH: usize = 40;+/// Bounded sheet chain followed when mapping a sheet to its parent window.+const MAX_SHEET_NESTING: usize = 8;+/// Resolve the CGWindowID of the top-level AX window that owns `element`.////// Prefers the element's `AXWindow` attribute and falls back to a bounded/// `AXParent` walk. `None` means ancestry could not be proven — callers must/// treat that as "not the requested window", never as a wildcard.///+/// A sheet is its own WindowServer window (and a remote Open/Save panel's+/// content lives in yet another, service-owned one), but it is modal to the+/// window it is attached to and is observed inside that window's snapshot.+/// An element under an `AXSheet` therefore resolves to the sheet's parent+/// window when AX proves that parent (issue #4392).+////// # Safety////// `element` must be a valid `AXUIElementRef` for the duration of the call.pub unsafe fn element_window_id(element: AXUIElementRef) -> Option<u32> {if let Some(window) = copy_element_attr(element, "AXWindow") {- let window_id = ax_get_window_id(window);+ let window_id = owning_window_id(window);CFRelease(window as CFTypeRef);if window_id.is_some() {return window_id;@@ -45,7 +55,7 @@ pub unsafe fn element_window_id(element: AXUIElementRef) -> Option<u32> {for _ in 0..MAX_ANCESTRY_DEPTH {match copy_string_attr(current, "AXRole").as_deref() {Some("AXWindow") | Some("AXSheet") => {- resolved = ax_get_window_id(current);+ resolved = owning_window_id(current);break;}Some("AXApplication") | None => break,@@ -67,6 +77,59 @@ pub unsafe fn element_window_id(element: AXUIElementRef) -> Option<u32> {resolved}+/// Map a window-role AX element to the CGWindowID of the top-level window it+/// belongs to: an `AXWindow` maps to itself, an `AXSheet` to the window it is+/// attached to (see [`element_window_id`]).+///+/// # Safety+///+/// `window` must be a valid `AXUIElementRef` for the duration of the call.+unsafe fn owning_window_id(window: AXUIElementRef) -> Option<u32> {+ resolve_owning_window(+ RetainedElement::retain(window as usize),+ MAX_SHEET_NESTING,+ |element| copy_string_attr(element.as_ptr() as AXUIElementRef, "AXRole"),+ |element| {+ copy_element_attr(element.as_ptr() as AXUIElementRef, "AXParent").map(|parent| {+ // copy_element_attr returned +1; hand that reference over.+ let retained = RetainedElement::retain(parent as usize);+ CFRelease(parent as CFTypeRef);+ retained+ })+ },+ |element| ax_get_window_id(element.as_ptr() as AXUIElementRef),+ )+}++/// Pure sheet-to-parent resolution behind [`owning_window_id`].+///+/// A sheet whose parent is not a window (or a sheet attached to another+/// sheet) is followed up to `max_nesting` levels. When no parent window is+/// proven, the sheet keeps its own window id: that never names a different+/// window than before, so it can only stay refused, never widen.+fn resolve_owning_window<E>(+ start: E,+ max_nesting: usize,+ role: impl Fn(&E) -> Option<String>,+ parent: impl Fn(&E) -> Option<E>,+ window_id: impl Fn(&E) -> Option<u32>,+) -> Option<u32> {+ let own_id = window_id(&start);+ let mut current = start;+ for _ in 0..=max_nesting {+ match role(¤t).as_deref() {+ Some("AXSheet") => {}+ Some("AXWindow") => return window_id(¤t),+ _ => return own_id,+ }+ current = match parent(¤t) {+ Some(next) => next,+ None => return own_id,+ };+ }+ own_id+}+/// The process's focused AX element, but only when it provably belongs to the/// requested window. Returns a retained element the caller must release.///@@ -206,7 +269,66 @@ pub fn gather_background_facts(#[cfg(test)]mod tests {- use super::{count_competing_keyboard_destinations, AxWindowRecord};+ use super::{count_competing_keyboard_destinations, resolve_owning_window, AxWindowRecord};++ /// A fake AX node: (role, AXParent index, CGWindowID).+ type FakeNode = (&'static str, Option<usize>, Option<u32>);++ fn resolve(tree: &[FakeNode], start: usize) -> Option<u32> {+ resolve_owning_window(+ start,+ 8,+ |&index| Some(tree[index].0.to_owned()),+ |&index| tree[index].1,+ |&index| tree[index].2,+ )+ }++ #[test]+ fn window_maps_to_itself() {+ assert_eq!(resolve(&[("AXWindow", None, Some(167))], 0), Some(167));+ }++ #[test]+ fn sheet_maps_to_the_window_it_is_attached_to() {+ // Open panel sheet 172 attached to window 167 (issue #4392).+ let tree = [+ ("AXWindow", None, Some(167)),+ ("AXSheet", Some(0), Some(172)),+ ];+ assert_eq!(resolve(&tree, 1), Some(167));+ }++ #[test]+ fn nested_sheets_map_to_the_root_window() {+ let tree = [+ ("AXWindow", None, Some(167)),+ ("AXSheet", Some(0), Some(172)),+ ("AXSheet", Some(1), Some(180)),+ ];+ assert_eq!(resolve(&tree, 2), Some(167));+ }++ #[test]+ fn sheet_without_a_proven_parent_window_keeps_its_own_id() {+ // A top-level consent sheet whose parent is the application, and a+ // sheet whose parent cannot be read, never resolve to another window.+ let tree = [+ ("AXApplication", None, None),+ ("AXSheet", Some(0), Some(172)),+ ];+ assert_eq!(resolve(&tree, 1), Some(172));+ assert_eq!(resolve(&[("AXSheet", None, Some(172))], 0), Some(172));+ }++ #[test]+ fn sheet_chain_is_bounded() {+ let tree = [+ ("AXSheet", Some(1), Some(172)),+ ("AXSheet", Some(0), Some(173)),+ ];+ assert_eq!(resolve(&tree, 0), Some(172));+ }fn ax_window(window_id: u32, minimized: Option<bool>) -> AxWindowRecord {AxWindowRecord {