Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
fix(cua-driver): recover rootless XWayland browser identities
injaneityef61a985d589
diff --git a/libs/cua-driver/rust/crates/platform-linux/src/browser_platform.rs b/libs/cua-driver/rust/crates/platform-linux/src/browser_platform.rsindex ddecce24e4..e6b96c5d31 100644--- a/libs/cua-driver/rust/crates/platform-linux/src/browser_platform.rs+++ b/libs/cua-driver/rust/crates/platform-linux/src/browser_platform.rs@@ -48,6 +48,50 @@ fn refusal(code: BrowserRefusalCode, message: impl Into<String>) -> BrowserRefusBrowserRefusal::new(code, message)}+async fn x11_native_window(pid: i64, window_id: u64) -> Option<(NativeWindowInfo, bool)> {+ let pid_u32 = u32::try_from(pid).ok()?;+ tokio::task::spawn_blocking(move || {+ select_x11_window(pid, window_id, crate::x11::list_windows(Some(pid_u32)))+ })+ .await+ .ok()+ .flatten()+}++fn select_x11_window(+ pid: i64,+ window_id: u64,+ windows: Vec<crate::x11::WindowInfo>,+) -> Option<(NativeWindowInfo, bool)> {+ let only = windows.len() == 1;+ let window = windows+ .into_iter()+ .find(|w| w.xid == window_id && w.pid.map(i64::from) == Some(pid))?;+ Some((+ NativeWindowInfo {+ pid,+ window_id,+ title: window.title,+ bounds: Rect::new(+ f64::from(window.x),+ f64::from(window.y),+ f64::from(window.width),+ f64::from(window.height),+ ),+ geometry_exact: true,+ ownership: NativeOwnershipProof {+ method: NativeOwnershipMethod::PlatformAttested,+ owner_pid: pid,+ detail: Some(+ "X11 _NET_WM_PID corroborated independently by the owned endpoint pid"+ .to_owned(),+ ),+ },+ },+ only,+ ))+}+fn hyprland_identity_matches(pid: u32, window_id: u64, owner_pid: u32, address: u64) -> bool {pid != 0 && window_id != 0 && owner_pid == pid && address == window_id}@@ -607,6 +651,9 @@ impl BrowserPlatform for LinuxBrowserPlatform {pid: i64,window_id: u64,) -> Result<NativeWindowInfo, BrowserRefusal> {+ if let Some((window, _)) = x11_native_window(pid, window_id).await {+ return Ok(window);+ }let pid_u32 = u32::try_from(pid).map_err(|_| {refusal(BrowserRefusalCode::BrowserWrongTargetRefused,@@ -739,39 +786,11 @@ impl BrowserPlatform for LinuxBrowserPlatform {});}- let window = tokio::task::spawn_blocking(move || {- crate::x11::list_windows(Some(pid_u32))- .into_iter()- .find(|window| window.xid == window_id)- })- .await- .ok()- .flatten()- .ok_or_else(|| {+ Err({refusal(BrowserRefusalCode::BrowserBindingStale,format!("X11 window {window_id} is not owned by pid {pid}"),)- })?;- Ok(NativeWindowInfo {- pid,- window_id,- title: window.title,- bounds: Rect::new(- f64::from(window.x),- f64::from(window.y),- f64::from(window.width),- f64::from(window.height),- ),- geometry_exact: true,- ownership: NativeOwnershipProof {- method: NativeOwnershipMethod::PlatformAttested,- owner_pid: pid,- detail: Some(- "X11 _NET_WM_PID corroborated independently by the owned endpoint pid"- .to_owned(),- ),- },})}@@ -780,6 +799,9 @@ impl BrowserPlatform for LinuxBrowserPlatform {pid: i64,window_id: u64,) -> Result<Option<bool>, BrowserRefusal> {+ if let Some((_, only)) = x11_native_window(pid, window_id).await {+ return Ok(Some(only));+ }let pid_u32 = u32::try_from(pid).map_err(|_| {refusal(BrowserRefusalCode::BrowserWrongTargetRefused,@@ -845,20 +867,7 @@ impl BrowserPlatform for LinuxBrowserPlatform {.collect::<Vec<_>>();return Ok(Some(owned.len() == 1 && owned[0] == window_id));}- let owned = tokio::task::spawn_blocking(move || {- crate::x11::list_windows(Some(pid_u32))- .into_iter()- .map(|window| window.xid)- .collect::<Vec<_>>()- })- .await- .map_err(|error| {- refusal(- BrowserRefusalCode::BrowserRouteUnavailable,- format!("could not enumerate X11 browser windows: {error}"),- )- })?;- Ok(Some(owned.len() == 1 && owned[0] == window_id))+ Ok(Some(false))}async fn discover_owned_endpoint(@@ -1324,6 +1333,40 @@ impl BrowserPlatform for LinuxBrowserPlatform {mod tests {use super::*;+ #[test]+ fn x11_identity_and_cardinality_precede_compositor_selection() {+ let window = |xid| crate::x11::WindowInfo {+ xid,+ pid: Some(42),+ app_name: String::new(),+ title: "browser".into(),+ is_on_screen: true,+ z_index: None,+ x: 1,+ y: 2,+ width: 3,+ height: 4,+ };+ let (native, only) = select_x11_window(42, 7, vec![window(7)]).unwrap();+ assert!(native.geometry_exact && only);+ assert_eq!(native.window_id, 7);+ assert!(+ !select_x11_window(42, 7, vec![window(7), window(8)])+ .unwrap()+ .1+ );+ for id in [8, u64::from(u32::MAX) + 1] {+ let mut compositor_id = None;+ let result = select_x11_window(42, id, vec![window(7)]).or_else(|| {+ compositor_id = Some(id);+ None+ });+ assert!(result.is_none());+ assert_eq!(compositor_id, Some(id));+ }+ assert!(select_x11_window(43, 7, vec![window(7)]).is_none());+ }+#[test]fn hyprland_browser_identity_requires_exact_pid_and_full_native_address() {let address = 0x1234_0000_0042;diff --git a/libs/cua-driver/rust/crates/platform-linux/src/x11/mod.rs b/libs/cua-driver/rust/crates/platform-linux/src/x11/mod.rsindex a1bdf0b73e..166cd512da 100644--- a/libs/cua-driver/rust/crates/platform-linux/src/x11/mod.rs+++ b/libs/cua-driver/rust/crates/platform-linux/src/x11/mod.rs@@ -169,6 +169,7 @@ pub(crate) fn get_window_list(conn: &RustConnection,root: Window,) -> Result<(Vec<Window>, WindowSource)> {+ let mut ewmh = false;let atom_names = ["_NET_CLIENT_LIST_STACKING", "_NET_CLIENT_LIST"];for name in &atom_names {if let Ok(atom) = get_atom(conn, name) {@@ -176,6 +177,7 @@ pub(crate) fn get_window_list(.get_property(false, root, atom, AtomEnum::WINDOW, 0, u32::MAX)?.reply(){+ ewmh |= reply.type_ != x11rb::NONE;let windows: Vec<Window> = reply.value32().map(|iter| iter.collect())@@ -187,10 +189,64 @@ pub(crate) fn get_window_list(}}- // No EWMH client-list property means there may be no window manager. In- // that case only expose mapped root children; unmapped Electron children- // can otherwise be reported before a late-starting WM reparents them.let tree = conn.query_tree(root)?.reply()?;+ let state = get_atom(conn, "WM_STATE")?;+ let pid = get_atom(conn, "_NET_WM_PID")?;+ let name = get_atom(conn, "_NET_WM_NAME")?;+ let clients = walk_clients(&tree.children, 1, &mut 0, &mut |windows| {+ let requests = windows+ .iter()+ .map(|&w| {+ Ok((+ conn.get_window_attributes(w)?,+ conn.get_property(false, w, state, state, 0, 2)?,+ conn.query_tree(w)?,+ ))+ })+ .collect::<Result<Vec<_>>>()?;+ requests+ .into_iter()+ .zip(windows)+ .map(|((attrs, wm, tree), &w)| {+ let valid = attrs+ .reply()+ .ok()+ .is_some_and(|a| a.map_state == MapState::VIEWABLE)+ && wm.reply().ok().is_some_and(|r| {+ valid_wm_state(+ r.value32()+ .map(|v| v.collect::<Vec<_>>())+ .unwrap_or_default()+ .as_slice(),+ )+ });+ let client = if valid {+ let owner = conn.get_property(false, w, pid, AtomEnum::CARDINAL, 0, 1)?;+ let title = conn.get_property(false, w, name, AtomEnum::ANY, 0, 1024)?;+ let legacy =+ conn.get_property(false, w, AtomEnum::WM_NAME, AtomEnum::ANY, 0, 1024)?;+ let owner = owner+ .reply()+ .ok()+ .and_then(|r| r.value32().and_then(|mut v| v.next()));+ let title = title+ .reply()+ .ok()+ .filter(|r| !r.value.is_empty())+ .or_else(|| legacy.reply().ok());+ title.is_some_and(|r| {+ client_has_identity(owner, &String::from_utf8_lossy(&r.value))+ })+ } else {+ false+ };+ Ok((tree.reply().map(|r| r.children).unwrap_or_default(), client))+ })+ .collect()+ })?;+ if let Some(clients) = clients.filter(|clients| ewmh || !clients.is_empty()) {+ return Ok((clients, WindowSource::ClientList));+ }let windows = tree.children.into_iter()@@ -206,7 +262,41 @@ pub(crate) fn get_window_list(}fn client_list_property(property_type: Atom, windows: &[Window]) -> Option<&[Window]> {- (property_type != x11rb::NONE).then_some(windows)+ (property_type != x11rb::NONE && !windows.is_empty()).then_some(windows)+}++fn client_has_identity(pid: Option<u32>, title: &str) -> bool {+ pid.is_some() && !title.trim_end_matches('\0').trim().is_empty()+}++fn valid_wm_state(values: &[u32]) -> bool {+ values.len() >= 2 && matches!(values[0], 1 | 3)+}++fn walk_clients(+ windows: &[Window],+ depth: usize,+ queued: &mut usize,+ lookup: &mut impl FnMut(&[Window]) -> Result<Vec<(Vec<Window>, bool)>>,+) -> Result<Option<Vec<Window>>> {+ if windows.is_empty() {+ return Ok(Some(Vec::new()));+ }+ if depth > 8 || windows.len() > 1024 - *queued {+ return Ok(None);+ }+ *queued += windows.len();+ let mut clients = Vec::new();+ for (&window, (children, client)) in windows.iter().zip(lookup(windows)?) {+ if client {+ clients.push(window);+ } else if let Some(found) = walk_clients(&children, depth + 1, queued, lookup)? {+ clients.extend(found);+ } else {+ return Ok(None);+ }+ }+ Ok(Some(clients))}fn fallback_window_is_listable(map_state: MapState) -> bool {@@ -746,8 +836,101 @@ mod tests {use super::*;#[test]- fn empty_present_client_list_does_not_fall_back_to_query_tree() {- assert_eq!(client_list_property(1, &[]), Some([].as_slice()));+ fn empty_present_client_list_allows_tree_fallback() {+ assert_eq!(client_list_property(1, &[]), None);+ }++ #[test]+ fn framed_clients_are_depth_first_and_stop_at_clients() {+ let found = walk_clients(&[1, 2], 1, &mut 0, &mut |ids| {+ Ok(ids+ .iter()+ .map(|id| match id {+ 1 => (vec![3], false),+ 3 => (vec![4], false),+ 4 => (vec![99], true),+ 2 => (vec![], true),+ _ => panic!("rendering descendant visited"),+ })+ .collect())+ })+ .unwrap();+ assert_eq!(found, Some(vec![4, 2]));+ assert_eq!(client_list_property(1, &[7]), Some([7].as_slice()));+ }++ #[test]+ fn traversal_exhaustion_discards_partial_clients() {+ let mut queued = 0;+ assert_eq!(+ walk_clients(&vec![7; 1024], 1, &mut queued, &mut |ids| {+ Ok(ids.iter().map(|_| (vec![], true)).collect())+ })+ .unwrap()+ .unwrap()+ .len(),+ 1024+ );+ assert_eq!(queued, 1024);+ let mut calls = 0;+ assert!(walk_clients(&vec![7; 1025], 1, &mut 0, &mut |_| {+ calls += 1;+ Ok(vec![])+ })+ .unwrap()+ .is_none());+ assert_eq!(calls, 0);+ for children in [vec![1; 1025], vec![1]] {+ let mut calls = 0;+ let result = walk_clients(&children, 1, &mut 0, &mut |ids| {+ calls += 1;+ Ok(ids.iter().map(|_| (vec![1], false)).collect())+ })+ .unwrap();+ assert_eq!(result, None);+ assert!(calls <= 8);+ }+ assert_eq!(+ walk_clients(&[1], 1, &mut 0, &mut |_| Ok(vec![(vec![], false)])).unwrap(),+ Some(vec![])+ );+ }++ #[test]+ fn no_wm_budget_exhaustion_keeps_complete_root_children() {+ let roots = vec![1, 2];+ let clients = walk_clients(&roots, 1, &mut 0, &mut |ids| {+ Ok(ids.iter().map(|_| (vec![3; 1024], false)).collect())+ })+ .unwrap();+ assert!(clients.is_none());+ let selected = clients+ .filter(|clients| !clients.is_empty())+ .unwrap_or_else(|| {+ roots+ .iter()+ .copied()+ .filter(|_| fallback_window_is_listable(MapState::VIEWABLE))+ .collect()+ });+ assert_eq!(selected, roots);+ }++ #[test]+ fn clients_require_pid_and_nonblank_title() {+ assert!(!client_has_identity(None, "browser"));+ assert!(!client_has_identity(Some(42), ""));+ assert!(!client_has_identity(Some(42), " \t"));+ assert!(client_has_identity(Some(42), "browser"));+ }++ #[test]+ fn only_normal_and_iconic_wm_states_are_clients() {+ for state in [vec![], vec![1], vec![0, 0], vec![2, 0]] {+ assert!(!valid_wm_state(&state));+ }+ assert!(valid_wm_state(&[1, 0]));+ assert!(valid_wm_state(&[3, 0]));}#[test]