Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

fix(cua-driver): recover rootless XWayland browser identities

injaneityef61a985d589
diff --git a/libs/cua-driver/rust/crates/platform-linux/src/browser_platform.rs b/libs/cua-driver/rust/crates/platform-linux/src/browser_platform.rs
index ddecce24e4..e6b96c5d31 100644
--- a/libs/cua-driver/rust/crates/platform-linux/src/browser_platform.rs
+++ b/libs/cua-driver/rust/crates/platform-linux/src/browser_platform.rs
@@ -48,6 +48,50 @@ fn refusal(code: BrowserRefusalCode, message: impl Into<String>) -> BrowserRefus
BrowserRefusal::new(code, message)
}
+async fn x11_native_window(pid: i64, window_id: u64) -> Option<(NativeWindowInfo, bool)> {
+ let pid_u32 = u32::try_from(pid).ok()?;
+ tokio::task::spawn_blocking(move || {
+ select_x11_window(pid, window_id, crate::x11::list_windows(Some(pid_u32)))
+ })
+ .await
+ .ok()
+ .flatten()
+}
+
+fn select_x11_window(
+ pid: i64,
+ window_id: u64,
+ windows: Vec<crate::x11::WindowInfo>,
+) -> Option<(NativeWindowInfo, bool)> {
+ let only = windows.len() == 1;
+ let window = windows
+ .into_iter()
+ .find(|w| w.xid == window_id && w.pid.map(i64::from) == Some(pid))?;
+ Some((
+ NativeWindowInfo {
+ pid,
+ window_id,
+ title: window.title,
+ bounds: Rect::new(
+ f64::from(window.x),
+ f64::from(window.y),
+ f64::from(window.width),
+ f64::from(window.height),
+ ),
+ geometry_exact: true,
+ ownership: NativeOwnershipProof {
+ method: NativeOwnershipMethod::PlatformAttested,
+ owner_pid: pid,
+ detail: Some(
+ "X11 _NET_WM_PID corroborated independently by the owned endpoint pid"
+ .to_owned(),
+ ),
+ },
+ },
+ only,
+ ))
+}
+
fn hyprland_identity_matches(pid: u32, window_id: u64, owner_pid: u32, address: u64) -> bool {
pid != 0 && window_id != 0 && owner_pid == pid && address == window_id
}
@@ -607,6 +651,9 @@ impl BrowserPlatform for LinuxBrowserPlatform {
pid: i64,
window_id: u64,
) -> Result<NativeWindowInfo, BrowserRefusal> {
+ if let Some((window, _)) = x11_native_window(pid, window_id).await {
+ return Ok(window);
+ }
let pid_u32 = u32::try_from(pid).map_err(|_| {
refusal(
BrowserRefusalCode::BrowserWrongTargetRefused,
@@ -739,39 +786,11 @@ impl BrowserPlatform for LinuxBrowserPlatform {
});
}
- let window = tokio::task::spawn_blocking(move || {
- crate::x11::list_windows(Some(pid_u32))
- .into_iter()
- .find(|window| window.xid == window_id)
- })
- .await
- .ok()
- .flatten()
- .ok_or_else(|| {
+ Err({
refusal(
BrowserRefusalCode::BrowserBindingStale,
format!("X11 window {window_id} is not owned by pid {pid}"),
)
- })?;
- Ok(NativeWindowInfo {
- pid,
- window_id,
- title: window.title,
- bounds: Rect::new(
- f64::from(window.x),
- f64::from(window.y),
- f64::from(window.width),
- f64::from(window.height),
- ),
- geometry_exact: true,
- ownership: NativeOwnershipProof {
- method: NativeOwnershipMethod::PlatformAttested,
- owner_pid: pid,
- detail: Some(
- "X11 _NET_WM_PID corroborated independently by the owned endpoint pid"
- .to_owned(),
- ),
- },
})
}
@@ -780,6 +799,9 @@ impl BrowserPlatform for LinuxBrowserPlatform {
pid: i64,
window_id: u64,
) -> Result<Option<bool>, BrowserRefusal> {
+ if let Some((_, only)) = x11_native_window(pid, window_id).await {
+ return Ok(Some(only));
+ }
let pid_u32 = u32::try_from(pid).map_err(|_| {
refusal(
BrowserRefusalCode::BrowserWrongTargetRefused,
@@ -845,20 +867,7 @@ impl BrowserPlatform for LinuxBrowserPlatform {
.collect::<Vec<_>>();
return Ok(Some(owned.len() == 1 && owned[0] == window_id));
}
- let owned = tokio::task::spawn_blocking(move || {
- crate::x11::list_windows(Some(pid_u32))
- .into_iter()
- .map(|window| window.xid)
- .collect::<Vec<_>>()
- })
- .await
- .map_err(|error| {
- refusal(
- BrowserRefusalCode::BrowserRouteUnavailable,
- format!("could not enumerate X11 browser windows: {error}"),
- )
- })?;
- Ok(Some(owned.len() == 1 && owned[0] == window_id))
+ Ok(Some(false))
}
async fn discover_owned_endpoint(
@@ -1324,6 +1333,40 @@ impl BrowserPlatform for LinuxBrowserPlatform {
mod tests {
use super::*;
+ #[test]
+ fn x11_identity_and_cardinality_precede_compositor_selection() {
+ let window = |xid| crate::x11::WindowInfo {
+ xid,
+ pid: Some(42),
+ app_name: String::new(),
+ title: "browser".into(),
+ is_on_screen: true,
+ z_index: None,
+ x: 1,
+ y: 2,
+ width: 3,
+ height: 4,
+ };
+ let (native, only) = select_x11_window(42, 7, vec![window(7)]).unwrap();
+ assert!(native.geometry_exact && only);
+ assert_eq!(native.window_id, 7);
+ assert!(
+ !select_x11_window(42, 7, vec![window(7), window(8)])
+ .unwrap()
+ .1
+ );
+ for id in [8, u64::from(u32::MAX) + 1] {
+ let mut compositor_id = None;
+ let result = select_x11_window(42, id, vec![window(7)]).or_else(|| {
+ compositor_id = Some(id);
+ None
+ });
+ assert!(result.is_none());
+ assert_eq!(compositor_id, Some(id));
+ }
+ assert!(select_x11_window(43, 7, vec![window(7)]).is_none());
+ }
+
#[test]
fn hyprland_browser_identity_requires_exact_pid_and_full_native_address() {
let address = 0x1234_0000_0042;
diff --git a/libs/cua-driver/rust/crates/platform-linux/src/x11/mod.rs b/libs/cua-driver/rust/crates/platform-linux/src/x11/mod.rs
index a1bdf0b73e..166cd512da 100644
--- a/libs/cua-driver/rust/crates/platform-linux/src/x11/mod.rs
+++ b/libs/cua-driver/rust/crates/platform-linux/src/x11/mod.rs
@@ -169,6 +169,7 @@ pub(crate) fn get_window_list(
conn: &RustConnection,
root: Window,
) -> Result<(Vec<Window>, WindowSource)> {
+ let mut ewmh = false;
let atom_names = ["_NET_CLIENT_LIST_STACKING", "_NET_CLIENT_LIST"];
for name in &atom_names {
if let Ok(atom) = get_atom(conn, name) {
@@ -176,6 +177,7 @@ pub(crate) fn get_window_list(
.get_property(false, root, atom, AtomEnum::WINDOW, 0, u32::MAX)?
.reply()
{
+ ewmh |= reply.type_ != x11rb::NONE;
let windows: Vec<Window> = reply
.value32()
.map(|iter| iter.collect())
@@ -187,10 +189,64 @@ pub(crate) fn get_window_list(
}
}
- // No EWMH client-list property means there may be no window manager. In
- // that case only expose mapped root children; unmapped Electron children
- // can otherwise be reported before a late-starting WM reparents them.
let tree = conn.query_tree(root)?.reply()?;
+ let state = get_atom(conn, "WM_STATE")?;
+ let pid = get_atom(conn, "_NET_WM_PID")?;
+ let name = get_atom(conn, "_NET_WM_NAME")?;
+ let clients = walk_clients(&tree.children, 1, &mut 0, &mut |windows| {
+ let requests = windows
+ .iter()
+ .map(|&w| {
+ Ok((
+ conn.get_window_attributes(w)?,
+ conn.get_property(false, w, state, state, 0, 2)?,
+ conn.query_tree(w)?,
+ ))
+ })
+ .collect::<Result<Vec<_>>>()?;
+ requests
+ .into_iter()
+ .zip(windows)
+ .map(|((attrs, wm, tree), &w)| {
+ let valid = attrs
+ .reply()
+ .ok()
+ .is_some_and(|a| a.map_state == MapState::VIEWABLE)
+ && wm.reply().ok().is_some_and(|r| {
+ valid_wm_state(
+ r.value32()
+ .map(|v| v.collect::<Vec<_>>())
+ .unwrap_or_default()
+ .as_slice(),
+ )
+ });
+ let client = if valid {
+ let owner = conn.get_property(false, w, pid, AtomEnum::CARDINAL, 0, 1)?;
+ let title = conn.get_property(false, w, name, AtomEnum::ANY, 0, 1024)?;
+ let legacy =
+ conn.get_property(false, w, AtomEnum::WM_NAME, AtomEnum::ANY, 0, 1024)?;
+ let owner = owner
+ .reply()
+ .ok()
+ .and_then(|r| r.value32().and_then(|mut v| v.next()));
+ let title = title
+ .reply()
+ .ok()
+ .filter(|r| !r.value.is_empty())
+ .or_else(|| legacy.reply().ok());
+ title.is_some_and(|r| {
+ client_has_identity(owner, &String::from_utf8_lossy(&r.value))
+ })
+ } else {
+ false
+ };
+ Ok((tree.reply().map(|r| r.children).unwrap_or_default(), client))
+ })
+ .collect()
+ })?;
+ if let Some(clients) = clients.filter(|clients| ewmh || !clients.is_empty()) {
+ return Ok((clients, WindowSource::ClientList));
+ }
let windows = tree
.children
.into_iter()
@@ -206,7 +262,41 @@ pub(crate) fn get_window_list(
}
fn client_list_property(property_type: Atom, windows: &[Window]) -> Option<&[Window]> {
- (property_type != x11rb::NONE).then_some(windows)
+ (property_type != x11rb::NONE && !windows.is_empty()).then_some(windows)
+}
+
+fn client_has_identity(pid: Option<u32>, title: &str) -> bool {
+ pid.is_some() && !title.trim_end_matches('\0').trim().is_empty()
+}
+
+fn valid_wm_state(values: &[u32]) -> bool {
+ values.len() >= 2 && matches!(values[0], 1 | 3)
+}
+
+fn walk_clients(
+ windows: &[Window],
+ depth: usize,
+ queued: &mut usize,
+ lookup: &mut impl FnMut(&[Window]) -> Result<Vec<(Vec<Window>, bool)>>,
+) -> Result<Option<Vec<Window>>> {
+ if windows.is_empty() {
+ return Ok(Some(Vec::new()));
+ }
+ if depth > 8 || windows.len() > 1024 - *queued {
+ return Ok(None);
+ }
+ *queued += windows.len();
+ let mut clients = Vec::new();
+ for (&window, (children, client)) in windows.iter().zip(lookup(windows)?) {
+ if client {
+ clients.push(window);
+ } else if let Some(found) = walk_clients(&children, depth + 1, queued, lookup)? {
+ clients.extend(found);
+ } else {
+ return Ok(None);
+ }
+ }
+ Ok(Some(clients))
}
fn fallback_window_is_listable(map_state: MapState) -> bool {
@@ -746,8 +836,101 @@ mod tests {
use super::*;
#[test]
- fn empty_present_client_list_does_not_fall_back_to_query_tree() {
- assert_eq!(client_list_property(1, &[]), Some([].as_slice()));
+ fn empty_present_client_list_allows_tree_fallback() {
+ assert_eq!(client_list_property(1, &[]), None);
+ }
+
+ #[test]
+ fn framed_clients_are_depth_first_and_stop_at_clients() {
+ let found = walk_clients(&[1, 2], 1, &mut 0, &mut |ids| {
+ Ok(ids
+ .iter()
+ .map(|id| match id {
+ 1 => (vec![3], false),
+ 3 => (vec![4], false),
+ 4 => (vec![99], true),
+ 2 => (vec![], true),
+ _ => panic!("rendering descendant visited"),
+ })
+ .collect())
+ })
+ .unwrap();
+ assert_eq!(found, Some(vec![4, 2]));
+ assert_eq!(client_list_property(1, &[7]), Some([7].as_slice()));
+ }
+
+ #[test]
+ fn traversal_exhaustion_discards_partial_clients() {
+ let mut queued = 0;
+ assert_eq!(
+ walk_clients(&vec![7; 1024], 1, &mut queued, &mut |ids| {
+ Ok(ids.iter().map(|_| (vec![], true)).collect())
+ })
+ .unwrap()
+ .unwrap()
+ .len(),
+ 1024
+ );
+ assert_eq!(queued, 1024);
+ let mut calls = 0;
+ assert!(walk_clients(&vec![7; 1025], 1, &mut 0, &mut |_| {
+ calls += 1;
+ Ok(vec![])
+ })
+ .unwrap()
+ .is_none());
+ assert_eq!(calls, 0);
+ for children in [vec![1; 1025], vec![1]] {
+ let mut calls = 0;
+ let result = walk_clients(&children, 1, &mut 0, &mut |ids| {
+ calls += 1;
+ Ok(ids.iter().map(|_| (vec![1], false)).collect())
+ })
+ .unwrap();
+ assert_eq!(result, None);
+ assert!(calls <= 8);
+ }
+ assert_eq!(
+ walk_clients(&[1], 1, &mut 0, &mut |_| Ok(vec![(vec![], false)])).unwrap(),
+ Some(vec![])
+ );
+ }
+
+ #[test]
+ fn no_wm_budget_exhaustion_keeps_complete_root_children() {
+ let roots = vec![1, 2];
+ let clients = walk_clients(&roots, 1, &mut 0, &mut |ids| {
+ Ok(ids.iter().map(|_| (vec![3; 1024], false)).collect())
+ })
+ .unwrap();
+ assert!(clients.is_none());
+ let selected = clients
+ .filter(|clients| !clients.is_empty())
+ .unwrap_or_else(|| {
+ roots
+ .iter()
+ .copied()
+ .filter(|_| fallback_window_is_listable(MapState::VIEWABLE))
+ .collect()
+ });
+ assert_eq!(selected, roots);
+ }
+
+ #[test]
+ fn clients_require_pid_and_nonblank_title() {
+ assert!(!client_has_identity(None, "browser"));
+ assert!(!client_has_identity(Some(42), ""));
+ assert!(!client_has_identity(Some(42), " \t"));
+ assert!(client_has_identity(Some(42), "browser"));
+ }
+
+ #[test]
+ fn only_normal_and_iconic_wm_states_are_clients() {
+ for state in [vec![], vec![1], vec![0, 0], vec![2, 0]] {
+ assert!(!valid_wm_state(&state));
+ }
+ assert!(valid_wm_state(&[1, 0]));
+ assert!(valid_wm_state(&[3, 0]));
}
#[test]