Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

fix(cua-driver): discover native Wayland windows in desktop snapshots

injaneitya038036c705a
diff --git a/docs/content/docs/cua-driver/reference/mcp-tools/apps-windows-and-screen.mdx b/docs/content/docs/cua-driver/reference/mcp-tools/apps-windows-and-screen.mdx
index 9a95e02b86..8bc33dfc1d 100644
--- a/docs/content/docs/cua-driver/reference/mcp-tools/apps-windows-and-screen.mdx
+++ b/docs/content/docs/cua-driver/reference/mcp-tools/apps-windows-and-screen.mdx
@@ -557,7 +557,7 @@ For the full AX subtree of a single window (with interactive element indices you
</Tab>
<Tab value="Linux">
-Return a lightweight snapshot of the desktop: running processes and on-screen visible X11 windows with their bounds and owner pid.
+Return a lightweight snapshot of the desktop: running processes and on-screen visible windows with their bounds and owner pid.
For the full AT-SPI subtree of a single window (with interactive element indices you can click by), use get_window_state instead: this is a fast discovery read.
diff --git a/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs b/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs
index 3be700798b..06b3fa4062 100644
--- a/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs
+++ b/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs
@@ -637,6 +637,15 @@ fn window_record_json(w: &crate::x11::WindowInfo) -> Value {
})
}
+fn visible_live_desktop_windows(
+ mut windows: Vec<crate::x11::WindowInfo>,
+) -> Vec<crate::x11::WindowInfo> {
+ windows.retain(|window| {
+ window.is_on_screen && window.pid.is_none_or(crate::proc_fs::is_process_live)
+ });
+ windows
+}
+
#[cfg(test)]
mod list_windows_tests;
@@ -13511,7 +13520,7 @@ impl Tool for GetAccessibilityTreeTool {
GAX_DEF.get_or_init(|| ToolDef {
name: "get_accessibility_tree".into(),
description: "Return a lightweight snapshot of the desktop: running processes and \
- on-screen visible X11 windows with their bounds and owner pid.\n\n\
+ on-screen visible windows with their bounds and owner pid.\n\n\
For the full AT-SPI subtree of a single window (with interactive element indices \
you can click by), use get_window_state instead — this is a fast discovery read."
.into(),
@@ -13526,12 +13535,16 @@ impl Tool for GetAccessibilityTreeTool {
let (procs, windows) = tokio::task::spawn_blocking(|| {
(
crate::proc_fs::list_processes(),
- crate::x11::list_windows(None),
+ visible_live_desktop_windows(crate::wayland::list_windows_dispatch(None)),
)
})
.await
.unwrap_or_default();
+ if crate::wayland::is_wayland() {
+ crate::wayland::remember_observed_window_origins(&windows);
+ }
+
let mut lines = vec![format!(
"{} running process(es), {} visible window(s)",
procs.len(),
@@ -13572,10 +13585,7 @@ impl Tool for GetAccessibilityTreeTool {
let structured = json!({
"processes": procs.iter().map(|p| json!({"pid":p.pid,"name":p.name})).collect::<Vec<_>>(),
- "windows": windows.iter().map(|w| json!({
- "window_id": w.xid, "pid": w.pid, "title": w.title,
- "x": w.x, "y": w.y, "width": w.width, "height": w.height
- })).collect::<Vec<_>>()
+ "windows": windows.iter().map(window_record_json).collect::<Vec<_>>()
});
ToolResult::text(lines.join("\n")).with_structured(structured)
}
diff --git a/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_/list_windows_tests.rs b/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_/list_windows_tests.rs
index d0f9b01cf1..e94134275a 100644
--- a/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_/list_windows_tests.rs
+++ b/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_/list_windows_tests.rs
@@ -57,6 +57,42 @@ fn unavailable_wayland_order_serializes_as_null() {
assert_eq!(window_record_json(&w)["z_index"], Value::Null);
}
+#[test]
+fn desktop_discovery_retains_only_visible_windows_with_live_or_unknown_owners() {
+ let window = |xid, pid, is_on_screen| crate::x11::WindowInfo {
+ xid,
+ pid,
+ app_name: "native-wayland-app".to_owned(),
+ title: "Example".to_owned(),
+ is_on_screen,
+ z_index: None,
+ x: 10,
+ y: 20,
+ width: 300,
+ height: 400,
+ };
+ let windows = visible_live_desktop_windows(vec![
+ window(1, Some(std::process::id()), true),
+ window(2, Some(std::process::id()), false),
+ window(3, Some(u32::MAX), true),
+ window(4, None, true),
+ window(5, None, false),
+ ]);
+
+ assert_eq!(
+ windows.iter().map(|w| w.xid).collect::<Vec<_>>(),
+ vec![1, 4]
+ );
+ let record = window_record_json(&windows[0]);
+ assert_eq!(
+ record["bounds"],
+ json!({"x": 10, "y": 20, "width": 300, "height": 400})
+ );
+ assert_eq!(record["app_name"], json!("native-wayland-app"));
+ assert_eq!(record["is_on_screen"], json!(true));
+ assert_eq!(record["z_index"], Value::Null);
+}
+
#[test]
fn chromium_launch_detection_uses_executable_basename() {
assert!(chromium_family_program("/usr/bin/google-chrome-stable"));
diff --git a/scripts/docs-generators/cli-specs/cua-driver-mcp-linux.json b/scripts/docs-generators/cli-specs/cua-driver-mcp-linux.json
index 73074144f0..02235f56bd 100644
--- a/scripts/docs-generators/cli-specs/cua-driver-mcp-linux.json
+++ b/scripts/docs-generators/cli-specs/cua-driver-mcp-linux.json
@@ -2246,7 +2246,7 @@
"read_only": false
},
{
- "description": "Return a lightweight snapshot of the desktop: running processes and on-screen visible X11 windows with their bounds and owner pid.\n\nFor the full AT-SPI subtree of a single window (with interactive element indices you can click by), use get_window_state instead — this is a fast discovery read.",
+ "description": "Return a lightweight snapshot of the desktop: running processes and on-screen visible windows with their bounds and owner pid.\n\nFor the full AT-SPI subtree of a single window (with interactive element indices you can click by), use get_window_state instead — this is a fast discovery read.",
"destructive": false,
"idempotent": true,
"input_schema": {