Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
fix(cua-driver): discover native Wayland windows in desktop snapshots
injaneitya038036c705a
diff --git a/docs/content/docs/cua-driver/reference/mcp-tools/apps-windows-and-screen.mdx b/docs/content/docs/cua-driver/reference/mcp-tools/apps-windows-and-screen.mdxindex 9a95e02b86..8bc33dfc1d 100644--- a/docs/content/docs/cua-driver/reference/mcp-tools/apps-windows-and-screen.mdx+++ b/docs/content/docs/cua-driver/reference/mcp-tools/apps-windows-and-screen.mdx@@ -557,7 +557,7 @@ For the full AX subtree of a single window (with interactive element indices you</Tab><Tab value="Linux">-Return a lightweight snapshot of the desktop: running processes and on-screen visible X11 windows with their bounds and owner pid.+Return a lightweight snapshot of the desktop: running processes and on-screen visible windows with their bounds and owner pid.For the full AT-SPI subtree of a single window (with interactive element indices you can click by), use get_window_state instead: this is a fast discovery read.diff --git a/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs b/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rsindex 3be700798b..06b3fa4062 100644--- a/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs+++ b/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs@@ -637,6 +637,15 @@ fn window_record_json(w: &crate::x11::WindowInfo) -> Value {})}+fn visible_live_desktop_windows(+ mut windows: Vec<crate::x11::WindowInfo>,+) -> Vec<crate::x11::WindowInfo> {+ windows.retain(|window| {+ window.is_on_screen && window.pid.is_none_or(crate::proc_fs::is_process_live)+ });+ windows+}+#[cfg(test)]mod list_windows_tests;@@ -13511,7 +13520,7 @@ impl Tool for GetAccessibilityTreeTool {GAX_DEF.get_or_init(|| ToolDef {name: "get_accessibility_tree".into(),description: "Return a lightweight snapshot of the desktop: running processes and \- on-screen visible X11 windows with their bounds and owner pid.\n\n\+ on-screen visible windows with their bounds and owner pid.\n\n\For the full AT-SPI subtree of a single window (with interactive element indices \you can click by), use get_window_state instead — this is a fast discovery read.".into(),@@ -13526,12 +13535,16 @@ impl Tool for GetAccessibilityTreeTool {let (procs, windows) = tokio::task::spawn_blocking(|| {(crate::proc_fs::list_processes(),- crate::x11::list_windows(None),+ visible_live_desktop_windows(crate::wayland::list_windows_dispatch(None)),)}).await.unwrap_or_default();+ if crate::wayland::is_wayland() {+ crate::wayland::remember_observed_window_origins(&windows);+ }+let mut lines = vec![format!("{} running process(es), {} visible window(s)",procs.len(),@@ -13572,10 +13585,7 @@ impl Tool for GetAccessibilityTreeTool {let structured = json!({"processes": procs.iter().map(|p| json!({"pid":p.pid,"name":p.name})).collect::<Vec<_>>(),- "windows": windows.iter().map(|w| json!({- "window_id": w.xid, "pid": w.pid, "title": w.title,- "x": w.x, "y": w.y, "width": w.width, "height": w.height- })).collect::<Vec<_>>()+ "windows": windows.iter().map(window_record_json).collect::<Vec<_>>()});ToolResult::text(lines.join("\n")).with_structured(structured)}diff --git a/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_/list_windows_tests.rs b/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_/list_windows_tests.rsindex d0f9b01cf1..e94134275a 100644--- a/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_/list_windows_tests.rs+++ b/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_/list_windows_tests.rs@@ -57,6 +57,42 @@ fn unavailable_wayland_order_serializes_as_null() {assert_eq!(window_record_json(&w)["z_index"], Value::Null);}+#[test]+fn desktop_discovery_retains_only_visible_windows_with_live_or_unknown_owners() {+ let window = |xid, pid, is_on_screen| crate::x11::WindowInfo {+ xid,+ pid,+ app_name: "native-wayland-app".to_owned(),+ title: "Example".to_owned(),+ is_on_screen,+ z_index: None,+ x: 10,+ y: 20,+ width: 300,+ height: 400,+ };+ let windows = visible_live_desktop_windows(vec![+ window(1, Some(std::process::id()), true),+ window(2, Some(std::process::id()), false),+ window(3, Some(u32::MAX), true),+ window(4, None, true),+ window(5, None, false),+ ]);++ assert_eq!(+ windows.iter().map(|w| w.xid).collect::<Vec<_>>(),+ vec![1, 4]+ );+ let record = window_record_json(&windows[0]);+ assert_eq!(+ record["bounds"],+ json!({"x": 10, "y": 20, "width": 300, "height": 400})+ );+ assert_eq!(record["app_name"], json!("native-wayland-app"));+ assert_eq!(record["is_on_screen"], json!(true));+ assert_eq!(record["z_index"], Value::Null);+}+#[test]fn chromium_launch_detection_uses_executable_basename() {assert!(chromium_family_program("/usr/bin/google-chrome-stable"));diff --git a/scripts/docs-generators/cli-specs/cua-driver-mcp-linux.json b/scripts/docs-generators/cli-specs/cua-driver-mcp-linux.jsonindex 73074144f0..02235f56bd 100644--- a/scripts/docs-generators/cli-specs/cua-driver-mcp-linux.json+++ b/scripts/docs-generators/cli-specs/cua-driver-mcp-linux.json@@ -2246,7 +2246,7 @@"read_only": false},{- "description": "Return a lightweight snapshot of the desktop: running processes and on-screen visible X11 windows with their bounds and owner pid.\n\nFor the full AT-SPI subtree of a single window (with interactive element indices you can click by), use get_window_state instead — this is a fast discovery read.",+ "description": "Return a lightweight snapshot of the desktop: running processes and on-screen visible windows with their bounds and owner pid.\n\nFor the full AT-SPI subtree of a single window (with interactive element indices you can click by), use get_window_state instead — this is a fast discovery read.","destructive": false,"idempotent": true,"input_schema": {