Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
fix(cua-driver): leave focus on a dialog the clicked app opened
injaneity6effe99e5743
diff --git a/libs/cua-driver/rust/crates/platform-linux/src/input/mod.rs b/libs/cua-driver/rust/crates/platform-linux/src/input/mod.rsindex 7a0cd50dae..729cacaeda 100644--- a/libs/cua-driver/rust/crates/platform-linux/src/input/mod.rs+++ b/libs/cua-driver/rust/crates/platform-linux/src/input/mod.rs@@ -321,6 +321,10 @@ fn master_keyboard_device_name(master_name: &str) -> String {}fn open_display() -> Result<*mut x11::xlib::Display> {+ open_named_display(ptr::null())+}++fn open_named_display(name: *const std::ffi::c_char) -> Result<*mut x11::xlib::Display> {match XLIB_THREADS_READY.get_or_init(|| {let rc = unsafe { x11::xlib::XInitThreads() };if rc == 0 {@@ -332,7 +336,7 @@ fn open_display() -> Result<*mut x11::xlib::Display> {Ok(()) => {}Err(err) => bail!("{err}"),}- let display = unsafe { x11::xlib::XOpenDisplay(ptr::null()) };+ let display = unsafe { x11::xlib::XOpenDisplay(name) };if display.is_null() {bail!("XOpenDisplay returned null");}@@ -2861,6 +2865,23 @@ fn restore_focus_state(display: *mut x11::xlib::Display, saved: &SavedFocus) {// ours has landed — so don't stop at first success: require the// active window to hold stable for consecutive checks, re-sending on// every regression, within a bounded budget.+ let name = unsafe { std::ffi::CStr::from_ptr(x11::xlib::XDisplayString(display)) };+ let conn = name+ .to_str()+ .ok()+ .and_then(|name| RustConnection::connect(Some(name)).ok())+ .map(|(conn, _)| conn);+ let prev_pid = conn+ .as_ref()+ .zip(u32::try_from(prev).ok())+ .and_then(|(conn, prev)| crate::x11::get_window_pid(conn, prev).ok().flatten());+ let own_dialog = |now: Option<x11::xlib::Window>| match (&conn, prev_pid, now) {+ (Some(conn), Some(pid), Some(now)) => now != prev && is_own_dialog(conn, pid, now),+ _ => false,+ };+ if own_dialog(ewmh_active_window(display)) {+ return;+ }sleep(Duration::from_millis(300));let mut stable = 0;for attempt in 0..15 {@@ -2870,6 +2891,8 @@ fn restore_focus_state(display: *mut x11::xlib::Display, saved: &SavedFocus) {if stable >= 3 {return;}+ } else if own_dialog(now) {+ return;} else {stable = 0;// MPX clicks can leave a core-protocol WM believing the@@ -2931,6 +2954,36 @@ fn restore_focus_state(display: *mut x11::xlib::Display, saved: &SavedFocus) {}}+fn is_own_dialog(conn: &RustConnection, previous_pid: u32, active: x11::xlib::Window) -> bool {+ let Ok(active) = u32::try_from(active) else {+ return false;+ };+ if crate::x11::get_window_pid(conn, active).ok().flatten() != Some(previous_pid) {+ return false;+ }+ let windows = |property: Atom, kind: Atom| -> Vec<u32> {+ conn.get_property(false, active, property, kind, 0, 32)+ .ok()+ .and_then(|cookie| cookie.reply().ok())+ .and_then(|reply| reply.value32().map(|values| values.collect()))+ .unwrap_or_default()+ };+ if windows(AtomEnum::WM_TRANSIENT_FOR.into(), AtomEnum::WINDOW.into())+ .iter()+ .any(|&parent| parent != x11rb::NONE)+ {+ return true;+ }+ let atom = |name: &str| crate::x11::get_atom(conn, name).ok();+ let (Some(window_type), Some(dialog)) = (+ atom("_NET_WM_WINDOW_TYPE"),+ atom("_NET_WM_WINDOW_TYPE_DIALOG"),+ ) else {+ return false;+ };+ windows(window_type, AtomEnum::ATOM.into()).contains(&dialog)+}+#[derive(Clone, Copy, Debug)]struct EventTarget {window: Window,@@ -4185,6 +4238,149 @@ pub(crate) fn key_name_to_keysym(key: &str) -> Result<u32> {Ok(keysym)}+#[cfg(test)]+mod own_dialog_tests {+ use super::*;+ use std::io::{BufRead, BufReader};+ use std::process::{Command, Stdio};+ use x11rb::protocol::{xproto::*, Event};+ use x11rb::wrapper::ConnectionExt as _;++ struct Server(std::process::Child);+ impl Drop for Server {+ fn drop(&mut self) {+ let _ = self.0.kill();+ let _ = self.0.wait();+ }+ }+ fn set32(conn: &RustConnection, win: u32, name: &str, ty: AtomEnum, values: &[u32]) {+ let atom = crate::x11::get_atom(conn, name).unwrap();+ conn.change_property32(PropMode::REPLACE, win, atom, ty, values)+ .unwrap();+ }+ #[test]+ fn focus_restore_leaves_focus_on_a_dialog_the_app_opened() {+ let mut server = Server(+ Command::new("Xvfb")+ .args(["-displayfd", "1", "-nolisten", "tcp"])+ .stdout(Stdio::piped())+ .spawn()+ .expect("install xvfb to run focus tests"),+ );+ let mut number = String::new();+ BufReader::new(server.0.stdout.take().unwrap())+ .read_line(&mut number)+ .unwrap();+ let name = format!(":{}", number.trim());+ let (conn, screen) = RustConnection::connect(Some(&name)).unwrap();+ let root = conn.setup().roots[screen].root;+ let window = |pid, transient| {+ let win = conn.generate_id().unwrap();+ conn.create_window(+ 0,+ win,+ root,+ 0,+ 0,+ 50,+ 50,+ 0,+ WindowClass::INPUT_OUTPUT,+ 0,+ &CreateWindowAux::new(),+ )+ .unwrap();+ if pid != 0 {+ set32(&conn, win, "_NET_WM_PID", AtomEnum::CARDINAL, &[pid]);+ }+ if transient != 0 {+ set32(+ &conn,+ win,+ "WM_TRANSIENT_FOR",+ AtomEnum::WINDOW,+ &[transient],+ );+ }+ conn.map_window(win).unwrap();+ win+ };+ let main = window(4242, 0);+ let dialog = window(4242, main);+ let sibling = window(4242, 0);+ let foreign = window(4343, main);+ let typed = window(4242, 0);+ let pidless = window(0, main);+ set32(+ &conn,+ typed,+ "_NET_WM_WINDOW_TYPE",+ AtomEnum::ATOM,+ &[crate::x11::get_atom(&conn, "_NET_WM_WINDOW_TYPE_DIALOG").unwrap()],+ );+ assert!(is_own_dialog(&conn, 4242, dialog.into()));+ assert!(is_own_dialog(&conn, 4242, typed.into()));+ for win in [sibling, foreign, pidless] {+ assert!(!is_own_dialog(&conn, 4242, win.into()));+ }+ conn.get_input_focus().unwrap().reply().unwrap();+ let cname = std::ffi::CString::new(name.clone()).unwrap();+ let display = open_named_display(cname.as_ptr()).unwrap();+ // The WM honors every activation, except the first delayed request opens a dialog.+ for delayed in [false, true] {+ let (wm, _) = RustConnection::connect(Some(&name)).unwrap();+ wm.change_window_attributes(+ root,+ &ChangeWindowAttributesAux::new().event_mask(EventMask::SUBSTRUCTURE_REDIRECT),+ )+ .unwrap()+ .check()+ .unwrap();+ set32(+ &conn,+ root,+ "_NET_ACTIVE_WINDOW",+ AtomEnum::WINDOW,+ &[sibling],+ );+ conn.set_input_focus(InputFocus::PARENT, sibling, 0u32)+ .unwrap();+ conn.get_input_focus().unwrap().reply().unwrap();+ let stop = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false));+ let done = stop.clone();+ let thread = std::thread::spawn(move || {+ let mut first = delayed;+ while !done.load(std::sync::atomic::Ordering::Relaxed) {+ match wm.poll_for_event().unwrap() {+ Some(Event::ClientMessage(event)) => {+ let target = if first { dialog } else { event.window };+ first = false;+ set32(&wm, root, "_NET_ACTIVE_WINDOW", AtomEnum::WINDOW, &[target]);+ wm.set_input_focus(InputFocus::PARENT, target, 0u32)+ .unwrap();+ wm.flush().unwrap();+ }+ _ => std::thread::sleep(Duration::from_millis(1)),+ }+ }+ });+ restore_focus_state(+ display,+ &SavedFocus {+ ewmh_active: Some(main.into()),+ core_focus: 0,+ core_revert_to: 0,+ },+ );+ let focus = conn.get_input_focus().unwrap().reply().unwrap().focus;+ stop.store(true, std::sync::atomic::Ordering::Relaxed);+ thread.join().unwrap();+ assert_eq!(focus, if delayed { dialog } else { main });+ }+ unsafe { x11::xlib::XCloseDisplay(display) };+ }+}+#[cfg(test)]mod key_name_alias_tests {use super::{diff --git a/nix/cua-driver/tests/rust-unit.nix b/nix/cua-driver/tests/rust-unit.nixindex 01b911fa71..6a77169078 100644--- a/nix/cua-driver/tests/rust-unit.nix+++ b/nix/cua-driver/tests/rust-unit.nix@@ -46,7 +46,7 @@ pkgs.rustPlatform.buildRustPackage {python3];# Transport tests launch an isolated D-Bus session instead of using the host bus.- nativeCheckInputs = [ pkgs.dbus ];+ nativeCheckInputs = [ pkgs.dbus pkgs.xorg.xorgserver ];CUA_TEST_PYTHON = "${pkgs.python3}/bin/python3";CUA_TEST_PYTHON_DYNAMIC_LINKER = pkgs.stdenv.cc.bintools.dynamicLinker;CUA_TEST_GLIBC_STATIC_LIB = "${pkgs.glibc.static}/lib";@@ -62,7 +62,7 @@ pkgs.rustPlatform.buildRustPackage {libei];- # The default Cargo test set is deliberately headless. The ignored desktop+ # Focus unit tests launch private Xvfb servers. The ignored desktop# matrix is run by the manual Linux e2e workflow and is not hidden in Nix.doCheck = true;