Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

fix(cua-driver): leave focus on a dialog the clicked app opened

injaneity6effe99e5743
diff --git a/libs/cua-driver/rust/crates/platform-linux/src/input/mod.rs b/libs/cua-driver/rust/crates/platform-linux/src/input/mod.rs
index 7a0cd50dae..729cacaeda 100644
--- a/libs/cua-driver/rust/crates/platform-linux/src/input/mod.rs
+++ b/libs/cua-driver/rust/crates/platform-linux/src/input/mod.rs
@@ -321,6 +321,10 @@ fn master_keyboard_device_name(master_name: &str) -> String {
}
fn open_display() -> Result<*mut x11::xlib::Display> {
+ open_named_display(ptr::null())
+}
+
+fn open_named_display(name: *const std::ffi::c_char) -> Result<*mut x11::xlib::Display> {
match XLIB_THREADS_READY.get_or_init(|| {
let rc = unsafe { x11::xlib::XInitThreads() };
if rc == 0 {
@@ -332,7 +336,7 @@ fn open_display() -> Result<*mut x11::xlib::Display> {
Ok(()) => {}
Err(err) => bail!("{err}"),
}
- let display = unsafe { x11::xlib::XOpenDisplay(ptr::null()) };
+ let display = unsafe { x11::xlib::XOpenDisplay(name) };
if display.is_null() {
bail!("XOpenDisplay returned null");
}
@@ -2861,6 +2865,23 @@ fn restore_focus_state(display: *mut x11::xlib::Display, saved: &SavedFocus) {
// ours has landed — so don't stop at first success: require the
// active window to hold stable for consecutive checks, re-sending on
// every regression, within a bounded budget.
+ let name = unsafe { std::ffi::CStr::from_ptr(x11::xlib::XDisplayString(display)) };
+ let conn = name
+ .to_str()
+ .ok()
+ .and_then(|name| RustConnection::connect(Some(name)).ok())
+ .map(|(conn, _)| conn);
+ let prev_pid = conn
+ .as_ref()
+ .zip(u32::try_from(prev).ok())
+ .and_then(|(conn, prev)| crate::x11::get_window_pid(conn, prev).ok().flatten());
+ let own_dialog = |now: Option<x11::xlib::Window>| match (&conn, prev_pid, now) {
+ (Some(conn), Some(pid), Some(now)) => now != prev && is_own_dialog(conn, pid, now),
+ _ => false,
+ };
+ if own_dialog(ewmh_active_window(display)) {
+ return;
+ }
sleep(Duration::from_millis(300));
let mut stable = 0;
for attempt in 0..15 {
@@ -2870,6 +2891,8 @@ fn restore_focus_state(display: *mut x11::xlib::Display, saved: &SavedFocus) {
if stable >= 3 {
return;
}
+ } else if own_dialog(now) {
+ return;
} else {
stable = 0;
// MPX clicks can leave a core-protocol WM believing the
@@ -2931,6 +2954,36 @@ fn restore_focus_state(display: *mut x11::xlib::Display, saved: &SavedFocus) {
}
}
+fn is_own_dialog(conn: &RustConnection, previous_pid: u32, active: x11::xlib::Window) -> bool {
+ let Ok(active) = u32::try_from(active) else {
+ return false;
+ };
+ if crate::x11::get_window_pid(conn, active).ok().flatten() != Some(previous_pid) {
+ return false;
+ }
+ let windows = |property: Atom, kind: Atom| -> Vec<u32> {
+ conn.get_property(false, active, property, kind, 0, 32)
+ .ok()
+ .and_then(|cookie| cookie.reply().ok())
+ .and_then(|reply| reply.value32().map(|values| values.collect()))
+ .unwrap_or_default()
+ };
+ if windows(AtomEnum::WM_TRANSIENT_FOR.into(), AtomEnum::WINDOW.into())
+ .iter()
+ .any(|&parent| parent != x11rb::NONE)
+ {
+ return true;
+ }
+ let atom = |name: &str| crate::x11::get_atom(conn, name).ok();
+ let (Some(window_type), Some(dialog)) = (
+ atom("_NET_WM_WINDOW_TYPE"),
+ atom("_NET_WM_WINDOW_TYPE_DIALOG"),
+ ) else {
+ return false;
+ };
+ windows(window_type, AtomEnum::ATOM.into()).contains(&dialog)
+}
+
#[derive(Clone, Copy, Debug)]
struct EventTarget {
window: Window,
@@ -4185,6 +4238,149 @@ pub(crate) fn key_name_to_keysym(key: &str) -> Result<u32> {
Ok(keysym)
}
+#[cfg(test)]
+mod own_dialog_tests {
+ use super::*;
+ use std::io::{BufRead, BufReader};
+ use std::process::{Command, Stdio};
+ use x11rb::protocol::{xproto::*, Event};
+ use x11rb::wrapper::ConnectionExt as _;
+
+ struct Server(std::process::Child);
+ impl Drop for Server {
+ fn drop(&mut self) {
+ let _ = self.0.kill();
+ let _ = self.0.wait();
+ }
+ }
+ fn set32(conn: &RustConnection, win: u32, name: &str, ty: AtomEnum, values: &[u32]) {
+ let atom = crate::x11::get_atom(conn, name).unwrap();
+ conn.change_property32(PropMode::REPLACE, win, atom, ty, values)
+ .unwrap();
+ }
+ #[test]
+ fn focus_restore_leaves_focus_on_a_dialog_the_app_opened() {
+ let mut server = Server(
+ Command::new("Xvfb")
+ .args(["-displayfd", "1", "-nolisten", "tcp"])
+ .stdout(Stdio::piped())
+ .spawn()
+ .expect("install xvfb to run focus tests"),
+ );
+ let mut number = String::new();
+ BufReader::new(server.0.stdout.take().unwrap())
+ .read_line(&mut number)
+ .unwrap();
+ let name = format!(":{}", number.trim());
+ let (conn, screen) = RustConnection::connect(Some(&name)).unwrap();
+ let root = conn.setup().roots[screen].root;
+ let window = |pid, transient| {
+ let win = conn.generate_id().unwrap();
+ conn.create_window(
+ 0,
+ win,
+ root,
+ 0,
+ 0,
+ 50,
+ 50,
+ 0,
+ WindowClass::INPUT_OUTPUT,
+ 0,
+ &CreateWindowAux::new(),
+ )
+ .unwrap();
+ if pid != 0 {
+ set32(&conn, win, "_NET_WM_PID", AtomEnum::CARDINAL, &[pid]);
+ }
+ if transient != 0 {
+ set32(
+ &conn,
+ win,
+ "WM_TRANSIENT_FOR",
+ AtomEnum::WINDOW,
+ &[transient],
+ );
+ }
+ conn.map_window(win).unwrap();
+ win
+ };
+ let main = window(4242, 0);
+ let dialog = window(4242, main);
+ let sibling = window(4242, 0);
+ let foreign = window(4343, main);
+ let typed = window(4242, 0);
+ let pidless = window(0, main);
+ set32(
+ &conn,
+ typed,
+ "_NET_WM_WINDOW_TYPE",
+ AtomEnum::ATOM,
+ &[crate::x11::get_atom(&conn, "_NET_WM_WINDOW_TYPE_DIALOG").unwrap()],
+ );
+ assert!(is_own_dialog(&conn, 4242, dialog.into()));
+ assert!(is_own_dialog(&conn, 4242, typed.into()));
+ for win in [sibling, foreign, pidless] {
+ assert!(!is_own_dialog(&conn, 4242, win.into()));
+ }
+ conn.get_input_focus().unwrap().reply().unwrap();
+ let cname = std::ffi::CString::new(name.clone()).unwrap();
+ let display = open_named_display(cname.as_ptr()).unwrap();
+ // The WM honors every activation, except the first delayed request opens a dialog.
+ for delayed in [false, true] {
+ let (wm, _) = RustConnection::connect(Some(&name)).unwrap();
+ wm.change_window_attributes(
+ root,
+ &ChangeWindowAttributesAux::new().event_mask(EventMask::SUBSTRUCTURE_REDIRECT),
+ )
+ .unwrap()
+ .check()
+ .unwrap();
+ set32(
+ &conn,
+ root,
+ "_NET_ACTIVE_WINDOW",
+ AtomEnum::WINDOW,
+ &[sibling],
+ );
+ conn.set_input_focus(InputFocus::PARENT, sibling, 0u32)
+ .unwrap();
+ conn.get_input_focus().unwrap().reply().unwrap();
+ let stop = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false));
+ let done = stop.clone();
+ let thread = std::thread::spawn(move || {
+ let mut first = delayed;
+ while !done.load(std::sync::atomic::Ordering::Relaxed) {
+ match wm.poll_for_event().unwrap() {
+ Some(Event::ClientMessage(event)) => {
+ let target = if first { dialog } else { event.window };
+ first = false;
+ set32(&wm, root, "_NET_ACTIVE_WINDOW", AtomEnum::WINDOW, &[target]);
+ wm.set_input_focus(InputFocus::PARENT, target, 0u32)
+ .unwrap();
+ wm.flush().unwrap();
+ }
+ _ => std::thread::sleep(Duration::from_millis(1)),
+ }
+ }
+ });
+ restore_focus_state(
+ display,
+ &SavedFocus {
+ ewmh_active: Some(main.into()),
+ core_focus: 0,
+ core_revert_to: 0,
+ },
+ );
+ let focus = conn.get_input_focus().unwrap().reply().unwrap().focus;
+ stop.store(true, std::sync::atomic::Ordering::Relaxed);
+ thread.join().unwrap();
+ assert_eq!(focus, if delayed { dialog } else { main });
+ }
+ unsafe { x11::xlib::XCloseDisplay(display) };
+ }
+}
+
#[cfg(test)]
mod key_name_alias_tests {
use super::{
diff --git a/nix/cua-driver/tests/rust-unit.nix b/nix/cua-driver/tests/rust-unit.nix
index 01b911fa71..6a77169078 100644
--- a/nix/cua-driver/tests/rust-unit.nix
+++ b/nix/cua-driver/tests/rust-unit.nix
@@ -46,7 +46,7 @@ pkgs.rustPlatform.buildRustPackage {
python3
];
# Transport tests launch an isolated D-Bus session instead of using the host bus.
- nativeCheckInputs = [ pkgs.dbus ];
+ nativeCheckInputs = [ pkgs.dbus pkgs.xorg.xorgserver ];
CUA_TEST_PYTHON = "${pkgs.python3}/bin/python3";
CUA_TEST_PYTHON_DYNAMIC_LINKER = pkgs.stdenv.cc.bintools.dynamicLinker;
CUA_TEST_GLIBC_STATIC_LIB = "${pkgs.glibc.static}/lib";
@@ -62,7 +62,7 @@ pkgs.rustPlatform.buildRustPackage {
libei
];
- # The default Cargo test set is deliberately headless. The ignored desktop
+ # Focus unit tests launch private Xvfb servers. The ignored desktop
# matrix is run by the manual Linux e2e workflow and is not hidden in Nix.
doCheck = true;