Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
fix(cua-driver/linux): fail loudly when X11 input can't be delivered on pure Wayland (#1921) (#1994)
Francesco Bonacci7c1d06bf19d2
diff --git a/libs/cua-driver/rust/crates/platform-linux/src/input/mod.rs b/libs/cua-driver/rust/crates/platform-linux/src/input/mod.rsindex 3aa627d449..846fbd0b0a 100644--- a/libs/cua-driver/rust/crates/platform-linux/src/input/mod.rs+++ b/libs/cua-driver/rust/crates/platform-linux/src/input/mod.rs@@ -1140,11 +1140,29 @@ fn button_state_mask(button: u8) -> KeyButMask {}}+/// Open an X11 connection for background input injection, failing *loudly* and+/// actionably when input cannot be delivered — rather than letting a pure+/// Wayland session fall through to an X11 path that silently no-ops yet reports+/// success (#1921). On a pure Wayland session with the native backend off, this+/// returns a clear error naming the fix; otherwise it connects and, on any+/// connect failure, surfaces DISPLAY so the cause is diagnosable.+fn connect_x11_for_input() -> Result<(RustConnection, usize)> {+ if let Some(reason) = crate::wayland::wayland_input_unavailable_reason() {+ bail!("{reason}");+ }+ RustConnection::connect(None).map_err(|e| {+ anyhow!(+ "cannot inject input: X11 connection failed (DISPLAY={:?}): {e}",+ std::env::var("DISPLAY").ok()+ )+ })+}+/// Send a synthetic FocusIn event to a window without changing the actual X11 input focus./// This can trigger toolkit-level focus handlers (e.g., Qt5's AT-SPI bridge) without/// moving the window manager's active window. Use with send_focus_out to restore state.pub fn send_focus_in(xid: u64) -> Result<()> {- let (conn, _) = RustConnection::connect(None)?;+ let (conn, _) = connect_x11_for_input()?;let window = xid as u32;let focus_in = FocusInEvent {@@ -1162,7 +1180,7 @@ pub fn send_focus_in(xid: u64) -> Result<()> {/// Send a synthetic FocusOut event to restore focus state after send_focus_in.pub fn send_focus_out(xid: u64) -> Result<()> {- let (conn, _) = RustConnection::connect(None)?;+ let (conn, _) = connect_x11_for_input()?;let window = xid as u32;let focus_out = FocusOutEvent {@@ -1180,7 +1198,7 @@ pub fn send_focus_out(xid: u64) -> Result<()> {/// Send a button click (down + up) to a window at window-local coordinates.pub fn send_click(xid: u64, x: i32, y: i32, count: usize, button: u8) -> Result<()> {- let (conn, _) = RustConnection::connect(None)?;+ let (conn, _) = connect_x11_for_input()?;let root = conn.setup().roots[0].root;for _ in 0..count {@@ -1244,7 +1262,7 @@ pub fn send_drag(steps: usize,button: u8,) -> Result<()> {- let (conn, _) = RustConnection::connect(None)?;+ let (conn, _) = connect_x11_for_input()?;let root = conn.setup().roots[0].root;let steps = steps.max(1);let step_delay_ms = if steps > 1 { duration_ms / steps as u64 } else { duration_ms };@@ -1309,7 +1327,7 @@ pub fn send_drag(}pub fn send_button_down(xid: u64, x: i32, y: i32, button: u8) -> Result<()> {- let (conn, _) = RustConnection::connect(None)?;+ let (conn, _) = connect_x11_for_input()?;let root = conn.setup().roots[0].root;let target = resolve_event_target(&conn, xid, x, y)?;let press = ButtonPressEvent {@@ -1333,7 +1351,7 @@ pub fn send_button_down(xid: u64, x: i32, y: i32, button: u8) -> Result<()> {}pub fn send_motion(xid: u64, x: i32, y: i32, button: Option<u8>) -> Result<()> {- let (conn, _) = RustConnection::connect(None)?;+ let (conn, _) = connect_x11_for_input()?;let root = conn.setup().roots[0].root;let target = resolve_event_target(&conn, xid, x, y)?;let motion = MotionNotifyEvent {@@ -1357,7 +1375,7 @@ pub fn send_motion(xid: u64, x: i32, y: i32, button: Option<u8>) -> Result<()> {}pub fn send_button_up(xid: u64, x: i32, y: i32, button: u8) -> Result<()> {- let (conn, _) = RustConnection::connect(None)?;+ let (conn, _) = connect_x11_for_input()?;let root = conn.setup().roots[0].root;let target = resolve_event_target(&conn, xid, x, y)?;let release = ButtonReleaseEvent {@@ -1387,7 +1405,7 @@ pub fn send_type_text(xid: u64, text: &str) -> Result<()> {/// Type a string with an additional `inter_char_ms` delay between each character.pub fn send_type_text_with_delay(xid: u64, text: &str, inter_char_ms: u64) -> Result<()> {- let (conn, _) = RustConnection::connect(None)?;+ let (conn, _) = connect_x11_for_input()?;let window = xid as u32;let root = conn.setup().roots[0].root;let mapping = conn.get_keyboard_mapping(8, 248)?.reply()?;@@ -1442,7 +1460,7 @@ pub fn send_type_text_with_delay(xid: u64, text: &str, inter_char_ms: u64) -> Re/// focuses the target by clicking it first. `\n`/`\t` map to Return/Tab.pub fn send_type_text_xtest(text: &str) -> Result<()> {use x11rb::protocol::xtest::ConnectionExt as _;- let (conn, _) = RustConnection::connect(None)?;+ let (conn, _) = connect_x11_for_input()?;let mapping = conn.get_keyboard_mapping(8, 248)?.reply()?;// Shift keycode (modifier index 0) for shifted characters.let modmap = conn.get_modifier_mapping()?.reply()?;@@ -1477,7 +1495,7 @@ pub fn send_type_text_xtest(text: &str) -> Result<()> {/// Send a named key press to a window.pub fn send_key(xid: u64, key: &str, modifiers: &[&str]) -> Result<()> {- let (conn, _) = RustConnection::connect(None)?;+ let (conn, _) = connect_x11_for_input()?;let window = xid as u32;let root = conn.setup().roots[0].root;diff --git a/libs/cua-driver/rust/crates/platform-linux/src/wayland/mod.rs b/libs/cua-driver/rust/crates/platform-linux/src/wayland/mod.rsindex a09482e7e9..92c2e167ad 100644--- a/libs/cua-driver/rust/crates/platform-linux/src/wayland/mod.rs+++ b/libs/cua-driver/rust/crates/platform-linux/src/wayland/mod.rs@@ -106,6 +106,30 @@ pub fn is_wayland() -> bool {&& std::env::var_os("DISPLAY").is_none()}+/// Reason string when X11 input injection cannot possibly work, so callers+/// **fail loudly** instead of falling through to an X11 path that no-ops yet+/// reports success. Triggers only on a *pure* Wayland session — `WAYLAND_DISPLAY`+/// set, no X11 `DISPLAY` — with the native-Wayland backend NOT opted in (so+/// [`is_wayland`] is false and the X11 path would be chosen). XWayland sessions+/// (where `DISPLAY` is set) and X11 sessions return `None` and proceed normally.+/// See #1921.+pub fn wayland_input_unavailable_reason() -> Option<String> {+ if std::env::var_os("WAYLAND_DISPLAY").is_some()+ && std::env::var_os("DISPLAY").is_none()+ && !wayland_enabled()+ {+ Some(format!(+ "input cannot be delivered: pure Wayland session (no X11 DISPLAY) and \+ the native-Wayland input backend is not enabled. Set {}=1 to enable \+ the Wayland backend (wlroots compositors: sway, labwc, hyprland), or \+ run the target under XWayland so an X11 DISPLAY is available.",+ ENABLE_WAYLAND_ENV+ ))+ } else {+ None+ }+}+fn wl_sockets(dir: &str) -> std::collections::HashSet<String> {std::fs::read_dir(dir).into_iter()