Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
fix(cua-driver): preserve delivered X11 mouse click timing (#3762)
Peter Steinberger1db9aca009b7
diff --git a/.github/workflows/ci-rust-linux.yml b/.github/workflows/ci-rust-linux.ymlindex b0fba747ea..ee83def195 100644--- a/.github/workflows/ci-rust-linux.yml+++ b/.github/workflows/ci-rust-linux.yml@@ -145,6 +145,12 @@ jobs:run: |cargo test -p cua-driver-e2e --test e2e_environment_preflight_test --locked -- \--ignored --exact last_frame_extraction_handles_long_final_duration_and_invalid_input --nocapture+ - name: Run X11 click delivery and timing regressions+ working-directory: libs/cua-driver/rust+ run: |+ xvfb-run -a --server-args="-screen 0 640x480x24 -noreset" \+ cargo test -p platform-linux --test click_input_x11 --locked -- \+ --ignored --nocapture --test-threads=1- name: Run X11 keyboard tap timing regressionworking-directory: libs/cua-driver/rustrun: |diff --git a/libs/cua-driver/rust/crates/platform-linux/src/input/mod.rs b/libs/cua-driver/rust/crates/platform-linux/src/input/mod.rsindex 180ebdba4b..bebb301304 100644--- a/libs/cua-driver/rust/crates/platform-linux/src/input/mod.rs+++ b/libs/cua-driver/rust/crates/platform-linux/src/input/mod.rs@@ -3301,8 +3301,8 @@ pub fn send_button_down(xid: u64, x: i32, y: i32, button: u8) -> Result<()> {state: KeyButMask::from(0u16),same_screen: true,};- conn.send_event(false, target.window, EventMask::BUTTON_PRESS, &press)?;- conn.flush()?;+ conn.send_event(false, target.window, EventMask::BUTTON_PRESS, &press)?+ .check()?;Ok(())}@@ -3351,8 +3351,8 @@ pub fn send_button_up(xid: u64, x: i32, y: i32, button: u8) -> Result<()> {state: button_state_mask(button),same_screen: true,};- conn.send_event(false, target.window, EventMask::BUTTON_RELEASE, &release)?;- conn.flush()?;+ conn.send_event(false, target.window, EventMask::BUTTON_RELEASE, &release)?+ .check()?;Ok(())}@@ -3640,6 +3640,7 @@ pub fn send_click_xtest_desktop_with_modifiers(modifier_keycodes.push(keycode);}let mut pressed = Vec::new();+ let mut button_pressed = false;let gesture_result = (|| -> Result<()> {for &keycode in &modifier_keycodes {conn.xtest_fake_input(KEY_PRESS_EVENT, keycode, 0, x11rb::NONE, 0, 0, 0)?;@@ -3650,23 +3651,32 @@ pub fn send_click_xtest_desktop_with_modifiers(conn.xtest_fake_input(MOTION_NOTIFY_EVENT, 0, 0, root, x as i16, y as i16, 0)?;let count = count.max(1);for click_index in 0..count {- conn.xtest_fake_input(BUTTON_PRESS_EVENT, button, 0, root, x as i16, y as i16, 0)?;- conn.xtest_fake_input(BUTTON_RELEASE_EVENT, button, 0, root, x as i16, y as i16, 0)?;+ let press =+ conn.xtest_fake_input(BUTTON_PRESS_EVENT, button, 0, root, x as i16, y as i16, 0)?;+ // Confirm delivery before sleeping; buffering both edges produces a zero-ms click.+ press.check()?;+ button_pressed = true;+ sleep(Duration::from_millis(CLICK_DELAY_MS));+ conn.xtest_fake_input(BUTTON_RELEASE_EVENT, button, 0, root, x as i16, y as i16, 0)?+ .check()?;+ button_pressed = false;if click_index + 1 < count {- // Chromium needs the first pair to reach the server before the- // second pair. A zero-gap batch produces two click events but- // no DOM dblclick event under Xvfb/Openbox.- conn.flush()?;+ // Keep the existing gap between completed click pairs.sleep(Duration::from_millis(DOUBLE_CLICK_DELAY_MS));}}Ok(())})();- // Always attempt to release every modifier that was successfully queued,- // including when a later pointer request fails. A failed gesture must not- // leave the desktop with a logically stuck Ctrl/Shift/Alt/Super key.+ // Attempt all releases after a partial failure. Do not strand buttons or modifiers.let mut release_result: Result<()> = Ok(());+ if button_pressed {+ release_result = (|| -> Result<()> {+ conn.xtest_fake_input(BUTTON_RELEASE_EVENT, button, 0, root, x as i16, y as i16, 0)?+ .check()?;+ Ok(())+ })();+ }for &keycode in pressed.iter().rev() {if let Err(error) =conn.xtest_fake_input(KEY_RELEASE_EVENT, keycode, 0, x11rb::NONE, 0, 0, 0)@@ -3676,15 +3686,16 @@ pub fn send_click_xtest_desktop_with_modifiers(}}}- conn.flush()?;- // Round-trip so the server processes the warp+button events before this- // short-lived connection drops. Pointer events happened to survive the close- // under Xtigervnc where keyboard events did not (see send_key_xtest), but make- // it explicit so the desktop click is reliable across X servers too.- let _ = conn.get_input_focus()?.reply();+ // Complete queued modifier releases before the short-lived connection closes,+ // while preserving the primary gesture or cleanup error when one exists.+ let completion_result = (|| -> Result<()> {+ conn.get_input_focus()?.reply()?;+ Ok(())+ })();drop(guards);gesture_result?;release_result?;+ completion_result?;Ok(())}diff --git a/libs/cua-driver/rust/crates/platform-linux/tests/click_input_x11.rs b/libs/cua-driver/rust/crates/platform-linux/tests/click_input_x11.rsnew file mode 100644index 0000000000..9b7b576c7a--- /dev/null+++ b/libs/cua-driver/rust/crates/platform-linux/tests/click_input_x11.rs@@ -0,0 +1,191 @@+// SPDX-License-Identifier: MIT++#![cfg(target_os = "linux")]++use std::time::{Duration, Instant};++use anyhow::{bail, Result};+use platform_linux::input::{+ send_button_down, send_button_up, send_click_with_modifiers,+ send_click_xtest_desktop_with_modifiers,+};+use x11rb::connection::Connection;+use x11rb::protocol::xproto::*;+use x11rb::protocol::Event;+use x11rb::rust_connection::RustConnection;++fn input_window(conn: &RustConnection, parent: Window, x: i16) -> Result<Window> {+ let window = conn.generate_id()?;+ conn.create_window(+ x11rb::COPY_DEPTH_FROM_PARENT,+ window,+ parent,+ x,+ 0,+ 200,+ 120,+ 0,+ WindowClass::INPUT_OUTPUT,+ 0,+ &CreateWindowAux::new()+ .override_redirect(1)+ .event_mask(EventMask::BUTTON_PRESS | EventMask::BUTTON_RELEASE),+ )?+ .check()?;+ conn.map_window(window)?.check()?;+ assert_eq!(+ conn.get_window_attributes(window)?.reply()?.map_state,+ MapState::VIEWABLE+ );+ Ok(window)+}++fn button_events(+ conn: &RustConnection,+ expected: usize,+) -> Result<Vec<(bool, ButtonPressEvent, Instant)>> {+ let deadline = Instant::now() + Duration::from_secs(3);+ let mut events = Vec::new();+ while events.len() < expected {+ if Instant::now() >= deadline {+ bail!("received {} of {expected} button events", events.len());+ }+ match conn.poll_for_event()? {+ Some(Event::ButtonPress(event)) => events.push((true, event, Instant::now())),+ Some(Event::ButtonRelease(event)) => events.push((false, event, Instant::now())),+ Some(Event::Error(error)) => bail!("X11 observer error: {error:?}"),+ _ => std::thread::sleep(Duration::from_millis(1)),+ }+ }+ Ok(events)+}++fn assert_pair(events: &[(bool, ButtonPressEvent, Instant)], target: Window, modified: bool) {+ assert!(events[0].0 && !events[1].0);+ let press = events[0].1;+ let release = events[1].1;+ assert_eq!(press.detail, 1);+ assert_eq!(release.detail, 1);+ assert_eq!(press.event, target);+ assert_eq!(release.event, target);+ assert!(!press.state.contains(KeyButMask::BUTTON1));+ assert!(release.state.contains(KeyButMask::BUTTON1));+ assert_eq!(press.state.contains(KeyButMask::CONTROL), modified);+ assert_eq!(release.state.contains(KeyButMask::CONTROL), modified);+}++/// These tests need a disposable display because XTest moves the real pointer.+/// xvfb-run -a cargo test -p platform-linux --test click_input_x11 -- --ignored --test-threads=1+#[test]+#[ignore = "requires an isolated X11 display with XTEST"]+fn desktop_clicks_deliver_a_hold_and_release_modifiers() -> Result<()> {+ let (conn, screen) = x11rb::connect(None)?;+ let root = conn.setup().roots[screen].root;+ let target = input_window(&conn, root, 0)?;+ conn.set_input_focus(InputFocus::PARENT, target, x11rb::CURRENT_TIME)?+ .check()?;++ for count in [1, 2, 3] {+ for modifiers in [vec![], vec!["ctrl"]] {+ send_click_xtest_desktop_with_modifiers(50, 50, 1, count, &modifiers)?;+ let events = button_events(&conn, count * 2)?;+ for pair in events.chunks_exact(2) {+ assert_pair(pair, target, !modifiers.is_empty());+ assert_eq!(pair[0].1.response_type & 0x80, 0);+ // Server timestamps cannot turn buffered transitions into a hold+ // merely because the observer thread was scheduled late.+ let held_ms = pair[1].1.time.wrapping_sub(pair[0].1.time);+ eprintln!("desktop count={count} modifiers={modifiers:?}: hold={held_ms} ms");+ assert!(held_ms >= 30, "desktop click held only {held_ms} ms");+ }+ for adjacent in events.chunks_exact(2).collect::<Vec<_>>().windows(2) {+ let gap_ms = adjacent[1][0].1.time.wrapping_sub(adjacent[0][1].1.time);+ assert!(gap_ms >= 40, "multi-click gap was only {gap_ms} ms");+ }+ let pointer = conn.query_pointer(root)?.reply()?;+ assert!(!pointer+ .mask+ .intersects(KeyButMask::BUTTON1 | KeyButMask::CONTROL));+ }+ }+ Ok(())+}++#[test]+#[ignore = "requires an isolated X11 display"]+fn background_clicks_deliver_spaced_pairs_without_changing_focus() -> Result<()> {+ let (conn, screen) = x11rb::connect(None)?;+ let root = conn.setup().roots[screen].root;+ let target = input_window(&conn, root, 0)?;+ let sentinel = input_window(&conn, root, 300)?;+ conn.set_input_focus(InputFocus::PARENT, sentinel, x11rb::CURRENT_TIME)?+ .check()?;++ for modifiers in [vec![], vec!["ctrl"]] {+ // XSendEvent preserves the supplied timestamp (CURRENT_TIME), so observe+ // arrivals concurrently rather than claiming those zero timestamps prove timing.+ let events = std::thread::scope(|scope| -> Result<_> {+ let sender = scope+ .spawn(|| send_click_with_modifiers(u64::from(target), 50, 50, 5, 1, &modifiers));+ let events = button_events(&conn, 10);+ sender.join().expect("click sender panicked")?;+ events+ })?;+ let mut holds = Vec::new();+ for pair in events.chunks_exact(2) {+ assert_pair(pair, target, !modifiers.is_empty());+ assert_ne!(pair[0].1.response_type & 0x80, 0);+ holds.push(pair[1].2.duration_since(pair[0].2).as_millis());+ }+ eprintln!("background modifiers={modifiers:?}: observed holds={holds:?} ms");+ // Allow an occasional delayed observer wakeup; a buffered press/release+ // pair remains near zero for every click on the original implementation.+ holds.sort_unstable();+ assert!(+ holds[2] >= 20,+ "background click median hold was only {} ms",+ holds[2]+ );+ assert_eq!(conn.get_input_focus()?.reply()?.focus, sentinel);+ }+ Ok(())+}++#[test]+#[ignore = "requires an isolated X11 display"]+fn separate_background_button_calls_deliver_ordered_pairs() -> Result<()> {+ let (conn, screen) = x11rb::connect(None)?;+ let root = conn.setup().roots[screen].root;+ let target = input_window(&conn, root, 0)?;+ let sentinel = input_window(&conn, root, 300)?;+ conn.set_input_focus(InputFocus::PARENT, sentinel, x11rb::CURRENT_TIME)?+ .check()?;++ for _ in 0..10 {+ send_button_down(u64::from(target), 50, 50, 1)?;+ send_button_up(u64::from(target), 50, 50, 1)?;+ let events = button_events(&conn, 2)?;+ assert_pair(&events, target, false);+ assert_eq!(conn.get_input_focus()?.reply()?.focus, sentinel);+ }+ Ok(())+}++#[test]+#[ignore = "requires an isolated X11 display with XTEST"]+fn rejected_desktop_button_releases_preceding_modifiers() -> Result<()> {+ let (conn, screen) = x11rb::connect(None)?;+ let root = conn.setup().roots[screen].root;+ let target = input_window(&conn, root, 0)?;+ conn.set_input_focus(InputFocus::PARENT, target, x11rb::CURRENT_TIME)?+ .check()?;++ // A real server rejection occurs after Ctrl-down has already been queued.+ // This exercises partial acquisition without a production injection seam.+ assert!(send_click_xtest_desktop_with_modifiers(50, 50, 0, 1, &["ctrl"]).is_err());+ let pointer = conn.query_pointer(root)?.reply()?;+ assert!(!pointer+ .mask+ .intersects(KeyButMask::BUTTON1 | KeyButMask::CONTROL));+ Ok(())+}