Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

fix(cua-driver): preserve delivered X11 mouse click timing (#3762)

Peter Steinberger1db9aca009b7
diff --git a/.github/workflows/ci-rust-linux.yml b/.github/workflows/ci-rust-linux.yml
index b0fba747ea..ee83def195 100644
--- a/.github/workflows/ci-rust-linux.yml
+++ b/.github/workflows/ci-rust-linux.yml
@@ -145,6 +145,12 @@ jobs:
run: |
cargo test -p cua-driver-e2e --test e2e_environment_preflight_test --locked -- \
--ignored --exact last_frame_extraction_handles_long_final_duration_and_invalid_input --nocapture
+ - name: Run X11 click delivery and timing regressions
+ working-directory: libs/cua-driver/rust
+ run: |
+ xvfb-run -a --server-args="-screen 0 640x480x24 -noreset" \
+ cargo test -p platform-linux --test click_input_x11 --locked -- \
+ --ignored --nocapture --test-threads=1
- name: Run X11 keyboard tap timing regression
working-directory: libs/cua-driver/rust
run: |
diff --git a/libs/cua-driver/rust/crates/platform-linux/src/input/mod.rs b/libs/cua-driver/rust/crates/platform-linux/src/input/mod.rs
index 180ebdba4b..bebb301304 100644
--- a/libs/cua-driver/rust/crates/platform-linux/src/input/mod.rs
+++ b/libs/cua-driver/rust/crates/platform-linux/src/input/mod.rs
@@ -3301,8 +3301,8 @@ pub fn send_button_down(xid: u64, x: i32, y: i32, button: u8) -> Result<()> {
state: KeyButMask::from(0u16),
same_screen: true,
};
- conn.send_event(false, target.window, EventMask::BUTTON_PRESS, &press)?;
- conn.flush()?;
+ conn.send_event(false, target.window, EventMask::BUTTON_PRESS, &press)?
+ .check()?;
Ok(())
}
@@ -3351,8 +3351,8 @@ pub fn send_button_up(xid: u64, x: i32, y: i32, button: u8) -> Result<()> {
state: button_state_mask(button),
same_screen: true,
};
- conn.send_event(false, target.window, EventMask::BUTTON_RELEASE, &release)?;
- conn.flush()?;
+ conn.send_event(false, target.window, EventMask::BUTTON_RELEASE, &release)?
+ .check()?;
Ok(())
}
@@ -3640,6 +3640,7 @@ pub fn send_click_xtest_desktop_with_modifiers(
modifier_keycodes.push(keycode);
}
let mut pressed = Vec::new();
+ let mut button_pressed = false;
let gesture_result = (|| -> Result<()> {
for &keycode in &modifier_keycodes {
conn.xtest_fake_input(KEY_PRESS_EVENT, keycode, 0, x11rb::NONE, 0, 0, 0)?;
@@ -3650,23 +3651,32 @@ pub fn send_click_xtest_desktop_with_modifiers(
conn.xtest_fake_input(MOTION_NOTIFY_EVENT, 0, 0, root, x as i16, y as i16, 0)?;
let count = count.max(1);
for click_index in 0..count {
- conn.xtest_fake_input(BUTTON_PRESS_EVENT, button, 0, root, x as i16, y as i16, 0)?;
- conn.xtest_fake_input(BUTTON_RELEASE_EVENT, button, 0, root, x as i16, y as i16, 0)?;
+ let press =
+ conn.xtest_fake_input(BUTTON_PRESS_EVENT, button, 0, root, x as i16, y as i16, 0)?;
+ // Confirm delivery before sleeping; buffering both edges produces a zero-ms click.
+ press.check()?;
+ button_pressed = true;
+ sleep(Duration::from_millis(CLICK_DELAY_MS));
+ conn.xtest_fake_input(BUTTON_RELEASE_EVENT, button, 0, root, x as i16, y as i16, 0)?
+ .check()?;
+ button_pressed = false;
if click_index + 1 < count {
- // Chromium needs the first pair to reach the server before the
- // second pair. A zero-gap batch produces two click events but
- // no DOM dblclick event under Xvfb/Openbox.
- conn.flush()?;
+ // Keep the existing gap between completed click pairs.
sleep(Duration::from_millis(DOUBLE_CLICK_DELAY_MS));
}
}
Ok(())
})();
- // Always attempt to release every modifier that was successfully queued,
- // including when a later pointer request fails. A failed gesture must not
- // leave the desktop with a logically stuck Ctrl/Shift/Alt/Super key.
+ // Attempt all releases after a partial failure. Do not strand buttons or modifiers.
let mut release_result: Result<()> = Ok(());
+ if button_pressed {
+ release_result = (|| -> Result<()> {
+ conn.xtest_fake_input(BUTTON_RELEASE_EVENT, button, 0, root, x as i16, y as i16, 0)?
+ .check()?;
+ Ok(())
+ })();
+ }
for &keycode in pressed.iter().rev() {
if let Err(error) =
conn.xtest_fake_input(KEY_RELEASE_EVENT, keycode, 0, x11rb::NONE, 0, 0, 0)
@@ -3676,15 +3686,16 @@ pub fn send_click_xtest_desktop_with_modifiers(
}
}
}
- conn.flush()?;
- // Round-trip so the server processes the warp+button events before this
- // short-lived connection drops. Pointer events happened to survive the close
- // under Xtigervnc where keyboard events did not (see send_key_xtest), but make
- // it explicit so the desktop click is reliable across X servers too.
- let _ = conn.get_input_focus()?.reply();
+ // Complete queued modifier releases before the short-lived connection closes,
+ // while preserving the primary gesture or cleanup error when one exists.
+ let completion_result = (|| -> Result<()> {
+ conn.get_input_focus()?.reply()?;
+ Ok(())
+ })();
drop(guards);
gesture_result?;
release_result?;
+ completion_result?;
Ok(())
}
diff --git a/libs/cua-driver/rust/crates/platform-linux/tests/click_input_x11.rs b/libs/cua-driver/rust/crates/platform-linux/tests/click_input_x11.rs
new file mode 100644
index 0000000000..9b7b576c7a
--- /dev/null
+++ b/libs/cua-driver/rust/crates/platform-linux/tests/click_input_x11.rs
@@ -0,0 +1,191 @@
+// SPDX-License-Identifier: MIT
+
+#![cfg(target_os = "linux")]
+
+use std::time::{Duration, Instant};
+
+use anyhow::{bail, Result};
+use platform_linux::input::{
+ send_button_down, send_button_up, send_click_with_modifiers,
+ send_click_xtest_desktop_with_modifiers,
+};
+use x11rb::connection::Connection;
+use x11rb::protocol::xproto::*;
+use x11rb::protocol::Event;
+use x11rb::rust_connection::RustConnection;
+
+fn input_window(conn: &RustConnection, parent: Window, x: i16) -> Result<Window> {
+ let window = conn.generate_id()?;
+ conn.create_window(
+ x11rb::COPY_DEPTH_FROM_PARENT,
+ window,
+ parent,
+ x,
+ 0,
+ 200,
+ 120,
+ 0,
+ WindowClass::INPUT_OUTPUT,
+ 0,
+ &CreateWindowAux::new()
+ .override_redirect(1)
+ .event_mask(EventMask::BUTTON_PRESS | EventMask::BUTTON_RELEASE),
+ )?
+ .check()?;
+ conn.map_window(window)?.check()?;
+ assert_eq!(
+ conn.get_window_attributes(window)?.reply()?.map_state,
+ MapState::VIEWABLE
+ );
+ Ok(window)
+}
+
+fn button_events(
+ conn: &RustConnection,
+ expected: usize,
+) -> Result<Vec<(bool, ButtonPressEvent, Instant)>> {
+ let deadline = Instant::now() + Duration::from_secs(3);
+ let mut events = Vec::new();
+ while events.len() < expected {
+ if Instant::now() >= deadline {
+ bail!("received {} of {expected} button events", events.len());
+ }
+ match conn.poll_for_event()? {
+ Some(Event::ButtonPress(event)) => events.push((true, event, Instant::now())),
+ Some(Event::ButtonRelease(event)) => events.push((false, event, Instant::now())),
+ Some(Event::Error(error)) => bail!("X11 observer error: {error:?}"),
+ _ => std::thread::sleep(Duration::from_millis(1)),
+ }
+ }
+ Ok(events)
+}
+
+fn assert_pair(events: &[(bool, ButtonPressEvent, Instant)], target: Window, modified: bool) {
+ assert!(events[0].0 && !events[1].0);
+ let press = events[0].1;
+ let release = events[1].1;
+ assert_eq!(press.detail, 1);
+ assert_eq!(release.detail, 1);
+ assert_eq!(press.event, target);
+ assert_eq!(release.event, target);
+ assert!(!press.state.contains(KeyButMask::BUTTON1));
+ assert!(release.state.contains(KeyButMask::BUTTON1));
+ assert_eq!(press.state.contains(KeyButMask::CONTROL), modified);
+ assert_eq!(release.state.contains(KeyButMask::CONTROL), modified);
+}
+
+/// These tests need a disposable display because XTest moves the real pointer.
+/// xvfb-run -a cargo test -p platform-linux --test click_input_x11 -- --ignored --test-threads=1
+#[test]
+#[ignore = "requires an isolated X11 display with XTEST"]
+fn desktop_clicks_deliver_a_hold_and_release_modifiers() -> Result<()> {
+ let (conn, screen) = x11rb::connect(None)?;
+ let root = conn.setup().roots[screen].root;
+ let target = input_window(&conn, root, 0)?;
+ conn.set_input_focus(InputFocus::PARENT, target, x11rb::CURRENT_TIME)?
+ .check()?;
+
+ for count in [1, 2, 3] {
+ for modifiers in [vec![], vec!["ctrl"]] {
+ send_click_xtest_desktop_with_modifiers(50, 50, 1, count, &modifiers)?;
+ let events = button_events(&conn, count * 2)?;
+ for pair in events.chunks_exact(2) {
+ assert_pair(pair, target, !modifiers.is_empty());
+ assert_eq!(pair[0].1.response_type & 0x80, 0);
+ // Server timestamps cannot turn buffered transitions into a hold
+ // merely because the observer thread was scheduled late.
+ let held_ms = pair[1].1.time.wrapping_sub(pair[0].1.time);
+ eprintln!("desktop count={count} modifiers={modifiers:?}: hold={held_ms} ms");
+ assert!(held_ms >= 30, "desktop click held only {held_ms} ms");
+ }
+ for adjacent in events.chunks_exact(2).collect::<Vec<_>>().windows(2) {
+ let gap_ms = adjacent[1][0].1.time.wrapping_sub(adjacent[0][1].1.time);
+ assert!(gap_ms >= 40, "multi-click gap was only {gap_ms} ms");
+ }
+ let pointer = conn.query_pointer(root)?.reply()?;
+ assert!(!pointer
+ .mask
+ .intersects(KeyButMask::BUTTON1 | KeyButMask::CONTROL));
+ }
+ }
+ Ok(())
+}
+
+#[test]
+#[ignore = "requires an isolated X11 display"]
+fn background_clicks_deliver_spaced_pairs_without_changing_focus() -> Result<()> {
+ let (conn, screen) = x11rb::connect(None)?;
+ let root = conn.setup().roots[screen].root;
+ let target = input_window(&conn, root, 0)?;
+ let sentinel = input_window(&conn, root, 300)?;
+ conn.set_input_focus(InputFocus::PARENT, sentinel, x11rb::CURRENT_TIME)?
+ .check()?;
+
+ for modifiers in [vec![], vec!["ctrl"]] {
+ // XSendEvent preserves the supplied timestamp (CURRENT_TIME), so observe
+ // arrivals concurrently rather than claiming those zero timestamps prove timing.
+ let events = std::thread::scope(|scope| -> Result<_> {
+ let sender = scope
+ .spawn(|| send_click_with_modifiers(u64::from(target), 50, 50, 5, 1, &modifiers));
+ let events = button_events(&conn, 10);
+ sender.join().expect("click sender panicked")?;
+ events
+ })?;
+ let mut holds = Vec::new();
+ for pair in events.chunks_exact(2) {
+ assert_pair(pair, target, !modifiers.is_empty());
+ assert_ne!(pair[0].1.response_type & 0x80, 0);
+ holds.push(pair[1].2.duration_since(pair[0].2).as_millis());
+ }
+ eprintln!("background modifiers={modifiers:?}: observed holds={holds:?} ms");
+ // Allow an occasional delayed observer wakeup; a buffered press/release
+ // pair remains near zero for every click on the original implementation.
+ holds.sort_unstable();
+ assert!(
+ holds[2] >= 20,
+ "background click median hold was only {} ms",
+ holds[2]
+ );
+ assert_eq!(conn.get_input_focus()?.reply()?.focus, sentinel);
+ }
+ Ok(())
+}
+
+#[test]
+#[ignore = "requires an isolated X11 display"]
+fn separate_background_button_calls_deliver_ordered_pairs() -> Result<()> {
+ let (conn, screen) = x11rb::connect(None)?;
+ let root = conn.setup().roots[screen].root;
+ let target = input_window(&conn, root, 0)?;
+ let sentinel = input_window(&conn, root, 300)?;
+ conn.set_input_focus(InputFocus::PARENT, sentinel, x11rb::CURRENT_TIME)?
+ .check()?;
+
+ for _ in 0..10 {
+ send_button_down(u64::from(target), 50, 50, 1)?;
+ send_button_up(u64::from(target), 50, 50, 1)?;
+ let events = button_events(&conn, 2)?;
+ assert_pair(&events, target, false);
+ assert_eq!(conn.get_input_focus()?.reply()?.focus, sentinel);
+ }
+ Ok(())
+}
+
+#[test]
+#[ignore = "requires an isolated X11 display with XTEST"]
+fn rejected_desktop_button_releases_preceding_modifiers() -> Result<()> {
+ let (conn, screen) = x11rb::connect(None)?;
+ let root = conn.setup().roots[screen].root;
+ let target = input_window(&conn, root, 0)?;
+ conn.set_input_focus(InputFocus::PARENT, target, x11rb::CURRENT_TIME)?
+ .check()?;
+
+ // A real server rejection occurs after Ctrl-down has already been queued.
+ // This exercises partial acquisition without a production injection seam.
+ assert!(send_click_xtest_desktop_with_modifiers(50, 50, 0, 1, &["ctrl"]).is_err());
+ let pointer = conn.query_pointer(root)?.reply()?;
+ assert!(!pointer
+ .mask
+ .intersects(KeyButMask::BUTTON1 | KeyButMask::CONTROL));
+ Ok(())
+}