Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
test(cua-driver): certify X11 overlay click-through lifecycle (#2867)
Francesco Bonaccic1e39252b0ca
diff --git a/.github/workflows/ci-rust-linux.yml b/.github/workflows/ci-rust-linux.ymlindex 303dde51d9..2edf9a90d9 100644--- a/.github/workflows/ci-rust-linux.yml+++ b/.github/workflows/ci-rust-linux.yml@@ -70,11 +70,15 @@ jobs:run: cargo test -p cua-driver -p cua-driver-contract -p cua-driver-core -p cua-driver-testkit -p cursor-overlay -p cursor-theme-cli -p platform-linux --all-targets --locked- name: Run X11 overlay click-through regressionworking-directory: libs/cua-driver/rust- run: >-- xvfb-run -a --server-args="-screen 0 1920x1080x24"- cargo test -p platform-linux- x11_overlay_input_shape_stays_empty_and_clicks_pass_through_after_resize- --locked -- --ignored --nocapture+ run: |+ xvfb-run -a --server-args="-screen 0 1920x1080x24" \+ cargo test -p platform-linux \+ x11_overlay_input_shape_stays_empty_and_clicks_pass_through_after_resize \+ --locked -- --ignored --nocapture+ xvfb-run -a --server-args="-screen 0 1920x1080x24" \+ cargo test -p platform-linux \+ x11_overlay_owner_stays_click_through_across_cursor_lifecycle_and_randr \+ --locked -- --ignored --nocapture- name: Run real uinput device-name regressionworking-directory: libs/cua-driver/rustrun: |diff --git a/libs/cua-driver/rust/crates/platform-linux/src/overlay.rs b/libs/cua-driver/rust/crates/platform-linux/src/overlay.rsindex c76af86b88..33fafa5f50 100644--- a/libs/cua-driver/rust/crates/platform-linux/src/overlay.rs+++ b/libs/cua-driver/rust/crates/platform-linux/src/overlay.rs@@ -42,6 +42,9 @@ static CMD_RX_CELL: Mutex<Option<std::sync::mpsc::Receiver<OverlayMsg>>> = Mutexstatic RENDER: Mutex<Option<RenderMap>> = Mutex::new(None);static ARRIVAL_TX: Mutex<Option<HashMap<CursorKey, tokio::sync::oneshot::Sender<()>>>> =Mutex::new(None);+#[cfg(all(test, target_os = "linux"))]+static X11_RANDR_REPAIR_COUNT: std::sync::atomic::AtomicUsize =+ std::sync::atomic::AtomicUsize::new(0);fn arrival_register(key: CursorKey, tx: tokio::sync::oneshot::Sender<()>) {let mut guard = ARRIVAL_TX.lock().unwrap();@@ -1088,6 +1091,8 @@ fn run_overlay_thread(cfg: CursorConfig, rx: std::sync::mpsc::Receiver<OverlayMstracing::warn!("X11 overlay geometry repair failed; disabling overlay: {e}");break;}+ #[cfg(test)]+ X11_RANDR_REPAIR_COUNT.fetch_add(1, std::sync::atomic::Ordering::SeqCst);compositor_present = x11_compositor_present(&conn, screen_num);tracing::debug!(width = geometry.0,@@ -1559,6 +1564,11 @@ fn bgra_and_visible_shape(mod tests {use super::*;+ fn drain_x11_test_events(conn: &impl x11rb::connection::Connection) -> anyhow::Result<()> {+ while conn.poll_for_event()?.is_some() {}+ Ok(())+ }+fn assert_x11_button_press_target(conn: &impl x11rb::connection::Connection,root: u32,@@ -1569,6 +1579,7 @@ mod tests {};use x11rb::protocol::xtest::ConnectionExt as XtestConnectionExt;+ drain_x11_test_events(conn)?;conn.xtest_fake_input(MOTION_NOTIFY_EVENT, 0, x11rb::CURRENT_TIME, root, 50, 50, 0)?.check()?;conn.xtest_fake_input(BUTTON_PRESS_EVENT, 1, x11rb::CURRENT_TIME, root, 0, 0, 0)?@@ -1596,6 +1607,223 @@ mod tests {anyhow::bail!("timed out waiting for an X11 button press")}+ #[test]+ #[ignore = "requires a live X11 server with XTEST, Shape, and RandR (run under xvfb-run or XWayland)"]+ fn x11_overlay_owner_stays_click_through_across_cursor_lifecycle_and_randr(+ ) -> anyhow::Result<()> {+ use x11rb::connection::Connection;+ use x11rb::protocol::randr::ConnectionExt as RandrConnectionExt;+ use x11rb::protocol::shape::{ConnectionExt as ShapeConnectionExt, SK};+ use x11rb::protocol::xproto::{+ AtomEnum, ConnectionExt as XprotoConnectionExt, CreateWindowAux, EventMask, MapState,+ Window, WindowClass,+ };++ fn find_named_window(+ conn: &impl Connection,+ root: Window,+ expected_name: &[u8],+ ) -> anyhow::Result<Window> {+ let deadline = Instant::now() + Duration::from_secs(3);+ while Instant::now() < deadline {+ for window in conn.query_tree(root)?.reply()?.children {+ let name = conn+ .get_property(+ false,+ window,+ AtomEnum::WM_NAME,+ AtomEnum::STRING,+ 0,+ u32::MAX,+ )?+ .reply()?;+ if name.value == expected_name {+ return Ok(window);+ }+ }+ std::thread::sleep(Duration::from_millis(10));+ }+ anyhow::bail!(+ "timed out waiting for X11 window {}",+ String::from_utf8_lossy(expected_name)+ )+ }++ fn wait_for_bounding_shape(+ conn: &impl Connection,+ overlay: Window,+ should_be_empty: bool,+ phase: &str,+ ) -> anyhow::Result<()> {+ let deadline = Instant::now() + Duration::from_secs(3);+ while Instant::now() < deadline {+ let shape = conn.shape_get_rectangles(overlay, SK::BOUNDING)?.reply()?;+ if shape.rectangles.is_empty() == should_be_empty {+ return Ok(());+ }+ std::thread::sleep(Duration::from_millis(10));+ }+ anyhow::bail!("overlay bounding shape did not settle during {phase}")+ }++ fn assert_click_through(+ conn: &impl Connection,+ root: Window,+ overlay: Window,+ target: Window,+ phase: &str,+ ) -> anyhow::Result<()> {+ let attributes = conn.get_window_attributes(overlay)?.reply()?;+ anyhow::ensure!(+ attributes.map_state == MapState::VIEWABLE,+ "overlay was not mapped during {phase}"+ );+ let input = conn.shape_get_rectangles(overlay, SK::INPUT)?.reply()?;+ anyhow::ensure!(+ input.rectangles.is_empty(),+ "overlay exposed {} ShapeInput rectangles during {phase}",+ input.rectangles.len()+ );+ assert_x11_button_press_target(conn, root, target).map_err(|error| {+ anyhow::anyhow!("click delivery failed during {phase}: {error}")+ })?;+ eprintln!(+ "{phase}: ShapeInput rectangles={}, click_target={target}",+ input.rectangles.len()+ );+ Ok(())+ }++ fn wait_for_cursor_move_from(x: f64, y: f64, phase: &str) -> anyhow::Result<()> {+ let deadline = Instant::now() + Duration::from_secs(3);+ while Instant::now() < deadline {+ let position = current_position();+ if (position.0 - x).hypot(position.1 - y) > 32.0 {+ eprintln!(+ "{phase}: cursor moved from ({x}, {y}) to ({}, {})",+ position.0, position.1+ );+ return Ok(());+ }+ std::thread::sleep(Duration::from_millis(10));+ }+ let position = current_position();+ anyhow::bail!(+ "cursor position remained ({}, {}) near ({x}, {y}) during {phase}",+ position.0,+ position.1+ )+ }++ let (conn, screen_num) = x11rb::connect(None)?;+ let screen = &conn.setup().roots[screen_num];+ let root = screen.root;+ let root_depth = screen.root_depth;+ let root_visual = screen.root_visual;+ let initial_width = screen.width_in_pixels;+ let initial_height = screen.height_in_pixels;+ let initial_mm_width = screen.width_in_millimeters;+ let initial_mm_height = screen.height_in_millimeters;++ let target = conn.generate_id()?;+ conn.create_window(+ root_depth,+ target,+ root,+ 0,+ 0,+ 400,+ 400,+ 0,+ WindowClass::INPUT_OUTPUT,+ root_visual,+ &CreateWindowAux::new().event_mask(EventMask::BUTTON_PRESS),+ )?+ .check()?;+ conn.map_window(target)?.check()?;+ conn.flush()?;++ let cursor_id = "issue-1819-live-shape-probe";+ let cfg = CursorConfig {+ cursor_id: cursor_id.to_owned(),+ ..CursorConfig::default()+ };+ init(cfg);+ run_on_thread();++ let title = format!("Cua.AgentCursorOverlay.{cursor_id}");+ let overlay = find_named_window(&conn, root, title.as_bytes())?;+ wait_for_bounding_shape(&conn, overlay, true, "daemon startup")?;+ assert_click_through(&conn, root, overlay, target, "daemon startup")?;++ send_command(OverlayCommand::SetEnabled(true));+ send_command(OverlayCommand::SnapTo {+ x: 160.0,+ y: 160.0,+ heading_radians: None,+ });+ wait_for_bounding_shape(&conn, overlay, false, "cursor show")?;+ assert_click_through(&conn, root, overlay, target, "cursor show")?;++ send_command(OverlayCommand::MoveTo {+ x: 240.0,+ y: 240.0,+ end_heading_radians: std::f64::consts::FRAC_PI_4,+ });+ wait_for_cursor_move_from(160.0, 160.0, "cursor move")?;+ wait_for_bounding_shape(&conn, overlay, false, "cursor move")?;+ assert_click_through(&conn, root, overlay, target, "cursor move")?;++ send_command(OverlayCommand::SetEnabled(false));+ wait_for_bounding_shape(&conn, overlay, true, "cursor hide")?;+ assert_click_through(&conn, root, overlay, target, "cursor hide")?;++ send_command(OverlayCommand::SetEnabled(true));+ send_command(OverlayCommand::SnapTo {+ x: 160.0,+ y: 160.0,+ heading_radians: None,+ });+ wait_for_bounding_shape(&conn, overlay, false, "cursor reshow")?;++ let repairs_before = X11_RANDR_REPAIR_COUNT.load(std::sync::atomic::Ordering::SeqCst);+ conn.randr_set_screen_size(+ root,+ initial_width,+ initial_height,+ u32::from(initial_mm_width).saturating_add(1),+ u32::from(initial_mm_height).saturating_add(1),+ )?+ .check()?;+ conn.flush()?;++ let deadline = Instant::now() + Duration::from_secs(3);+ let repairs_after = loop {+ let repairs = X11_RANDR_REPAIR_COUNT.load(std::sync::atomic::Ordering::SeqCst);+ if repairs > repairs_before {+ break repairs;+ }+ if Instant::now() >= deadline {+ anyhow::bail!("overlay owner did not repair after the RandR configuration event");+ }+ std::thread::sleep(Duration::from_millis(10));+ };+ let geometry = conn.get_geometry(overlay)?.reply()?;+ anyhow::ensure!(+ geometry.width == initial_width && geometry.height == initial_height,+ "overlay geometry became {}x{} after same-size RandR repair",+ geometry.width,+ geometry.height+ );+ eprintln!(+ "RandR repair: count={repairs_after}, overlay_geometry={}x{}",+ geometry.width, geometry.height+ );+ wait_for_bounding_shape(&conn, overlay, false, "RandR repair")?;+ assert_click_through(&conn, root, overlay, target, "RandR repair")?;+ Ok(())+ }+#[test]#[ignore = "requires a live X11 server (run under xvfb-run)"]fn x11_overlay_input_shape_stays_empty_and_clicks_pass_through_after_resize(