Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
fix(cua-driver): keep an ended session's master pointer until its popup menu closes
injaneitya08193ac9152
diff --git a/libs/cua-driver/rust/crates/platform-linux/src/input/mod.rs b/libs/cua-driver/rust/crates/platform-linux/src/input/mod.rsindex 729cacaeda..47faf96ad9 100644--- a/libs/cua-driver/rust/crates/platform-linux/src/input/mod.rs+++ b/libs/cua-driver/rust/crates/platform-linux/src/input/mod.rs@@ -43,7 +43,7 @@ use std::ptr;use std::sync::atomic::{AtomicU32, AtomicU64, Ordering};use std::sync::{Arc, Mutex, OnceLock};use std::thread::sleep;-use std::time::Duration;+use std::time::{Duration, Instant};use x11rb::connection::Connection;use x11rb::protocol::xproto::*;use x11rb::rust_connection::RustConnection;@@ -155,6 +155,7 @@ static XLIB_THREADS_READY: OnceLock<Result<(), String>> = OnceLock::new();static MPX_OP_LOCK: Mutex<()> = Mutex::new(());static MPX_LAST_USE: OnceLock<Mutex<HashMap<String, std::time::Instant>>> = OnceLock::new();static MPX_IDLE_REAPER: std::sync::Once = std::sync::Once::new();+static MPX_CLICK_POPUPS: OnceLock<Mutex<HashMap<String, ClickPopups>>> = OnceLock::new();/// A session's retained master pair is removed after this much inactivity;/// `end_session` and the startup reaper cover the explicit and crash cases./// `XIRemoveMaster` churns the XInput hierarchy, and mutter 42.x segfaults when@@ -165,6 +166,7 @@ static MPX_IDLE_REAPER: std::sync::Once = std::sync::Once::new();/// `end_session`; the startup reaper reclaims anything it misses.const MPX_IDLE_TTL: Duration = Duration::from_secs(1800);const MPX_IDLE_REAPER_PERIOD: Duration = Duration::from_secs(60);+const MPX_POPUP_QUIET: Duration = Duration::from_millis(250);static MPX_NAME_COUNTER: AtomicU64 = AtomicU64::new(1);// evdev 0.12.2 asserts `name.len() + 1 < UINPUT_MAX_NAME_SIZE` while building// a device. Linux defines UINPUT_MAX_NAME_SIZE as 80, leaving 78 usable bytes.@@ -189,6 +191,34 @@ fn mpx_last_use() -> &'static Mutex<HashMap<String, std::time::Instant>> {MPX_LAST_USE.get_or_init(|| Mutex::new(HashMap::new()))}+#[derive(Default)]+struct ClickPopups {+ windows: Vec<u64>,+ removal_due: Option<Instant>,+}++fn mpx_click_popups() -> &'static Mutex<HashMap<String, ClickPopups>> {+ MPX_CLICK_POPUPS.get_or_init(|| Mutex::new(HashMap::new()))+}++fn mark_mpx_use(cursor_id: &str) {+ mpx_last_use()+ .lock()+ .unwrap()+ .insert(cursor_id.to_owned(), Instant::now());+ if let Some(popups) = mpx_click_popups().lock().unwrap().get_mut(cursor_id) {+ popups.removal_due = None;+ }+}++fn track_click_popups(cursor_id: &str, opened: impl IntoIterator<Item = u64>) {+ let mut tracked = mpx_click_popups().lock().unwrap();+ let windows = &mut tracked.entry(cursor_id.to_owned()).or_default().windows;+ windows.extend(opened);+ windows.sort_unstable();+ windows.dedup();+}+/// Take the MPX operation lock for one call on `cursor_id`, stamp its last/// use, and make sure the idle reaper is running. Hold the guard for the/// whole operation.@@ -196,10 +226,7 @@ fn mpx_op_guard(cursor_id: &str) -> std::sync::MutexGuard<'static, ()> {let guard = MPX_OP_LOCK.lock().unwrap_or_else(|poisoned| poisoned.into_inner());- mpx_last_use()- .lock()- .unwrap()- .insert(cursor_id.to_owned(), std::time::Instant::now());+ mark_mpx_use(cursor_id);MPX_IDLE_REAPER.call_once(|| {std::thread::Builder::new().name("cua-mpx-idle-reaper".into())@@ -214,7 +241,7 @@ fn mpx_op_guard(cursor_id: &str) -> std::sync::MutexGuard<'static, ()> {};for cursor_id in stale {tracing::info!(cursor_id, "removing idle MPX master pair");- forget_master_pointer(&cursor_id);+ forget_master_pointer_locked(&cursor_id);}}).ok();@@ -719,7 +746,78 @@ fn ensure_master_pointer_for_session(}pub fn forget_master_pointer(cursor_id: &str) {- mpx_last_use().lock().unwrap().remove(cursor_id);+ let _op = MPX_OP_LOCK+ .lock()+ .unwrap_or_else(|poisoned| poisoned.into_inner());+ forget_master_pointer_locked(cursor_id);+}++fn forget_master_pointer_locked(cursor_id: &str) {+ let mut tracked = mpx_click_popups().lock().unwrap();+ let mut last_use = mpx_last_use().lock().unwrap();+ match tracked.get_mut(cursor_id) {+ Some(popups) if popups.removal_due.is_some() => {}+ Some(popups) if !popups.windows.is_empty() => {+ let remove_by = *last_use.get(cursor_id).unwrap_or(&Instant::now()) + MPX_IDLE_TTL;+ match spawn_popup_waiter(cursor_id.to_owned(), popups.windows.clone(), remove_by) {+ Ok(_) => {+ popups.removal_due = Some(remove_by);+ last_use.remove(cursor_id);+ }+ Err(error) => tracing::warn!(cursor_id, %error, "MPX popup waiter not started"),+ }+ }+ _ => {+ tracked.remove(cursor_id);+ last_use.remove(cursor_id);+ remove_master_pair(cursor_id);+ }+ }+}++fn popups_mapped(windows: &[u64]) -> Option<bool> {+ let (conn, _) = RustConnection::connect(None).ok()?;+ let mut mapped = false;+ for &window in windows {+ match conn.get_window_attributes(window as Window).ok()?.reply() {+ Ok(attrs) => mapped |= attrs.map_state == MapState::VIEWABLE,+ Err(x11rb::errors::ReplyError::X11Error(_)) => {}+ Err(_) => return None,+ }+ }+ Some(mapped)+}++fn popup_wait_over(mapped: Option<bool>, quiet: &mut Option<Instant>, remove_by: Instant) -> bool {+ let now = Instant::now();+ *quiet = (mapped == Some(false)).then(|| quiet.unwrap_or(now));+ quiet.is_some_and(|since| now - since >= MPX_POPUP_QUIET) || now >= remove_by+}++fn spawn_popup_waiter(cursor_id: String, windows: Vec<u64>, remove_by: Instant) -> Result<()> {+ std::thread::Builder::new()+ .name("cua-mpx-popup-wait".into())+ .spawn(move || {+ let mut quiet = None;+ loop {+ sleep(Duration::from_millis(100));+ let _op = MPX_OP_LOCK+ .lock()+ .unwrap_or_else(|poisoned| poisoned.into_inner());+ let mut tracked = mpx_click_popups().lock().unwrap();+ if tracked.get(&cursor_id).and_then(|p| p.removal_due) != Some(remove_by) {+ return;+ }+ if popup_wait_over(popups_mapped(&windows), &mut quiet, remove_by) {+ tracked.remove(&cursor_id);+ return remove_master_pair(&cursor_id);+ }+ }+ })?;+ Ok(())+}++fn remove_master_pair(cursor_id: &str) {// Drop the uinput slaves first: closing the fds unplugs them, so the master// removal below never has to hand a live slave back to the user's core// devices (XIAttachToMaster only re-homes slaves that still exist).@@ -1440,11 +1538,8 @@ pub fn send_parallel_virtual_pointer_drags(drags: &[(String, VirtualPointerDrag).map(|(id, _)| id.as_str()).unwrap_or("default"),);- {- let mut last_use = mpx_last_use().lock().unwrap();- for (cursor_id, _) in drags {- last_use.insert(cursor_id.clone(), std::time::Instant::now());- }+ for (cursor_id, _) in drags {+ mark_mpx_use(cursor_id);}let display = open_display()?;supports_parallel_pointer_injection(display)?;@@ -2561,10 +2656,6 @@ pub fn send_virtual_pointer_click(click: &VirtualPointerClick,) -> Result<PointerEffect> {let _op = mpx_op_guard(cursor_id);- mpx_last_use()- .lock()- .unwrap()- .insert(cursor_id.to_owned(), std::time::Instant::now());let display = open_display()?;supports_parallel_pointer_injection(display)?;let saved_focus = save_focus_state(display);@@ -2631,6 +2722,7 @@ pub fn send_virtual_pointer_click(click.y,);effect.retargeted_to = retargeted_to;+ track_click_popups(cursor_id, effect.popups.iter().map(|popup| popup.window));Ok(effect)})();@@ -2650,10 +2742,6 @@ pub fn send_virtual_pointer_drag(drag: &VirtualPointerDrag,) -> Result<PointerEffect> {let _op = mpx_op_guard(cursor_id);- mpx_last_use()- .lock()- .unwrap()- .insert(cursor_id.to_owned(), std::time::Instant::now());let display = open_display()?;supports_parallel_pointer_injection(display)?;let saved_focus = save_focus_state(display);@@ -5052,6 +5140,96 @@ mod path_tests {assert!(stale_cursor_ids(&std::collections::HashMap::new(), now, MPX_IDLE_TTL).is_empty());}+ fn popup_removal_due(cursor_id: &str) -> Option<Option<std::time::Instant>> {+ super::mpx_click_popups()+ .lock()+ .unwrap()+ .get(cursor_id)+ .map(|popups| popups.removal_due)+ }++ fn mpx_op_lock() -> std::sync::MutexGuard<'static, ()> {+ super::MPX_OP_LOCK+ .lock()+ .unwrap_or_else(|poisoned| poisoned.into_inner())+ }++ fn end_session_after_click(cursor_id: &str, opened: &[u64]) {+ super::mark_mpx_use(cursor_id);+ super::track_click_popups(cursor_id, opened.iter().copied());+ super::forget_master_pointer_locked(cursor_id);+ }++ #[test]+ fn a_second_end_leaves_the_master_to_the_pending_popup_waiter() {+ let _op = mpx_op_lock();+ let cursor_id = "popup-test-double-end";+ end_session_after_click(cursor_id, &[0x7fff_0001]);+ let due = popup_removal_due(cursor_id);+ assert!(matches!(due, Some(Some(_))));+ super::forget_master_pointer_locked(cursor_id);+ assert_eq!(popup_removal_due(cursor_id), due);+ assert!(!super::mpx_last_use()+ .lock()+ .unwrap()+ .contains_key(cursor_id));+ }++ #[test]+ fn an_end_without_tracked_popups_removes_the_master_at_once() {+ let _op = mpx_op_lock();+ end_session_after_click("popup-test-no-popup", &[]);+ assert_eq!(popup_removal_due("popup-test-no-popup"), None);+ assert!(!super::mpx_last_use()+ .lock()+ .unwrap()+ .contains_key("popup-test-no-popup"));+ }++ #[test]+ fn a_click_that_opens_nothing_keeps_earlier_tracked_popups() {+ let _op = mpx_op_lock();+ let cursor_id = "popup-test-union";+ super::track_click_popups(cursor_id, [5, 3]);+ super::track_click_popups(cursor_id, []);+ super::track_click_popups(cursor_id, [3, 7]);+ let windows = super::mpx_click_popups().lock().unwrap()[cursor_id]+ .windows+ .clone();+ assert_eq!(windows, vec![3, 5, 7]);+ }++ #[test]+ fn reusing_an_ended_session_disowns_its_popup_waiter() {+ let _op = mpx_op_lock();+ let cursor_id = "popup-test-reuse";+ end_session_after_click(cursor_id, &[0x7fff_0003]);+ assert!(matches!(popup_removal_due(cursor_id), Some(Some(_))));+ super::mark_mpx_use(cursor_id);+ assert_eq!(popup_removal_due(cursor_id), Some(None));+ super::forget_master_pointer_locked(cursor_id);+ assert!(matches!(popup_removal_due(cursor_id), Some(Some(_))));+ }++ #[test]+ fn a_popup_that_remaps_during_the_quiet_wait_keeps_the_master_until_the_deadline() {+ use super::{popup_wait_over, MPX_POPUP_QUIET};+ let now = std::time::Instant::now();+ let deadline = now + std::time::Duration::from_secs(60);+ let quiet_elapsed = Some(now - MPX_POPUP_QUIET * 2);+ let mut quiet = quiet_elapsed;+ assert!(!popup_wait_over(Some(true), &mut quiet, deadline));+ assert_eq!(quiet, None);+ assert!(!popup_wait_over(Some(false), &mut quiet, deadline));+ assert!(quiet.is_some());+ let mut quiet = quiet_elapsed;+ assert!(!popup_wait_over(None, &mut quiet, deadline));+ assert_eq!(quiet, None);+ let mut quiet = quiet_elapsed;+ assert!(popup_wait_over(Some(false), &mut quiet, deadline));+ assert!(popup_wait_over(Some(true), &mut None, now));+ }+#[test]fn slave_pointer_name_fits_evdev_uinput_limit() {for cursor_id in ["m".repeat(200), "cursor-鼠".repeat(50)] {