Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
fix(cua-driver): refuse background pixel clicks LibreOffice cannot receive (#4864)
Francesco Bonacci49e924c46328
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/input/pointer_toolkit.rs b/libs/cua-driver/rust/crates/platform-macos/src/input/pointer_toolkit.rsindex d6ac6057d8..49fefea4cb 100644--- a/libs/cua-driver/rust/crates/platform-macos/src/input/pointer_toolkit.rs+++ b/libs/cua-driver/rust/crates/platform-macos/src/input/pointer_toolkit.rs@@ -15,16 +15,40 @@//! loaded only from the dyld shared cache is not visible this way; such targets//! fall back to the honest "not driver-verified" background result.-/// A UI toolkit known to derive click locations from the hardware pointer.+/// A UI toolkit that background (PID-routed) mouse events cannot reach at the+/// requested point.#[derive(Clone, Copy, Debug, Eq, PartialEq)]pub enum PointerReadingToolkit {+ /// Tk derives click locations from the hardware pointer.Tk,+ /// LibreOffice's macOS VCL plug-in (`libvclplug_osxlo.dylib`) drops+ /// PID-routed mouse events: a background click or right-click on a Calc+ /// cell leaves the selection where it was (VM check, 8 Oct 2026), and+ /// the AX press its cells accept does not select them either.+ Vcl,}impl PointerReadingToolkit {pub fn name(self) -> &'static str {match self {Self::Tk => "tk",+ Self::Vcl => "libreoffice-vcl",+ }+ }++ /// Why background delivery cannot reach this toolkit, as a clause that+ /// follows "its <name> toolkit".+ pub fn why(self) -> &'static str {+ match self {+ Self::Tk => {+ "derives click locations from the hardware pointer, and background delivery \+ never moves the pointer, so the click would land wherever the pointer \+ currently is"+ }+ Self::Vcl => {+ "ignores background (PID-routed) mouse events, so the click would not reach \+ the target and the selection would stay where it is"+ }}}}@@ -40,6 +64,10 @@ pub fn classify_image_path(path: &str) -> Option<PointerReadingToolkit> {}let file = path.rsplit('/').next().unwrap_or(path);let lower = file.to_ascii_lowercase();+ // LibreOffice's macOS VCL plug-in: `libvclplug_osxlo.dylib`.+ if lower.starts_with("libvclplug_osx") && lower.ends_with(".dylib") {+ return Some(PointerReadingToolkit::Vcl);+ }if lower.starts_with("_tkinter") && lower.ends_with(".so") {return Some(PointerReadingToolkit::Tk);}@@ -205,6 +233,22 @@ mod tests {}}+ #[test]+ fn classifies_the_libreoffice_vcl_plugin() {+ assert_eq!(+ classify_image_path(+ "/Applications/LibreOffice.app/Contents/Frameworks/libvclplug_osxlo.dylib"+ ),+ Some(PointerReadingToolkit::Vcl)+ );+ for path in [+ "/Applications/LibreOffice.app/Contents/Frameworks/libvcllo.dylib",+ "/Applications/LibreOffice.app/Contents/Frameworks/libvclplug_osxlo.dylib.bak",+ ] {+ assert_eq!(classify_image_path(path), None, "{path}");+ }+ }+#[cfg(target_os = "macos")]#[test]fn kernel_region_structure_matches_proc_info_layout() {diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/click.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/click.rsindex bb1ed790c6..52b59c5df4 100644--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/click.rs+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/click.rs@@ -1050,6 +1050,21 @@ impl Tool for ClickTool {CFRelease(element as _);return Ok(false);}+ // An element that does not advertise AXPress can still+ // accept it as a no-op. On a toolkit that drops routed+ // mouse events (LibreOffice's Calc cells) that no-op used+ // to come back as a pressed click; fall through so the+ // pixel route refuses instead.+ if !focus_only+ && !crate::ax::bindings::copy_action_names(element)+ .iter()+ .any(|action| action == "AXPress")+ && crate::input::pointer_toolkit::detect(pid)+ == Some(crate::input::pointer_toolkit::PointerReadingToolkit::Vcl)+ {+ CFRelease(element as _);+ return Ok(false);+ }let delivered = if focus_only {crate::input::ax_actions::focus_element(element as usize).is_ok()} else {diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/pixel_route.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/pixel_route.rsindex 36a4df3ff3..b7233f50ed 100644--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/pixel_route.rs+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/pixel_route.rs@@ -81,17 +81,24 @@ pub(crate) fn pointer_reading_background_refusal(event_kind: &str,) -> ToolResult {let name = toolkit.name();+ let why = toolkit.why();cua_driver_core::delivery::background_unavailable_result(format!("Background pixel click is not available for pid {pid}: its {name} toolkit \- derives click locations from the hardware pointer, and background delivery \- never moves the pointer, so the click would land wherever the pointer \- currently is. Retry this action with delivery_mode:\"foreground\"; Cua \+ {why}. Retry this action with delivery_mode:\"foreground\"; Cua \Driver will activate the window, move the pointer to the target, and click."),"background_unavailable",- "the target toolkit reads the hardware pointer position, which background \- PID-routed events cannot move; retry with delivery_mode:\"foreground\".",+ match toolkit {+ PointerReadingToolkit::Tk => {+ "the target toolkit reads the hardware pointer position, which background \+ PID-routed events cannot move; retry with delivery_mode:\"foreground\"."+ }+ PointerReadingToolkit::Vcl => {+ "the target toolkit ignores background PID-routed mouse events; retry with \+ delivery_mode:\"foreground\"."+ }+ },serde_json::json!({"event_kind": event_kind,"reason": "pointer_reading_toolkit",@@ -197,6 +204,20 @@ mod tests {assert_eq!(structured["escalation"]["recommended"], "foreground");}+ #[test]+ fn libreoffice_background_clicks_are_refused_with_the_foreground_route() {+ assert_eq!(+ pixel_click_route(false, true, Some(PointerReadingToolkit::Vcl)),+ PixelClickRoute::RefuseBackground(PointerReadingToolkit::Vcl)+ );+ let result =+ pointer_reading_background_refusal(42, PointerReadingToolkit::Vcl, "mouse_click");+ let structured = result.structured_content.expect("structured refusal");+ assert_eq!(structured["code"], "background_unavailable");+ assert_eq!(structured["toolkit"], "libreoffice-vcl");+ assert_eq!(structured["escalation"]["recommended"], "foreground");+ }+#[test]fn foreground_failure_is_structured() {let result = foreground_unavailable("click", 7, "window never focused");