Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

fix(cua-driver): refuse background pixel clicks LibreOffice cannot receive (#4864)

Francesco Bonacci49e924c46328
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/input/pointer_toolkit.rs b/libs/cua-driver/rust/crates/platform-macos/src/input/pointer_toolkit.rs
index d6ac6057d8..49fefea4cb 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/input/pointer_toolkit.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/input/pointer_toolkit.rs
@@ -15,16 +15,40 @@
//! loaded only from the dyld shared cache is not visible this way; such targets
//! fall back to the honest "not driver-verified" background result.
-/// A UI toolkit known to derive click locations from the hardware pointer.
+/// A UI toolkit that background (PID-routed) mouse events cannot reach at the
+/// requested point.
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub enum PointerReadingToolkit {
+ /// Tk derives click locations from the hardware pointer.
Tk,
+ /// LibreOffice's macOS VCL plug-in (`libvclplug_osxlo.dylib`) drops
+ /// PID-routed mouse events: a background click or right-click on a Calc
+ /// cell leaves the selection where it was (VM check, 8 Oct 2026), and
+ /// the AX press its cells accept does not select them either.
+ Vcl,
}
impl PointerReadingToolkit {
pub fn name(self) -> &'static str {
match self {
Self::Tk => "tk",
+ Self::Vcl => "libreoffice-vcl",
+ }
+ }
+
+ /// Why background delivery cannot reach this toolkit, as a clause that
+ /// follows "its <name> toolkit".
+ pub fn why(self) -> &'static str {
+ match self {
+ Self::Tk => {
+ "derives click locations from the hardware pointer, and background delivery \
+ never moves the pointer, so the click would land wherever the pointer \
+ currently is"
+ }
+ Self::Vcl => {
+ "ignores background (PID-routed) mouse events, so the click would not reach \
+ the target and the selection would stay where it is"
+ }
}
}
}
@@ -40,6 +64,10 @@ pub fn classify_image_path(path: &str) -> Option<PointerReadingToolkit> {
}
let file = path.rsplit('/').next().unwrap_or(path);
let lower = file.to_ascii_lowercase();
+ // LibreOffice's macOS VCL plug-in: `libvclplug_osxlo.dylib`.
+ if lower.starts_with("libvclplug_osx") && lower.ends_with(".dylib") {
+ return Some(PointerReadingToolkit::Vcl);
+ }
if lower.starts_with("_tkinter") && lower.ends_with(".so") {
return Some(PointerReadingToolkit::Tk);
}
@@ -205,6 +233,22 @@ mod tests {
}
}
+ #[test]
+ fn classifies_the_libreoffice_vcl_plugin() {
+ assert_eq!(
+ classify_image_path(
+ "/Applications/LibreOffice.app/Contents/Frameworks/libvclplug_osxlo.dylib"
+ ),
+ Some(PointerReadingToolkit::Vcl)
+ );
+ for path in [
+ "/Applications/LibreOffice.app/Contents/Frameworks/libvcllo.dylib",
+ "/Applications/LibreOffice.app/Contents/Frameworks/libvclplug_osxlo.dylib.bak",
+ ] {
+ assert_eq!(classify_image_path(path), None, "{path}");
+ }
+ }
+
#[cfg(target_os = "macos")]
#[test]
fn kernel_region_structure_matches_proc_info_layout() {
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/click.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/click.rs
index bb1ed790c6..52b59c5df4 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/click.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/click.rs
@@ -1050,6 +1050,21 @@ impl Tool for ClickTool {
CFRelease(element as _);
return Ok(false);
}
+ // An element that does not advertise AXPress can still
+ // accept it as a no-op. On a toolkit that drops routed
+ // mouse events (LibreOffice's Calc cells) that no-op used
+ // to come back as a pressed click; fall through so the
+ // pixel route refuses instead.
+ if !focus_only
+ && !crate::ax::bindings::copy_action_names(element)
+ .iter()
+ .any(|action| action == "AXPress")
+ && crate::input::pointer_toolkit::detect(pid)
+ == Some(crate::input::pointer_toolkit::PointerReadingToolkit::Vcl)
+ {
+ CFRelease(element as _);
+ return Ok(false);
+ }
let delivered = if focus_only {
crate::input::ax_actions::focus_element(element as usize).is_ok()
} else {
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/pixel_route.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/pixel_route.rs
index 36a4df3ff3..b7233f50ed 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/pixel_route.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/pixel_route.rs
@@ -81,17 +81,24 @@ pub(crate) fn pointer_reading_background_refusal(
event_kind: &str,
) -> ToolResult {
let name = toolkit.name();
+ let why = toolkit.why();
cua_driver_core::delivery::background_unavailable_result(
format!(
"Background pixel click is not available for pid {pid}: its {name} toolkit \
- derives click locations from the hardware pointer, and background delivery \
- never moves the pointer, so the click would land wherever the pointer \
- currently is. Retry this action with delivery_mode:\"foreground\"; Cua \
+ {why}. Retry this action with delivery_mode:\"foreground\"; Cua \
Driver will activate the window, move the pointer to the target, and click."
),
"background_unavailable",
- "the target toolkit reads the hardware pointer position, which background \
- PID-routed events cannot move; retry with delivery_mode:\"foreground\".",
+ match toolkit {
+ PointerReadingToolkit::Tk => {
+ "the target toolkit reads the hardware pointer position, which background \
+ PID-routed events cannot move; retry with delivery_mode:\"foreground\"."
+ }
+ PointerReadingToolkit::Vcl => {
+ "the target toolkit ignores background PID-routed mouse events; retry with \
+ delivery_mode:\"foreground\"."
+ }
+ },
serde_json::json!({
"event_kind": event_kind,
"reason": "pointer_reading_toolkit",
@@ -197,6 +204,20 @@ mod tests {
assert_eq!(structured["escalation"]["recommended"], "foreground");
}
+ #[test]
+ fn libreoffice_background_clicks_are_refused_with_the_foreground_route() {
+ assert_eq!(
+ pixel_click_route(false, true, Some(PointerReadingToolkit::Vcl)),
+ PixelClickRoute::RefuseBackground(PointerReadingToolkit::Vcl)
+ );
+ let result =
+ pointer_reading_background_refusal(42, PointerReadingToolkit::Vcl, "mouse_click");
+ let structured = result.structured_content.expect("structured refusal");
+ assert_eq!(structured["code"], "background_unavailable");
+ assert_eq!(structured["toolkit"], "libreoffice-vcl");
+ assert_eq!(structured["escalation"]["recommended"], "foreground");
+ }
+
#[test]
fn foreground_failure_is_structured() {
let result = foreground_unavailable("click", 7, "window never focused");