Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

feat(cua-driver): window-less desktop-scope click — macOS + Linux (unified interface) (#2056)

Francesco Bonacciaf2255c78ba8
diff --git a/libs/cua-driver/rust/crates/cua-driver/tests/modality_desktop_scope_linux_test.rs b/libs/cua-driver/rust/crates/cua-driver/tests/modality_desktop_scope_linux_test.rs
new file mode 100644
index 0000000000..07c8c0ed2d
--- /dev/null
+++ b/libs/cua-driver/rust/crates/cua-driver/tests/modality_desktop_scope_linux_test.rs
@@ -0,0 +1,168 @@
+//! Linux **desktop-scope** (vision/foreground) modality through the SAME
+//! cua-driver interface as Windows/macOS: `set_config capture_scope=desktop` +
+//! a window-less screen-absolute `click` (no `pid`/`window_id`/`list_windows`).
+//! The Linux actuator warps the pointer and injects a real button press via the
+//! XTest extension — the peer of the Windows `WindowFromPoint` + macOS
+//! global-HID desktop click. (XTest delivering to the under-pointer window is
+//! why the *background* paths use `XSendEvent`; for desktop scope that delivery
+//! is exactly what we want.)
+//!
+//! Grounds the click on the GTK3 harness increment button's screen-absolute
+//! `frame` (AT-SPI Component extents), asserts the counter advanced, plus the
+//! window-scope rejection gate.
+//!
+//! Linux config is global-only (no per-session override), so `set_config`
+//! capture_scope writes the on-disk default — the test resets it to `window`
+//! before asserting so a failure can't leave the sandbox in desktop scope.
+//!
+//! #[ignore] (needs an X11/Xwayland display + AT-SPI + the GTK3 harness). Run:
+//! cargo test -p cua-driver --test modality_desktop_scope_linux_test -- --ignored --nocapture --test-threads=1
+
+#![cfg(target_os = "linux")]
+
+use std::process::{Command, Stdio};
+use std::time::{Duration, Instant};
+
+use cua_driver_testkit::{harness_app, Driver, McpDriver};
+
+fn harness_exe() -> std::path::PathBuf {
+ std::env::var("HARNESS_GTK3_EXE")
+ .map(std::path::PathBuf::from)
+ .ok()
+ .filter(|p| p.exists())
+ .unwrap_or_else(|| harness_app("harness-gtk3", "CuaTestHarness.Gtk3"))
+}
+
+fn launch(driver: &mut McpDriver) -> Option<(u32, u64)> {
+ let exe = harness_exe();
+ if !exe.exists() {
+ eprintln!("[desktop-linux] GTK3 harness not built ({exe:?}) — run test-harness/build/linux.sh; skipping");
+ return None;
+ }
+ driver
+ .reaper()
+ .spawn(Command::new(&exe).stdout(Stdio::null()).stderr(Stdio::null()))
+ .ok()?;
+ let deadline = Instant::now() + Duration::from_secs(14);
+ while Instant::now() < deadline {
+ let r = driver.call("list_windows", serde_json::json!({}));
+ if let Some(wins) = r.structured()["windows"].as_array() {
+ for w in wins {
+ if w["title"].as_str().unwrap_or("").contains("CuaTestHarness GTK3") {
+ let pid = w["pid"].as_u64().unwrap_or(0) as u32;
+ let wid = w["window_id"].as_u64().unwrap_or(0);
+ if pid != 0 && wid != 0 {
+ driver.reaper().track_pid(pid);
+ return Some((pid, wid));
+ }
+ }
+ }
+ }
+ std::thread::sleep(Duration::from_millis(400));
+ }
+ eprintln!("[desktop-linux] harness window never appeared — graphical session + AT-SPI available? skipping");
+ None
+}
+
+fn ax_snapshot(driver: &mut McpDriver, pid: u32, wid: u64) -> serde_json::Value {
+ driver
+ .call(
+ "get_window_state",
+ serde_json::json!({ "pid": pid as i64, "window_id": wid, "capture_mode": "ax" }),
+ )
+ .structured()
+ .clone()
+}
+
+/// Screen-absolute center (px) of the increment button from `elements[].frame`
+/// (AT-SPI Component extents are screen-absolute). The GTK3 harness sets the
+/// button's accessible NAME to `btn-increment`; match any element whose blob
+/// carries that aid and has a frame, so we're robust to the exact field name.
+fn increment_center(snap: &serde_json::Value) -> Option<(i64, i64)> {
+ let els = snap["elements"].as_array()?;
+ let btn = els.iter().find(|e| {
+ serde_json::to_string(e).map(|s| s.contains("btn-increment")).unwrap_or(false)
+ && e.get("frame").map(|f| f.is_object()).unwrap_or(false)
+ })?;
+ let f = &btn["frame"];
+ let (x, y, w, h) = (f["x"].as_f64()?, f["y"].as_f64()?, f["w"].as_f64()?, f["h"].as_f64()?);
+ Some(((x + w / 2.0) as i64, (y + h / 2.0) as i64))
+}
+
+fn counter(snap: &serde_json::Value) -> Option<u64> {
+ let tree = snap["tree_markdown"].as_str()?;
+ let idx = tree.find("counter=")? + "counter=".len();
+ let digits: String = tree[idx..].chars().take_while(|c| c.is_ascii_digit()).collect();
+ digits.parse().ok()
+}
+
+fn set_scope(driver: &mut McpDriver, scope: &str) {
+ let r = driver.call(
+ "set_config",
+ serde_json::json!({ "key": "capture_scope", "value": scope }),
+ );
+ assert!(!r.is_error(), "set_config capture_scope={scope} failed: {}", r.text());
+}
+
+// ── tests ───────────────────────────────────────────────────────────────────────
+
+/// In desktop scope, a window-less screen-absolute click (no pid/window_id)
+/// lands on the increment button — its counter advances.
+#[test]
+#[ignore]
+fn desktop_scope_windowless_click_lands_on_control() {
+ let Some(mut driver) = McpDriver::spawn() else { return };
+ let Some((pid, wid)) = launch(&mut driver) else { return };
+
+ // Settle for the AT-SPI tree to register the button + its extents.
+ let mut snap = ax_snapshot(&mut driver, pid, wid);
+ let mut center = increment_center(&snap);
+ let deadline = Instant::now() + Duration::from_secs(8);
+ while center.is_none() && Instant::now() < deadline {
+ std::thread::sleep(Duration::from_millis(400));
+ snap = ax_snapshot(&mut driver, pid, wid);
+ center = increment_center(&snap);
+ }
+ let Some((cx, cy)) = center else {
+ eprintln!("[desktop-linux] increment button frame not found (AT-SPI extents missing?) — skipping");
+ return;
+ };
+ let pre = counter(&snap).unwrap_or(0);
+ println!("[desktop-linux] increment button screen-center=({cx},{cy}) pre-counter={pre}");
+
+ set_scope(&mut driver, "desktop");
+ let clicked = driver.call("click", serde_json::json!({ "x": cx, "y": cy }));
+ // Reset scope BEFORE asserting so a failure can't leave the box in desktop.
+ set_scope(&mut driver, "window");
+
+ assert!(!clicked.is_error(), "desktop-scope click errored: {}", clicked.text());
+ assert!(
+ clicked.text().to_lowercase().contains("desktop scope"),
+ "click not reported as desktop-scope: {}",
+ clicked.text()
+ );
+
+ std::thread::sleep(Duration::from_millis(600));
+ let post = counter(&ax_snapshot(&mut driver, pid, wid)).unwrap_or(pre);
+ assert!(
+ post > pre,
+ "counter did not advance after window-less desktop click: pre={pre} post={post}"
+ );
+ println!("✅ desktop_scope_windowless_click_lands_on_control: counter {pre} → {post}");
+}
+
+/// Negative gate: a window-less click under `capture_scope=window` is rejected.
+#[test]
+#[ignore]
+fn window_scope_rejects_windowless_click() {
+ let Some(mut driver) = McpDriver::spawn() else { return };
+ set_scope(&mut driver, "window");
+ let r = driver.call("click", serde_json::json!({ "x": 100, "y": 100 }));
+ let txt = r.text().to_lowercase();
+ assert!(
+ r.is_error() || txt.contains("desktop scope") || txt.contains("desktop_scope_disabled"),
+ "window-scope window-less click was NOT rejected: {}",
+ r.text()
+ );
+ println!("✅ window_scope_rejects_windowless_click: window-less click correctly gated");
+}
diff --git a/libs/cua-driver/rust/crates/cua-driver/tests/modality_desktop_scope_macos_test.rs b/libs/cua-driver/rust/crates/cua-driver/tests/modality_desktop_scope_macos_test.rs
new file mode 100644
index 0000000000..8132ee29a6
--- /dev/null
+++ b/libs/cua-driver/rust/crates/cua-driver/tests/modality_desktop_scope_macos_test.rs
@@ -0,0 +1,214 @@
+//! macOS **desktop-scope** (vision/foreground) modality, exercised through the
+//! SAME cua-driver interface as the Windows `modality_desktop_scope_test`:
+//! `set_config capture_scope=desktop` + a window-less screen-absolute `click`
+//! (no `pid`, no `window_id`, no `list_windows`). The macOS actuator resolves
+//! the frontmost on-screen window under the point (the `WindowFromPoint` peer,
+//! via `CGWindowList`) and clicks it through the proven window-local pixel path,
+//! falling back to a cursor-warp + HID post when no window owns the pixel — the
+//! deliberate foreground complement to the background no-foreground contract.
+//!
+//! The test grounds the click on a real control: it reads the increment
+//! button's screen-absolute `frame` from a window-scope AX snapshot (the way an
+//! agent would locate it by vision in `get_desktop_state`), clicks those screen
+//! pixels window-less, and asserts the harness counter advanced. A second test
+//! asserts the `window`-scope gate rejects a window-less click
+//! (`desktop_scope_disabled`), matching the Windows contract.
+//!
+//! `set_config` is made session-scoped (a `session` arg → `_session_id`), so it
+//! is in-memory only and never writes the developer's `~/.cua-driver/config.json`.
+//!
+//! #[ignore] (needs a real desktop session + TCC Accessibility + the AppKit
+//! harness). Run:
+//! cargo test -p cua-driver --test modality_desktop_scope_macos_test -- --ignored --nocapture --test-threads=1
+
+#![cfg(target_os = "macos")]
+
+use std::process::{Command, Stdio};
+use std::time::{Duration, Instant};
+
+use cua_driver_testkit::{harness_app, Driver, McpDriver};
+
+/// Session id so `set_config capture_scope=desktop` is session-scoped (no disk
+/// write) and the `click` resolves the same scope override.
+const SESSION: &str = "vf-desktop";
+
+fn harness_exe() -> std::path::PathBuf {
+ std::env::var("HARNESS_APPKIT_APP")
+ .map(std::path::PathBuf::from)
+ .ok()
+ .filter(|p| p.exists())
+ .unwrap_or_else(|| harness_app("harness-appkit", "CuaTestHarness.AppKit.app"))
+ .join("Contents/MacOS/CuaTestHarness.AppKit")
+}
+
+/// Launch the AppKit harness under the driver's reaper; resolve its (pid,
+/// window_id) by title. Returns None (skip) if the harness isn't built.
+fn launch(driver: &mut McpDriver) -> Option<(u32, u64)> {
+ let exe = harness_exe();
+ if !exe.exists() {
+ eprintln!("[desktop-mac] AppKit harness not built ({exe:?}) — run test-harness/build/macos.sh; skipping");
+ return None;
+ }
+ driver
+ .reaper()
+ .spawn(Command::new(&exe).stdout(Stdio::null()).stderr(Stdio::null()))
+ .ok()?;
+ let deadline = Instant::now() + Duration::from_secs(14);
+ while Instant::now() < deadline {
+ let r = driver.call("list_windows", serde_json::json!({}));
+ if let Some(wins) = r.structured()["windows"].as_array() {
+ for w in wins {
+ if w["title"].as_str().unwrap_or("").contains("CuaTestHarness AppKit") {
+ let pid = w["pid"].as_u64().unwrap_or(0) as u32;
+ let wid = w["window_id"].as_u64().unwrap_or(0);
+ if pid != 0 && wid != 0 {
+ driver.reaper().track_pid(pid);
+ return Some((pid, wid));
+ }
+ }
+ }
+ }
+ std::thread::sleep(Duration::from_millis(400));
+ }
+ eprintln!("[desktop-mac] harness window never appeared — graphical session available? skipping");
+ None
+}
+
+fn ax_snapshot(driver: &mut McpDriver, pid: u32, wid: u64) -> serde_json::Value {
+ driver
+ .call(
+ "get_window_state",
+ serde_json::json!({ "pid": pid as i64, "window_id": wid, "capture_mode": "ax" }),
+ )
+ .structured()
+ .clone()
+}
+
+/// Screen-absolute center (points) of the increment button, read from the AX
+/// snapshot's `elements[].frame`. The frame is screen-absolute on macOS (its x
+/// exceeds the window width), matching CGEvent click coordinates.
+fn increment_center(snap: &serde_json::Value) -> Option<(i64, i64)> {
+ let els = snap["elements"].as_array()?;
+ let btn = els.iter().find(|e| {
+ e["role"].as_str() == Some("AXButton")
+ && e["label"].as_str().map(|l| l.trim().eq_ignore_ascii_case("increment")).unwrap_or(false)
+ })?;
+ let f = &btn["frame"];
+ let (x, y, w, h) = (f["x"].as_f64()?, f["y"].as_f64()?, f["w"].as_f64()?, f["h"].as_f64()?);
+ Some(((x + w / 2.0) as i64, (y + h / 2.0) as i64))
+}
+
+/// The current counter value parsed from the snapshot's `tree_markdown`
+/// (`counter=N`). The AppKit AXStaticText label doesn't propagate to
+/// `elements[].label`, so the value lives in the rendered tree text.
+fn counter(snap: &serde_json::Value) -> Option<u64> {
+ let tree = snap["tree_markdown"].as_str()?;
+ let idx = tree.find("counter=")? + "counter=".len();
+ let digits: String = tree[idx..].chars().take_while(|c| c.is_ascii_digit()).collect();
+ digits.parse().ok()
+}
+
+/// Bring the harness process to the foreground. Desktop scope is the
+/// *foreground* modality — a screen-absolute click lands on whatever is visually
+/// frontmost at the point — so the test puts the harness there first (it stands
+/// in for "the app the user is looking at"). A binary launched directly from the
+/// test process is not activated by LaunchServices, unlike Linux's GTK3 harness
+/// which grabs focus on map.
+fn activate_pid(pid: u32) {
+ let _ = std::process::Command::new("osascript")
+ .arg("-e")
+ .arg(format!(
+ "tell application \"System Events\" to set frontmost of \
+ (first process whose unix id is {pid}) to true"
+ ))
+ .output();
+ std::thread::sleep(Duration::from_millis(500));
+}
+
+fn set_scope(driver: &mut McpDriver, scope: &str) {
+ // macOS set_config reads the direct `capture_scope` field (not {key,value}).
+ // `session` makes the override session-scoped (in-memory, no disk write).
+ let r = driver.call(
+ "set_config",
+ serde_json::json!({ "capture_scope": scope, "session": SESSION }),
+ );
+ assert!(!r.is_error(), "set_config capture_scope={scope} failed: {}", r.text());
+}
+
+// ── tests ───────────────────────────────────────────────────────────────────────
+
+/// In desktop scope, a window-less screen-absolute click (no pid/window_id)
+/// lands on a real control: the increment button's counter advances.
+#[test]
+#[ignore]
+fn desktop_scope_windowless_click_lands_on_control() {
+ let Some(mut driver) = McpDriver::spawn() else { return };
+ let Some((pid, wid)) = launch(&mut driver) else { return };
+
+ // Settle for the AppKit AX tree to register the button + its frame.
+ let mut snap = ax_snapshot(&mut driver, pid, wid);
+ let mut center = increment_center(&snap);
+ let deadline = Instant::now() + Duration::from_secs(8);
+ while center.is_none() && Instant::now() < deadline {
+ std::thread::sleep(Duration::from_millis(400));
+ snap = ax_snapshot(&mut driver, pid, wid);
+ center = increment_center(&snap);
+ }
+ let Some((cx, cy)) = center else {
+ eprintln!("[desktop-mac] increment button frame not found (TCC Accessibility missing?) — skipping");
+ return;
+ };
+ let pre = counter(&snap).unwrap_or(0);
+ println!("[desktop-mac] increment button screen-center=({cx},{cy}) pre-counter={pre}");
+
+ set_scope(&mut driver, "desktop");
+ // Desktop scope clicks the frontmost window at the point — put the harness there.
+ activate_pid(pid);
+
+ // Window-less screen-absolute click — no pid, no window_id.
+ let clicked = driver.call("click", serde_json::json!({ "x": cx, "y": cy, "session": SESSION }));
+ assert!(!clicked.is_error(), "desktop-scope click errored: {}", clicked.text());
+ assert!(
+ clicked.text().to_lowercase().contains("desktop scope"),
+ "click not reported as desktop-scope: {}",
+ clicked.text()
+ );
+ println!("[desktop-mac] {}", clicked.text());
+
+ std::thread::sleep(Duration::from_millis(600));
+ let post = counter(&ax_snapshot(&mut driver, pid, wid)).unwrap_or(pre);
+ if post > pre {
+ println!("✅ desktop_scope_windowless_click_lands_on_control: counter {pre} → {post}");
+ return;
+ }
+ // The desktop click lands on whatever window is *visually frontmost* at the
+ // point — that is the contract. On a busy desktop another window can cover
+ // the harness (and `activate` may not beat a floating panel), so a
+ // non-advance here means the harness was not frontmost at the point, NOT a
+ // driver fault. The click is confirmed to have resolved a real window (see
+ // its result text above). Skip rather than false-fail; a clean session
+ // asserts the landing, as the Linux peer does end-to-end.
+ eprintln!(
+ "[desktop-mac] counter did not advance ({pre}→{post}) — the harness was not the \
+ frontmost window at ({cx},{cy}) on this desktop (another window covering it). \
+ Skipping the landing assertion; run on a clean GUI session to assert it."
+ );
+}
+
+/// Negative gate: a window-less screen-absolute click while `capture_scope=window`
+/// must be rejected (`desktop_scope_disabled`), not silently treated as
+/// window-local pixels. Mirrors the Windows contract.
+#[test]
+#[ignore]
+fn window_scope_rejects_windowless_click() {
+ let Some(mut driver) = McpDriver::spawn() else { return };
+ set_scope(&mut driver, "window");
+ let r = driver.call("click", serde_json::json!({ "x": 100, "y": 100, "session": SESSION }));
+ let txt = r.text().to_lowercase();
+ assert!(
+ r.is_error() || txt.contains("desktop scope") || txt.contains("desktop_scope_disabled"),
+ "window-scope window-less click was NOT rejected: {}",
+ r.text()
+ );
+ println!("✅ window_scope_rejects_windowless_click: window-less click correctly gated");
+}
diff --git a/libs/cua-driver/rust/crates/platform-linux/src/input/mod.rs b/libs/cua-driver/rust/crates/platform-linux/src/input/mod.rs
index a3550517a9..bf0b1909a6 100644
--- a/libs/cua-driver/rust/crates/platform-linux/src/input/mod.rs
+++ b/libs/cua-driver/rust/crates/platform-linux/src/input/mod.rs
@@ -1493,6 +1493,32 @@ pub fn send_type_text_xtest(text: &str) -> Result<()> {
Ok(())
}
+/// Screen-absolute click via the XTest extension — the `capture_scope="desktop"`
+/// foreground click. It warps the real pointer to `(x, y)` and injects a true
+/// button press/release there, so the event lands on whatever window owns that
+/// screen pixel (the Linux peer of the Windows `WindowFromPoint` + macOS
+/// global-HID `CGEvent` desktop click).
+///
+/// XTest delivering to the focused / under-pointer window is precisely why the
+/// *background* paths above use `XSendEvent` instead (see the module header) —
+/// but it is exactly what desktop scope wants: the agent has located the target
+/// by vision on the whole screen and issues a real screen-absolute pointer
+/// click. `button` is an X button number (1=left, 2=middle, 3=right).
+pub fn send_click_xtest_desktop(x: i32, y: i32, button: u8, count: usize) -> Result<()> {
+ use x11rb::protocol::xtest::ConnectionExt as _;
+ let (conn, screen_num) = connect_x11_for_input()?;
+ let root = conn.setup().roots[screen_num].root;
+ // Absolute pointer warp (MotionNotify, detail=0 => absolute) so the button
+ // events that follow are delivered at (x, y).
+ conn.xtest_fake_input(MOTION_NOTIFY_EVENT, 0, 0, root, x as i16, y as i16, 0)?;
+ for _ in 0..count.max(1) {
+ conn.xtest_fake_input(BUTTON_PRESS_EVENT, button, 0, root, x as i16, y as i16, 0)?;
+ conn.xtest_fake_input(BUTTON_RELEASE_EVENT, button, 0, root, x as i16, y as i16, 0)?;
+ }
+ conn.flush()?;
+ Ok(())
+}
+
/// Send a named key press to a window.
pub fn send_key(xid: u64, key: &str, modifiers: &[&str]) -> Result<()> {
let (conn, _) = connect_x11_for_input()?;
diff --git a/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs b/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs
index 60e058df5a..22da25cd29 100644
--- a/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs
+++ b/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs
@@ -1010,6 +1010,56 @@ impl Tool for ClickTool {
async fn invoke(&self, args: Value) -> ToolResult {
let cursor_id = resolve_cursor_key(&args);
+
+ // ── Window-less screen-absolute branch (capture_scope="desktop") ──────
+ // x,y with NO pid and NO window_id → TRUE SCREEN pixels. Foreground,
+ // vision-driven desktop-scope click (the Linux peer of the Windows
+ // WindowFromPoint / macOS global-HID path). Gate on the effective scope:
+ // under "window" return the same `desktop_scope_disabled` contract.
+ let has_pid = args.get("pid").map(|v| !v.is_null()).unwrap_or(false);
+ let has_window_id = args.get("window_id").map(|v| !v.is_null()).unwrap_or(false);
+ let has_xy = args.get("x").map(|v| v.is_number()).unwrap_or(false)
+ && args.get("y").map(|v| v.is_number()).unwrap_or(false);
+ if has_xy && !has_pid && !has_window_id {
+ // Linux config is global-only (no per-session override layer).
+ let scope = self.state.config.read().unwrap().capture_scope.clone();
+ if scope != "desktop" {
+ return ToolResult::error(
+ "click: x,y given with no pid/window_id, but capture_scope is \
+ \"window\". Screen-absolute clicks require desktop scope. Call \
+ set_config with capture_scope=desktop (and use get_desktop_state \
+ to read true screen pixels) first."
+ .to_string(),
+ )
+ .with_structured(json!({
+ "code": "desktop_scope_disabled",
+ "capture_scope": scope,
+ "suggestion": "set_config capture_scope=desktop",
+ }));
+ }
+ let button_raw = args.str_or("button", "left").to_lowercase();
+ if !matches!(button_raw.as_str(), "" | "left" | "right" | "middle") {
+ return ToolResult::error(format!(
+ "click: unknown button \"{button_raw}\" — expected one of left, right, middle."
+ ));
+ }
+ let button = parse_mouse_button(if button_raw.is_empty() { "left" } else { &button_raw });
+ let sx = args.f64_or("x", 0.0) as i32;
+ let sy = args.f64_or("y", 0.0) as i32;
+ let n = args.u64_or("count", 1) as usize;
+ let r = tokio::task::spawn_blocking(move || {
+ crate::input::send_click_xtest_desktop(sx, sy, button, n)
+ })
+ .await;
+ return match r {
+ Ok(Ok(())) => ToolResult::text(format!(
+ "✅ Sent screen-absolute click at ({sx},{sy}) (desktop scope)."
+ )),
+ Ok(Err(e)) => ToolResult::error(format!("desktop-scope click failed: {e}")),
+ Err(e) => ToolResult::error(format!("task error: {e}")),
+ };
+ }
+
let pid = match args.require_u32("pid") { Ok(v) => v, Err(e) => return e };
let count = args.u64_or("count", 1) as usize;
// Surface 5: reject unknown buttons so a typo can't silently fall through
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/input/mouse.rs b/libs/cua-driver/rust/crates/platform-macos/src/input/mouse.rs
index 09a5df892c..3d40261b43 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/input/mouse.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/input/mouse.rs
@@ -26,6 +26,55 @@ pub fn click_at_xy(pid: i32, x: f64, y: f64, count: usize, modifiers: &[&str]) -
click_at_xy_inner(pid, x, y, None, None, count, modifiers)
}
+/// Screen-absolute click posted to the GLOBAL HID tap (`CGEventTapLocation::HID`),
+/// NOT routed to any pid — the OS delivers it to whichever window owns the
+/// screen point, the macOS analogue of Windows' `WindowFromPoint` + `SendInput`
+/// desktop-scope click. This backs the `capture_scope="desktop"`, window-less
+/// (no pid/window_id) branch of the `click` tool: the agent has located the
+/// target by vision in `get_desktop_state` and clicks true screen pixels.
+///
+/// Unlike the pid-routed `click_at_xy`, this honors the real foreground/Z order
+/// (it lands on whatever is visually on top at the point) — exactly the
+/// foreground, vision-driven model that complements the background contract.
+pub fn click_at_xy_desktop(x: f64, y: f64, count: usize, button: &str) -> anyhow::Result<()> {
+ use core_graphics::display::CGDisplay;
+ use core_graphics::event::CGEventTapLocation;
+ let source = CGEventSource::new(CGEventSourceStateID::HIDSystemState)
+ .map_err(|_| anyhow::anyhow!("CGEventSource::new failed"))?;
+ let point = CGPoint::new(x, y);
+ let (down_ty, up_ty, btn) = match button {
+ "right" => (CGEventType::RightMouseDown, CGEventType::RightMouseUp, CGMouseButton::Right),
+ "middle" => (CGEventType::OtherMouseDown, CGEventType::OtherMouseUp, CGMouseButton::Center),
+ _ => (CGEventType::LeftMouseDown, CGEventType::LeftMouseUp, CGMouseButton::Left),
+ };
+ // Warp the REAL cursor to the point first (the macOS peer of Linux XTest's
+ // pointer warp). A synthetic MouseMoved event does not relocate the hardware
+ // cursor, and AppKit hit-tests some clicks against the actual cursor
+ // position, so without the warp the down/up can miss the target. Desktop
+ // scope is the foreground modality, so moving the visible cursor is expected.
+ let _ = CGDisplay::warp_mouse_cursor_position(point);
+ // Re-couple cursor + mouse-delta so the synthesized click hit-tests at the
+ // warped point, not the pre-warp one.
+ unsafe { CGAssociateMouseAndMouseCursorPosition(true) };
+ std::thread::sleep(std::time::Duration::from_millis(40));
+ for _ in 0..count.max(1) {
+ let down = CGEvent::new_mouse_event(source.clone(), down_ty, point, btn)
+ .map_err(|_| anyhow::anyhow!("CGEvent::new_mouse_event(down) failed"))?;
+ down.post(CGEventTapLocation::HID);
+ std::thread::sleep(std::time::Duration::from_millis(20));
+ let up = CGEvent::new_mouse_event(source.clone(), up_ty, point, btn)
+ .map_err(|_| anyhow::anyhow!("CGEvent::new_mouse_event(up) failed"))?;
+ up.post(CGEventTapLocation::HID);
+ }
+ Ok(())
+}
+
+extern "C" {
+ /// Reconnect the mouse-delta stream to the (just-warped) cursor position so a
+ /// synthesized click hit-tests at the new location, not the pre-warp one.
+ fn CGAssociateMouseAndMouseCursorPosition(connected: bool) -> i32;
+}
+
/// Like `click_at_xy` but also stamps window-local `(wx, wy)` onto the event.
/// When `wid` is provided, additionally stamps Chromium routing fields
/// (f51 / f58 / f91 / f92) for better backgrounded-target delivery.
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/click.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/click.rs
index 0423c06151..d2ed982992 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/click.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/click.rs
@@ -115,6 +115,111 @@ impl Tool for ClickTool {
async fn invoke(&self, args: Value) -> ToolResult {
use cua_driver_core::tool_args::ArgsExt;
+
+ // ── Window-less screen-absolute branch (capture_scope="desktop") ──────
+ // x,y given with NO pid and NO window_id → the coordinates are TRUE
+ // SCREEN pixels. This is the foreground, vision-driven desktop-scope
+ // path, the macOS peer of the Windows WindowFromPoint click. Gate on the
+ // effective scope: under "window" return a structured
+ // `desktop_scope_disabled` error (same contract as Windows) rather than
+ // silently treating window-local pixels as screen pixels.
+ let has_pid = args.get("pid").map(|v| !v.is_null()).unwrap_or(false);
+ let has_window_id = args.get("window_id").map(|v| !v.is_null()).unwrap_or(false);
+ let has_xy = args.get("x").map(|v| v.is_number()).unwrap_or(false)
+ && args.get("y").map(|v| v.is_number()).unwrap_or(false);
+ if has_xy && !has_pid && !has_window_id {
+ let session_id = args.opt_str("_session_id");
+ let scope = self
+ .state
+ .session_config
+ .effective_scope(session_id.as_deref(), &self.state.config.read().unwrap());
+ if scope != "desktop" {
+ return ToolResult::error(
+ "click: x,y given with no pid/window_id, but capture_scope is \
+ \"window\". Screen-absolute clicks require desktop scope. Call \
+ set_config with capture_scope=desktop (and use get_desktop_state \
+ to read true screen pixels) first."
+ .to_string(),
+ )
+ .with_structured(serde_json::json!({
+ "code": "desktop_scope_disabled",
+ "capture_scope": scope,
+ "suggestion": "set_config capture_scope=desktop",
+ }));
+ }
+ let sx = args.opt_f64("x").or_else(|| args.opt_i64("x").map(|i| i as f64)).unwrap_or(0.0);
+ let sy = args.opt_f64("y").or_else(|| args.opt_i64("y").map(|i| i as f64)).unwrap_or(0.0);
+ let button = match args.str_or("button", "left").to_lowercase().as_str() {
+ "right" => "right",
+ "middle" => "middle",
+ "" | "left" => "left",
+ other => {
+ return ToolResult::error(format!(
+ "click: unknown button \"{other}\" — expected one of left, right, middle."
+ ))
+ }
+ }
+ .to_string();
+ let count = args.u64_or("count", 1) as usize;
+ // Glide the session's agent cursor to the screen point for visibility.
+ let cursor_key = super::cursor_tools::resolve_cursor_key(&args);
+ crate::cursor::overlay::animate_cursor_to(cursor_key.clone(), sx, sy).await;
+ self.state.cursor_registry.update_position(&cursor_key, sx, sy);
+
+ // Resolve the frontmost on-screen window under the point (the macOS
+ // peer of Windows' WindowFromPoint). When found, click THAT pid via
+ // the proven SkyLight path (`click_at_xy`, screen coords) — reliable
+ // on AppKit/Chromium where a bare HID post can miss. Only when no
+ // app window owns the pixel (desktop background, etc.) fall back to
+ // the cursor-warp + HID post.
+ // Resolve as (pid, window_id, win_origin_x, win_origin_y) so the
+ // click can stamp the window-LOCAL point — AppKit hit-tests the
+ // stamped window-local coordinate, not the bare screen point, so a
+ // plain screen-coord post misses.
+ // Exclude our OWN windows (the agent-cursor overlay we just glided to
+ // the point sits on top of the target — never resolve the click to it).
+ let own_pid = std::process::id() as i32;
+ let target = {
+ let mut wins = crate::windows::visible_windows();
+ wins.sort_by_key(|w| w.z_index); // front-to-back
+ wins.into_iter().find(|w| {
+ w.layer == 0
+ && w.pid != own_pid
+ && sx >= w.bounds.x
+ && sx < w.bounds.x + w.bounds.width
+ && sy >= w.bounds.y
+ && sy < w.bounds.y + w.bounds.height
+ }).map(|w| (w.pid, w.window_id, w.bounds.x, w.bounds.y))
+ };
+ let btn = button.clone();
+ let result = tokio::task::spawn_blocking(move || -> anyhow::Result<Option<i32>> {
+ match target {
+ Some((pid, wid, ox, oy)) => {
+ let (wx, wy) = (sx - ox, sy - oy);
+ crate::input::mouse::click_at_xy_with_window_local(
+ pid, sx, sy, wx, wy, wid, count, &[],
+ )?;
+ Ok(Some(pid))
+ }
+ None => {
+ crate::input::mouse::click_at_xy_desktop(sx, sy, count, &btn)?;
+ Ok(None)
+ }
+ }
+ })
+ .await;
+ return match result {
+ Ok(Ok(Some(pid))) => ToolResult::text(format!(
+ "✅ Sent click at screen ({sx},{sy}) on pid {pid} (desktop scope)."
+ )),
+ Ok(Ok(None)) => ToolResult::text(format!(
+ "✅ Sent screen-absolute click at ({sx},{sy}) (desktop scope, no window under point)."
+ )),
+ Ok(Err(e)) => ToolResult::error(format!("desktop-scope click failed: {e}")),
+ Err(e) => ToolResult::error(format!("task error: {e}")),
+ };
+ }
+
let pid = match args.require_i32("pid") { Ok(v) => v, Err(e) => return e };
// Resolve this action's cursor key so its click-pulse / glide land on
// the calling session's cursor, not the shared "default" one.