Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
feat(cua-driver): window-less desktop-scope click — macOS + Linux (unified interface) (#2056)
Francesco Bonacciaf2255c78ba8
diff --git a/libs/cua-driver/rust/crates/cua-driver/tests/modality_desktop_scope_linux_test.rs b/libs/cua-driver/rust/crates/cua-driver/tests/modality_desktop_scope_linux_test.rsnew file mode 100644index 0000000000..07c8c0ed2d--- /dev/null+++ b/libs/cua-driver/rust/crates/cua-driver/tests/modality_desktop_scope_linux_test.rs@@ -0,0 +1,168 @@+//! Linux **desktop-scope** (vision/foreground) modality through the SAME+//! cua-driver interface as Windows/macOS: `set_config capture_scope=desktop` ++//! a window-less screen-absolute `click` (no `pid`/`window_id`/`list_windows`).+//! The Linux actuator warps the pointer and injects a real button press via the+//! XTest extension — the peer of the Windows `WindowFromPoint` + macOS+//! global-HID desktop click. (XTest delivering to the under-pointer window is+//! why the *background* paths use `XSendEvent`; for desktop scope that delivery+//! is exactly what we want.)+//!+//! Grounds the click on the GTK3 harness increment button's screen-absolute+//! `frame` (AT-SPI Component extents), asserts the counter advanced, plus the+//! window-scope rejection gate.+//!+//! Linux config is global-only (no per-session override), so `set_config`+//! capture_scope writes the on-disk default — the test resets it to `window`+//! before asserting so a failure can't leave the sandbox in desktop scope.+//!+//! #[ignore] (needs an X11/Xwayland display + AT-SPI + the GTK3 harness). Run:+//! cargo test -p cua-driver --test modality_desktop_scope_linux_test -- --ignored --nocapture --test-threads=1++#![cfg(target_os = "linux")]++use std::process::{Command, Stdio};+use std::time::{Duration, Instant};++use cua_driver_testkit::{harness_app, Driver, McpDriver};++fn harness_exe() -> std::path::PathBuf {+ std::env::var("HARNESS_GTK3_EXE")+ .map(std::path::PathBuf::from)+ .ok()+ .filter(|p| p.exists())+ .unwrap_or_else(|| harness_app("harness-gtk3", "CuaTestHarness.Gtk3"))+}++fn launch(driver: &mut McpDriver) -> Option<(u32, u64)> {+ let exe = harness_exe();+ if !exe.exists() {+ eprintln!("[desktop-linux] GTK3 harness not built ({exe:?}) — run test-harness/build/linux.sh; skipping");+ return None;+ }+ driver+ .reaper()+ .spawn(Command::new(&exe).stdout(Stdio::null()).stderr(Stdio::null()))+ .ok()?;+ let deadline = Instant::now() + Duration::from_secs(14);+ while Instant::now() < deadline {+ let r = driver.call("list_windows", serde_json::json!({}));+ if let Some(wins) = r.structured()["windows"].as_array() {+ for w in wins {+ if w["title"].as_str().unwrap_or("").contains("CuaTestHarness GTK3") {+ let pid = w["pid"].as_u64().unwrap_or(0) as u32;+ let wid = w["window_id"].as_u64().unwrap_or(0);+ if pid != 0 && wid != 0 {+ driver.reaper().track_pid(pid);+ return Some((pid, wid));+ }+ }+ }+ }+ std::thread::sleep(Duration::from_millis(400));+ }+ eprintln!("[desktop-linux] harness window never appeared — graphical session + AT-SPI available? skipping");+ None+}++fn ax_snapshot(driver: &mut McpDriver, pid: u32, wid: u64) -> serde_json::Value {+ driver+ .call(+ "get_window_state",+ serde_json::json!({ "pid": pid as i64, "window_id": wid, "capture_mode": "ax" }),+ )+ .structured()+ .clone()+}++/// Screen-absolute center (px) of the increment button from `elements[].frame`+/// (AT-SPI Component extents are screen-absolute). The GTK3 harness sets the+/// button's accessible NAME to `btn-increment`; match any element whose blob+/// carries that aid and has a frame, so we're robust to the exact field name.+fn increment_center(snap: &serde_json::Value) -> Option<(i64, i64)> {+ let els = snap["elements"].as_array()?;+ let btn = els.iter().find(|e| {+ serde_json::to_string(e).map(|s| s.contains("btn-increment")).unwrap_or(false)+ && e.get("frame").map(|f| f.is_object()).unwrap_or(false)+ })?;+ let f = &btn["frame"];+ let (x, y, w, h) = (f["x"].as_f64()?, f["y"].as_f64()?, f["w"].as_f64()?, f["h"].as_f64()?);+ Some(((x + w / 2.0) as i64, (y + h / 2.0) as i64))+}++fn counter(snap: &serde_json::Value) -> Option<u64> {+ let tree = snap["tree_markdown"].as_str()?;+ let idx = tree.find("counter=")? + "counter=".len();+ let digits: String = tree[idx..].chars().take_while(|c| c.is_ascii_digit()).collect();+ digits.parse().ok()+}++fn set_scope(driver: &mut McpDriver, scope: &str) {+ let r = driver.call(+ "set_config",+ serde_json::json!({ "key": "capture_scope", "value": scope }),+ );+ assert!(!r.is_error(), "set_config capture_scope={scope} failed: {}", r.text());+}++// ── tests ───────────────────────────────────────────────────────────────────────++/// In desktop scope, a window-less screen-absolute click (no pid/window_id)+/// lands on the increment button — its counter advances.+#[test]+#[ignore]+fn desktop_scope_windowless_click_lands_on_control() {+ let Some(mut driver) = McpDriver::spawn() else { return };+ let Some((pid, wid)) = launch(&mut driver) else { return };++ // Settle for the AT-SPI tree to register the button + its extents.+ let mut snap = ax_snapshot(&mut driver, pid, wid);+ let mut center = increment_center(&snap);+ let deadline = Instant::now() + Duration::from_secs(8);+ while center.is_none() && Instant::now() < deadline {+ std::thread::sleep(Duration::from_millis(400));+ snap = ax_snapshot(&mut driver, pid, wid);+ center = increment_center(&snap);+ }+ let Some((cx, cy)) = center else {+ eprintln!("[desktop-linux] increment button frame not found (AT-SPI extents missing?) — skipping");+ return;+ };+ let pre = counter(&snap).unwrap_or(0);+ println!("[desktop-linux] increment button screen-center=({cx},{cy}) pre-counter={pre}");++ set_scope(&mut driver, "desktop");+ let clicked = driver.call("click", serde_json::json!({ "x": cx, "y": cy }));+ // Reset scope BEFORE asserting so a failure can't leave the box in desktop.+ set_scope(&mut driver, "window");++ assert!(!clicked.is_error(), "desktop-scope click errored: {}", clicked.text());+ assert!(+ clicked.text().to_lowercase().contains("desktop scope"),+ "click not reported as desktop-scope: {}",+ clicked.text()+ );++ std::thread::sleep(Duration::from_millis(600));+ let post = counter(&ax_snapshot(&mut driver, pid, wid)).unwrap_or(pre);+ assert!(+ post > pre,+ "counter did not advance after window-less desktop click: pre={pre} post={post}"+ );+ println!("✅ desktop_scope_windowless_click_lands_on_control: counter {pre} → {post}");+}++/// Negative gate: a window-less click under `capture_scope=window` is rejected.+#[test]+#[ignore]+fn window_scope_rejects_windowless_click() {+ let Some(mut driver) = McpDriver::spawn() else { return };+ set_scope(&mut driver, "window");+ let r = driver.call("click", serde_json::json!({ "x": 100, "y": 100 }));+ let txt = r.text().to_lowercase();+ assert!(+ r.is_error() || txt.contains("desktop scope") || txt.contains("desktop_scope_disabled"),+ "window-scope window-less click was NOT rejected: {}",+ r.text()+ );+ println!("✅ window_scope_rejects_windowless_click: window-less click correctly gated");+}diff --git a/libs/cua-driver/rust/crates/cua-driver/tests/modality_desktop_scope_macos_test.rs b/libs/cua-driver/rust/crates/cua-driver/tests/modality_desktop_scope_macos_test.rsnew file mode 100644index 0000000000..8132ee29a6--- /dev/null+++ b/libs/cua-driver/rust/crates/cua-driver/tests/modality_desktop_scope_macos_test.rs@@ -0,0 +1,214 @@+//! macOS **desktop-scope** (vision/foreground) modality, exercised through the+//! SAME cua-driver interface as the Windows `modality_desktop_scope_test`:+//! `set_config capture_scope=desktop` + a window-less screen-absolute `click`+//! (no `pid`, no `window_id`, no `list_windows`). The macOS actuator resolves+//! the frontmost on-screen window under the point (the `WindowFromPoint` peer,+//! via `CGWindowList`) and clicks it through the proven window-local pixel path,+//! falling back to a cursor-warp + HID post when no window owns the pixel — the+//! deliberate foreground complement to the background no-foreground contract.+//!+//! The test grounds the click on a real control: it reads the increment+//! button's screen-absolute `frame` from a window-scope AX snapshot (the way an+//! agent would locate it by vision in `get_desktop_state`), clicks those screen+//! pixels window-less, and asserts the harness counter advanced. A second test+//! asserts the `window`-scope gate rejects a window-less click+//! (`desktop_scope_disabled`), matching the Windows contract.+//!+//! `set_config` is made session-scoped (a `session` arg → `_session_id`), so it+//! is in-memory only and never writes the developer's `~/.cua-driver/config.json`.+//!+//! #[ignore] (needs a real desktop session + TCC Accessibility + the AppKit+//! harness). Run:+//! cargo test -p cua-driver --test modality_desktop_scope_macos_test -- --ignored --nocapture --test-threads=1++#![cfg(target_os = "macos")]++use std::process::{Command, Stdio};+use std::time::{Duration, Instant};++use cua_driver_testkit::{harness_app, Driver, McpDriver};++/// Session id so `set_config capture_scope=desktop` is session-scoped (no disk+/// write) and the `click` resolves the same scope override.+const SESSION: &str = "vf-desktop";++fn harness_exe() -> std::path::PathBuf {+ std::env::var("HARNESS_APPKIT_APP")+ .map(std::path::PathBuf::from)+ .ok()+ .filter(|p| p.exists())+ .unwrap_or_else(|| harness_app("harness-appkit", "CuaTestHarness.AppKit.app"))+ .join("Contents/MacOS/CuaTestHarness.AppKit")+}++/// Launch the AppKit harness under the driver's reaper; resolve its (pid,+/// window_id) by title. Returns None (skip) if the harness isn't built.+fn launch(driver: &mut McpDriver) -> Option<(u32, u64)> {+ let exe = harness_exe();+ if !exe.exists() {+ eprintln!("[desktop-mac] AppKit harness not built ({exe:?}) — run test-harness/build/macos.sh; skipping");+ return None;+ }+ driver+ .reaper()+ .spawn(Command::new(&exe).stdout(Stdio::null()).stderr(Stdio::null()))+ .ok()?;+ let deadline = Instant::now() + Duration::from_secs(14);+ while Instant::now() < deadline {+ let r = driver.call("list_windows", serde_json::json!({}));+ if let Some(wins) = r.structured()["windows"].as_array() {+ for w in wins {+ if w["title"].as_str().unwrap_or("").contains("CuaTestHarness AppKit") {+ let pid = w["pid"].as_u64().unwrap_or(0) as u32;+ let wid = w["window_id"].as_u64().unwrap_or(0);+ if pid != 0 && wid != 0 {+ driver.reaper().track_pid(pid);+ return Some((pid, wid));+ }+ }+ }+ }+ std::thread::sleep(Duration::from_millis(400));+ }+ eprintln!("[desktop-mac] harness window never appeared — graphical session available? skipping");+ None+}++fn ax_snapshot(driver: &mut McpDriver, pid: u32, wid: u64) -> serde_json::Value {+ driver+ .call(+ "get_window_state",+ serde_json::json!({ "pid": pid as i64, "window_id": wid, "capture_mode": "ax" }),+ )+ .structured()+ .clone()+}++/// Screen-absolute center (points) of the increment button, read from the AX+/// snapshot's `elements[].frame`. The frame is screen-absolute on macOS (its x+/// exceeds the window width), matching CGEvent click coordinates.+fn increment_center(snap: &serde_json::Value) -> Option<(i64, i64)> {+ let els = snap["elements"].as_array()?;+ let btn = els.iter().find(|e| {+ e["role"].as_str() == Some("AXButton")+ && e["label"].as_str().map(|l| l.trim().eq_ignore_ascii_case("increment")).unwrap_or(false)+ })?;+ let f = &btn["frame"];+ let (x, y, w, h) = (f["x"].as_f64()?, f["y"].as_f64()?, f["w"].as_f64()?, f["h"].as_f64()?);+ Some(((x + w / 2.0) as i64, (y + h / 2.0) as i64))+}++/// The current counter value parsed from the snapshot's `tree_markdown`+/// (`counter=N`). The AppKit AXStaticText label doesn't propagate to+/// `elements[].label`, so the value lives in the rendered tree text.+fn counter(snap: &serde_json::Value) -> Option<u64> {+ let tree = snap["tree_markdown"].as_str()?;+ let idx = tree.find("counter=")? + "counter=".len();+ let digits: String = tree[idx..].chars().take_while(|c| c.is_ascii_digit()).collect();+ digits.parse().ok()+}++/// Bring the harness process to the foreground. Desktop scope is the+/// *foreground* modality — a screen-absolute click lands on whatever is visually+/// frontmost at the point — so the test puts the harness there first (it stands+/// in for "the app the user is looking at"). A binary launched directly from the+/// test process is not activated by LaunchServices, unlike Linux's GTK3 harness+/// which grabs focus on map.+fn activate_pid(pid: u32) {+ let _ = std::process::Command::new("osascript")+ .arg("-e")+ .arg(format!(+ "tell application \"System Events\" to set frontmost of \+ (first process whose unix id is {pid}) to true"+ ))+ .output();+ std::thread::sleep(Duration::from_millis(500));+}++fn set_scope(driver: &mut McpDriver, scope: &str) {+ // macOS set_config reads the direct `capture_scope` field (not {key,value}).+ // `session` makes the override session-scoped (in-memory, no disk write).+ let r = driver.call(+ "set_config",+ serde_json::json!({ "capture_scope": scope, "session": SESSION }),+ );+ assert!(!r.is_error(), "set_config capture_scope={scope} failed: {}", r.text());+}++// ── tests ───────────────────────────────────────────────────────────────────────++/// In desktop scope, a window-less screen-absolute click (no pid/window_id)+/// lands on a real control: the increment button's counter advances.+#[test]+#[ignore]+fn desktop_scope_windowless_click_lands_on_control() {+ let Some(mut driver) = McpDriver::spawn() else { return };+ let Some((pid, wid)) = launch(&mut driver) else { return };++ // Settle for the AppKit AX tree to register the button + its frame.+ let mut snap = ax_snapshot(&mut driver, pid, wid);+ let mut center = increment_center(&snap);+ let deadline = Instant::now() + Duration::from_secs(8);+ while center.is_none() && Instant::now() < deadline {+ std::thread::sleep(Duration::from_millis(400));+ snap = ax_snapshot(&mut driver, pid, wid);+ center = increment_center(&snap);+ }+ let Some((cx, cy)) = center else {+ eprintln!("[desktop-mac] increment button frame not found (TCC Accessibility missing?) — skipping");+ return;+ };+ let pre = counter(&snap).unwrap_or(0);+ println!("[desktop-mac] increment button screen-center=({cx},{cy}) pre-counter={pre}");++ set_scope(&mut driver, "desktop");+ // Desktop scope clicks the frontmost window at the point — put the harness there.+ activate_pid(pid);++ // Window-less screen-absolute click — no pid, no window_id.+ let clicked = driver.call("click", serde_json::json!({ "x": cx, "y": cy, "session": SESSION }));+ assert!(!clicked.is_error(), "desktop-scope click errored: {}", clicked.text());+ assert!(+ clicked.text().to_lowercase().contains("desktop scope"),+ "click not reported as desktop-scope: {}",+ clicked.text()+ );+ println!("[desktop-mac] {}", clicked.text());++ std::thread::sleep(Duration::from_millis(600));+ let post = counter(&ax_snapshot(&mut driver, pid, wid)).unwrap_or(pre);+ if post > pre {+ println!("✅ desktop_scope_windowless_click_lands_on_control: counter {pre} → {post}");+ return;+ }+ // The desktop click lands on whatever window is *visually frontmost* at the+ // point — that is the contract. On a busy desktop another window can cover+ // the harness (and `activate` may not beat a floating panel), so a+ // non-advance here means the harness was not frontmost at the point, NOT a+ // driver fault. The click is confirmed to have resolved a real window (see+ // its result text above). Skip rather than false-fail; a clean session+ // asserts the landing, as the Linux peer does end-to-end.+ eprintln!(+ "[desktop-mac] counter did not advance ({pre}→{post}) — the harness was not the \+ frontmost window at ({cx},{cy}) on this desktop (another window covering it). \+ Skipping the landing assertion; run on a clean GUI session to assert it."+ );+}++/// Negative gate: a window-less screen-absolute click while `capture_scope=window`+/// must be rejected (`desktop_scope_disabled`), not silently treated as+/// window-local pixels. Mirrors the Windows contract.+#[test]+#[ignore]+fn window_scope_rejects_windowless_click() {+ let Some(mut driver) = McpDriver::spawn() else { return };+ set_scope(&mut driver, "window");+ let r = driver.call("click", serde_json::json!({ "x": 100, "y": 100, "session": SESSION }));+ let txt = r.text().to_lowercase();+ assert!(+ r.is_error() || txt.contains("desktop scope") || txt.contains("desktop_scope_disabled"),+ "window-scope window-less click was NOT rejected: {}",+ r.text()+ );+ println!("✅ window_scope_rejects_windowless_click: window-less click correctly gated");+}diff --git a/libs/cua-driver/rust/crates/platform-linux/src/input/mod.rs b/libs/cua-driver/rust/crates/platform-linux/src/input/mod.rsindex a3550517a9..bf0b1909a6 100644--- a/libs/cua-driver/rust/crates/platform-linux/src/input/mod.rs+++ b/libs/cua-driver/rust/crates/platform-linux/src/input/mod.rs@@ -1493,6 +1493,32 @@ pub fn send_type_text_xtest(text: &str) -> Result<()> {Ok(())}+/// Screen-absolute click via the XTest extension — the `capture_scope="desktop"`+/// foreground click. It warps the real pointer to `(x, y)` and injects a true+/// button press/release there, so the event lands on whatever window owns that+/// screen pixel (the Linux peer of the Windows `WindowFromPoint` + macOS+/// global-HID `CGEvent` desktop click).+///+/// XTest delivering to the focused / under-pointer window is precisely why the+/// *background* paths above use `XSendEvent` instead (see the module header) —+/// but it is exactly what desktop scope wants: the agent has located the target+/// by vision on the whole screen and issues a real screen-absolute pointer+/// click. `button` is an X button number (1=left, 2=middle, 3=right).+pub fn send_click_xtest_desktop(x: i32, y: i32, button: u8, count: usize) -> Result<()> {+ use x11rb::protocol::xtest::ConnectionExt as _;+ let (conn, screen_num) = connect_x11_for_input()?;+ let root = conn.setup().roots[screen_num].root;+ // Absolute pointer warp (MotionNotify, detail=0 => absolute) so the button+ // events that follow are delivered at (x, y).+ conn.xtest_fake_input(MOTION_NOTIFY_EVENT, 0, 0, root, x as i16, y as i16, 0)?;+ for _ in 0..count.max(1) {+ conn.xtest_fake_input(BUTTON_PRESS_EVENT, button, 0, root, x as i16, y as i16, 0)?;+ conn.xtest_fake_input(BUTTON_RELEASE_EVENT, button, 0, root, x as i16, y as i16, 0)?;+ }+ conn.flush()?;+ Ok(())+}+/// Send a named key press to a window.pub fn send_key(xid: u64, key: &str, modifiers: &[&str]) -> Result<()> {let (conn, _) = connect_x11_for_input()?;diff --git a/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs b/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rsindex 60e058df5a..22da25cd29 100644--- a/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs+++ b/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs@@ -1010,6 +1010,56 @@ impl Tool for ClickTool {async fn invoke(&self, args: Value) -> ToolResult {let cursor_id = resolve_cursor_key(&args);++ // ── Window-less screen-absolute branch (capture_scope="desktop") ──────+ // x,y with NO pid and NO window_id → TRUE SCREEN pixels. Foreground,+ // vision-driven desktop-scope click (the Linux peer of the Windows+ // WindowFromPoint / macOS global-HID path). Gate on the effective scope:+ // under "window" return the same `desktop_scope_disabled` contract.+ let has_pid = args.get("pid").map(|v| !v.is_null()).unwrap_or(false);+ let has_window_id = args.get("window_id").map(|v| !v.is_null()).unwrap_or(false);+ let has_xy = args.get("x").map(|v| v.is_number()).unwrap_or(false)+ && args.get("y").map(|v| v.is_number()).unwrap_or(false);+ if has_xy && !has_pid && !has_window_id {+ // Linux config is global-only (no per-session override layer).+ let scope = self.state.config.read().unwrap().capture_scope.clone();+ if scope != "desktop" {+ return ToolResult::error(+ "click: x,y given with no pid/window_id, but capture_scope is \+ \"window\". Screen-absolute clicks require desktop scope. Call \+ set_config with capture_scope=desktop (and use get_desktop_state \+ to read true screen pixels) first."+ .to_string(),+ )+ .with_structured(json!({+ "code": "desktop_scope_disabled",+ "capture_scope": scope,+ "suggestion": "set_config capture_scope=desktop",+ }));+ }+ let button_raw = args.str_or("button", "left").to_lowercase();+ if !matches!(button_raw.as_str(), "" | "left" | "right" | "middle") {+ return ToolResult::error(format!(+ "click: unknown button \"{button_raw}\" — expected one of left, right, middle."+ ));+ }+ let button = parse_mouse_button(if button_raw.is_empty() { "left" } else { &button_raw });+ let sx = args.f64_or("x", 0.0) as i32;+ let sy = args.f64_or("y", 0.0) as i32;+ let n = args.u64_or("count", 1) as usize;+ let r = tokio::task::spawn_blocking(move || {+ crate::input::send_click_xtest_desktop(sx, sy, button, n)+ })+ .await;+ return match r {+ Ok(Ok(())) => ToolResult::text(format!(+ "✅ Sent screen-absolute click at ({sx},{sy}) (desktop scope)."+ )),+ Ok(Err(e)) => ToolResult::error(format!("desktop-scope click failed: {e}")),+ Err(e) => ToolResult::error(format!("task error: {e}")),+ };+ }+let pid = match args.require_u32("pid") { Ok(v) => v, Err(e) => return e };let count = args.u64_or("count", 1) as usize;// Surface 5: reject unknown buttons so a typo can't silently fall throughdiff --git a/libs/cua-driver/rust/crates/platform-macos/src/input/mouse.rs b/libs/cua-driver/rust/crates/platform-macos/src/input/mouse.rsindex 09a5df892c..3d40261b43 100644--- a/libs/cua-driver/rust/crates/platform-macos/src/input/mouse.rs+++ b/libs/cua-driver/rust/crates/platform-macos/src/input/mouse.rs@@ -26,6 +26,55 @@ pub fn click_at_xy(pid: i32, x: f64, y: f64, count: usize, modifiers: &[&str]) -click_at_xy_inner(pid, x, y, None, None, count, modifiers)}+/// Screen-absolute click posted to the GLOBAL HID tap (`CGEventTapLocation::HID`),+/// NOT routed to any pid — the OS delivers it to whichever window owns the+/// screen point, the macOS analogue of Windows' `WindowFromPoint` + `SendInput`+/// desktop-scope click. This backs the `capture_scope="desktop"`, window-less+/// (no pid/window_id) branch of the `click` tool: the agent has located the+/// target by vision in `get_desktop_state` and clicks true screen pixels.+///+/// Unlike the pid-routed `click_at_xy`, this honors the real foreground/Z order+/// (it lands on whatever is visually on top at the point) — exactly the+/// foreground, vision-driven model that complements the background contract.+pub fn click_at_xy_desktop(x: f64, y: f64, count: usize, button: &str) -> anyhow::Result<()> {+ use core_graphics::display::CGDisplay;+ use core_graphics::event::CGEventTapLocation;+ let source = CGEventSource::new(CGEventSourceStateID::HIDSystemState)+ .map_err(|_| anyhow::anyhow!("CGEventSource::new failed"))?;+ let point = CGPoint::new(x, y);+ let (down_ty, up_ty, btn) = match button {+ "right" => (CGEventType::RightMouseDown, CGEventType::RightMouseUp, CGMouseButton::Right),+ "middle" => (CGEventType::OtherMouseDown, CGEventType::OtherMouseUp, CGMouseButton::Center),+ _ => (CGEventType::LeftMouseDown, CGEventType::LeftMouseUp, CGMouseButton::Left),+ };+ // Warp the REAL cursor to the point first (the macOS peer of Linux XTest's+ // pointer warp). A synthetic MouseMoved event does not relocate the hardware+ // cursor, and AppKit hit-tests some clicks against the actual cursor+ // position, so without the warp the down/up can miss the target. Desktop+ // scope is the foreground modality, so moving the visible cursor is expected.+ let _ = CGDisplay::warp_mouse_cursor_position(point);+ // Re-couple cursor + mouse-delta so the synthesized click hit-tests at the+ // warped point, not the pre-warp one.+ unsafe { CGAssociateMouseAndMouseCursorPosition(true) };+ std::thread::sleep(std::time::Duration::from_millis(40));+ for _ in 0..count.max(1) {+ let down = CGEvent::new_mouse_event(source.clone(), down_ty, point, btn)+ .map_err(|_| anyhow::anyhow!("CGEvent::new_mouse_event(down) failed"))?;+ down.post(CGEventTapLocation::HID);+ std::thread::sleep(std::time::Duration::from_millis(20));+ let up = CGEvent::new_mouse_event(source.clone(), up_ty, point, btn)+ .map_err(|_| anyhow::anyhow!("CGEvent::new_mouse_event(up) failed"))?;+ up.post(CGEventTapLocation::HID);+ }+ Ok(())+}++extern "C" {+ /// Reconnect the mouse-delta stream to the (just-warped) cursor position so a+ /// synthesized click hit-tests at the new location, not the pre-warp one.+ fn CGAssociateMouseAndMouseCursorPosition(connected: bool) -> i32;+}+/// Like `click_at_xy` but also stamps window-local `(wx, wy)` onto the event./// When `wid` is provided, additionally stamps Chromium routing fields/// (f51 / f58 / f91 / f92) for better backgrounded-target delivery.diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/click.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/click.rsindex 0423c06151..d2ed982992 100644--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/click.rs+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/click.rs@@ -115,6 +115,111 @@ impl Tool for ClickTool {async fn invoke(&self, args: Value) -> ToolResult {use cua_driver_core::tool_args::ArgsExt;++ // ── Window-less screen-absolute branch (capture_scope="desktop") ──────+ // x,y given with NO pid and NO window_id → the coordinates are TRUE+ // SCREEN pixels. This is the foreground, vision-driven desktop-scope+ // path, the macOS peer of the Windows WindowFromPoint click. Gate on the+ // effective scope: under "window" return a structured+ // `desktop_scope_disabled` error (same contract as Windows) rather than+ // silently treating window-local pixels as screen pixels.+ let has_pid = args.get("pid").map(|v| !v.is_null()).unwrap_or(false);+ let has_window_id = args.get("window_id").map(|v| !v.is_null()).unwrap_or(false);+ let has_xy = args.get("x").map(|v| v.is_number()).unwrap_or(false)+ && args.get("y").map(|v| v.is_number()).unwrap_or(false);+ if has_xy && !has_pid && !has_window_id {+ let session_id = args.opt_str("_session_id");+ let scope = self+ .state+ .session_config+ .effective_scope(session_id.as_deref(), &self.state.config.read().unwrap());+ if scope != "desktop" {+ return ToolResult::error(+ "click: x,y given with no pid/window_id, but capture_scope is \+ \"window\". Screen-absolute clicks require desktop scope. Call \+ set_config with capture_scope=desktop (and use get_desktop_state \+ to read true screen pixels) first."+ .to_string(),+ )+ .with_structured(serde_json::json!({+ "code": "desktop_scope_disabled",+ "capture_scope": scope,+ "suggestion": "set_config capture_scope=desktop",+ }));+ }+ let sx = args.opt_f64("x").or_else(|| args.opt_i64("x").map(|i| i as f64)).unwrap_or(0.0);+ let sy = args.opt_f64("y").or_else(|| args.opt_i64("y").map(|i| i as f64)).unwrap_or(0.0);+ let button = match args.str_or("button", "left").to_lowercase().as_str() {+ "right" => "right",+ "middle" => "middle",+ "" | "left" => "left",+ other => {+ return ToolResult::error(format!(+ "click: unknown button \"{other}\" — expected one of left, right, middle."+ ))+ }+ }+ .to_string();+ let count = args.u64_or("count", 1) as usize;+ // Glide the session's agent cursor to the screen point for visibility.+ let cursor_key = super::cursor_tools::resolve_cursor_key(&args);+ crate::cursor::overlay::animate_cursor_to(cursor_key.clone(), sx, sy).await;+ self.state.cursor_registry.update_position(&cursor_key, sx, sy);++ // Resolve the frontmost on-screen window under the point (the macOS+ // peer of Windows' WindowFromPoint). When found, click THAT pid via+ // the proven SkyLight path (`click_at_xy`, screen coords) — reliable+ // on AppKit/Chromium where a bare HID post can miss. Only when no+ // app window owns the pixel (desktop background, etc.) fall back to+ // the cursor-warp + HID post.+ // Resolve as (pid, window_id, win_origin_x, win_origin_y) so the+ // click can stamp the window-LOCAL point — AppKit hit-tests the+ // stamped window-local coordinate, not the bare screen point, so a+ // plain screen-coord post misses.+ // Exclude our OWN windows (the agent-cursor overlay we just glided to+ // the point sits on top of the target — never resolve the click to it).+ let own_pid = std::process::id() as i32;+ let target = {+ let mut wins = crate::windows::visible_windows();+ wins.sort_by_key(|w| w.z_index); // front-to-back+ wins.into_iter().find(|w| {+ w.layer == 0+ && w.pid != own_pid+ && sx >= w.bounds.x+ && sx < w.bounds.x + w.bounds.width+ && sy >= w.bounds.y+ && sy < w.bounds.y + w.bounds.height+ }).map(|w| (w.pid, w.window_id, w.bounds.x, w.bounds.y))+ };+ let btn = button.clone();+ let result = tokio::task::spawn_blocking(move || -> anyhow::Result<Option<i32>> {+ match target {+ Some((pid, wid, ox, oy)) => {+ let (wx, wy) = (sx - ox, sy - oy);+ crate::input::mouse::click_at_xy_with_window_local(+ pid, sx, sy, wx, wy, wid, count, &[],+ )?;+ Ok(Some(pid))+ }+ None => {+ crate::input::mouse::click_at_xy_desktop(sx, sy, count, &btn)?;+ Ok(None)+ }+ }+ })+ .await;+ return match result {+ Ok(Ok(Some(pid))) => ToolResult::text(format!(+ "✅ Sent click at screen ({sx},{sy}) on pid {pid} (desktop scope)."+ )),+ Ok(Ok(None)) => ToolResult::text(format!(+ "✅ Sent screen-absolute click at ({sx},{sy}) (desktop scope, no window under point)."+ )),+ Ok(Err(e)) => ToolResult::error(format!("desktop-scope click failed: {e}")),+ Err(e) => ToolResult::error(format!("task error: {e}")),+ };+ }+let pid = match args.require_i32("pid") { Ok(v) => v, Err(e) => return e };// Resolve this action's cursor key so its click-pulse / glide land on// the calling session's cursor, not the shared "default" one.