Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
fix(cua-driver): harden macOS input and capture delivery
Francesco Bonacciec908f74cb9f
diff --git a/libs/cua-driver/rust/crates/cua-driver/tests/harness_swiftui_test.rs b/libs/cua-driver/rust/crates/cua-driver/tests/harness_swiftui_test.rsindex 42fa67a1e1..27435a4f47 100644--- a/libs/cua-driver/rust/crates/cua-driver/tests/harness_swiftui_test.rs+++ b/libs/cua-driver/rust/crates/cua-driver/tests/harness_swiftui_test.rs@@ -26,7 +26,9 @@ use cua_driver_testkit::{harness_app, Driver, McpDriver, ToolResponse};fn harness_exe() -> PathBuf {if let Ok(p) = std::env::var("HARNESS_SWIFTUI_APP") {let pb = PathBuf::from(p).join("Contents/MacOS/CuaTestHarness.SwiftUI");- if pb.exists() { return pb; }+ if pb.exists() {+ return pb;+ }}harness_app("harness-swiftui",@@ -34,7 +36,10 @@ fn harness_exe() -> PathBuf {)}-struct Harness { _app: Child, pid: u32 }+struct Harness {+ _app: Child,+ pid: u32,+}impl Harness {fn launch() -> Option<Self> {@@ -44,8 +49,10 @@ impl Harness {return None;}let app = Command::new(&exe)- .stdout(Stdio::null()).stderr(Stdio::null())- .spawn().ok()?;+ .stdout(Stdio::null())+ .stderr(Stdio::null())+ .spawn()+ .ok()?;let pid = app.id();std::thread::sleep(Duration::from_millis(900));Some(Self { _app: app, pid })@@ -76,11 +83,16 @@ fn snapshot_elements(driver: &mut McpDriver, pid: u32, window_id: u64) -> ToolRefn harness_swiftui_smoke() {let harness = match Harness::launch() {Some(h) => h,- None => { eprintln!("harness not built — skipping"); return; }+ None => {+ eprintln!("harness not built — skipping");+ return;+ }};println!("harness pid={}", harness.pid);- let Some(mut driver) = McpDriver::spawn() else { return };+ let Some(mut driver) = McpDriver::spawn_macos_daemon_proxy() else {+ return;+ };let (wid, title) = driver.find_window(harness.pid as i64, "CuaTestHarness SwiftUI")@@ -88,11 +100,11 @@ fn harness_swiftui_smoke() {println!("main window: id={wid} title={title:?}");let snap = snapshot_elements(&mut driver, harness.pid, wid);- if looks_empty(snap.text()) {+ if looks_empty(snap.tree_text()) {eprintln!("AX empty — TCC not granted; skipping element-assertions");return;}- let text = snap.text();+ let text = snap.tree_text();println!("snapshot:\n{text}");// SwiftUI Text views render as AXStaticText leaves and don't propagate@@ -100,17 +112,22 @@ fn harness_swiftui_smoke() {// quirk as AppKit's NSTextField label mode + WPF's TextBlock). Assert// on text content for labels, AX-id only for actionable controls.for aid in [- "btn-increment", "btn-reset",+ "btn-increment",+ "btn-reset","txt-input","btn-open-popover","btn-exit",] {- assert!(has_id(snap.text(), aid),- "missing AX identifier {aid} in SwiftUI snapshot");+ assert!(+ has_id(snap.tree_text(), aid),+ "missing AX identifier {aid} in SwiftUI snapshot"+ );}- assert!(text.contains("HARNESS_TEXT_MARKER_v1"),- "text_body marker not in SwiftUI snapshot");+ assert!(+ text.contains("HARNESS_TEXT_MARKER_v1"),+ "text_body marker not in SwiftUI snapshot"+ );}/// popover: click the popover trigger, verify the popover body text appears@@ -120,29 +137,38 @@ fn harness_swiftui_smoke() {fn harness_swiftui_popover() {let harness = match Harness::launch() {Some(h) => h,- None => { eprintln!("harness not built — skipping"); return; }+ None => {+ eprintln!("harness not built — skipping");+ return;+ }};- let Some(mut driver) = McpDriver::spawn() else { return };+ let Some(mut driver) = McpDriver::spawn_macos_daemon_proxy() else {+ return;+ };let (wid, _) = driver.find_window(harness.pid as i64, "CuaTestHarness SwiftUI").expect("main window not found");let snap_pre = snapshot_elements(&mut driver, harness.pid, wid);- if looks_empty(snap_pre.text()) {- eprintln!("AX empty — TCC not granted; skipping"); return;+ if looks_empty(snap_pre.tree_text()) {+ eprintln!("AX empty — TCC not granted; skipping");+ return;}// Verify popover body is NOT yet in the tree.- let pre_text = snap_pre.text().to_owned();- assert!(!pre_text.contains("POPOVER_MARKER_v1"),- "popover body unexpectedly present BEFORE open");-- let trigger_idx: u64 = if let Some(i) = element_index_by_id(snap_pre.text(), "btn-open-popover") {- i- } else {- eprintln!("popover trigger not found, skipping");- return;- };+ let pre_text = snap_pre.tree_text().to_owned();+ assert!(+ !pre_text.contains("POPOVER_MARKER_v1"),+ "popover body unexpectedly present BEFORE open"+ );++ let trigger_idx: u64 =+ if let Some(i) = element_index_by_id(snap_pre.tree_text(), "btn-open-popover") {+ i+ } else {+ eprintln!("popover trigger not found, skipping");+ return;+ };let click = driver.call("click",serde_json::json!({@@ -159,18 +185,23 @@ fn harness_swiftui_popover() {// Popovers may live in a separate AXWindow on macOS — try the main// window first, then list_windows for additional candidates.let snap_post = snapshot_elements(&mut driver, harness.pid, wid);- let mut found_marker = snap_post.text().contains("POPOVER_MARKER_v1");+ let mut found_marker = snap_post.tree_text().contains("POPOVER_MARKER_v1");if !found_marker {// Walk any new windows for the same pid.- let resp = driver.call("list_windows",- serde_json::json!({ "pid": harness.pid as i64 }));+ let resp = driver.call(+ "list_windows",+ serde_json::json!({ "pid": harness.pid as i64 }),+ );if let Some(wins) = resp.structured()["windows"].as_array() {for w in wins {if let Some(other_wid) = w["window_id"].as_u64() {- if other_wid == wid { continue; }+ if other_wid == wid {+ continue;+ }let s = snapshot_elements(&mut driver, harness.pid, other_wid);- if s.text().contains("POPOVER_MARKER_v1") {- found_marker = true; break;+ if s.tree_text().contains("POPOVER_MARKER_v1") {+ found_marker = true;+ break;}}}diff --git a/libs/cua-driver/rust/crates/cua-driver/tests/modality_desktop_scope_macos_test.rs b/libs/cua-driver/rust/crates/cua-driver/tests/modality_desktop_scope_macos_test.rsindex 39db3d248e..d1cef67d7c 100644--- a/libs/cua-driver/rust/crates/cua-driver/tests/modality_desktop_scope_macos_test.rs+++ b/libs/cua-driver/rust/crates/cua-driver/tests/modality_desktop_scope_macos_test.rs@@ -51,14 +51,22 @@ fn launch(driver: &mut McpDriver) -> Option<(u32, u64)> {}driver.reaper()- .spawn(Command::new(&exe).stdout(Stdio::null()).stderr(Stdio::null()))+ .spawn(+ Command::new(&exe)+ .stdout(Stdio::null())+ .stderr(Stdio::null()),+ ).ok()?;let deadline = Instant::now() + Duration::from_secs(14);while Instant::now() < deadline {let r = driver.call("list_windows", serde_json::json!({}));if let Some(wins) = r.structured()["windows"].as_array() {for w in wins {- if w["title"].as_str().unwrap_or("").contains("CuaTestHarness AppKit") {+ if w["title"]+ .as_str()+ .unwrap_or("")+ .contains("CuaTestHarness AppKit")+ {let pid = w["pid"].as_u64().unwrap_or(0) as u32;let wid = w["window_id"].as_u64().unwrap_or(0);if pid != 0 && wid != 0 {@@ -70,7 +78,9 @@ fn launch(driver: &mut McpDriver) -> Option<(u32, u64)> {}std::thread::sleep(Duration::from_millis(400));}- eprintln!("[desktop-mac] harness window never appeared — graphical session available? skipping");+ eprintln!(+ "[desktop-mac] harness window never appeared — graphical session available? skipping"+ );None}@@ -91,10 +101,18 @@ fn increment_center(snap: &serde_json::Value) -> Option<(i64, i64)> {let els = snap["elements"].as_array()?;let btn = els.iter().find(|e| {e["role"].as_str() == Some("AXButton")- && e["label"].as_str().map(|l| l.trim().eq_ignore_ascii_case("increment")).unwrap_or(false)+ && e["label"]+ .as_str()+ .map(|l| l.trim().eq_ignore_ascii_case("increment"))+ .unwrap_or(false)})?;let f = &btn["frame"];- let (x, y, w, h) = (f["x"].as_f64()?, f["y"].as_f64()?, f["w"].as_f64()?, f["h"].as_f64()?);+ let (x, y, w, h) = (+ f["x"].as_f64()?,+ f["y"].as_f64()?,+ f["w"].as_f64()?,+ f["h"].as_f64()?,+ );Some(((x + w / 2.0) as i64, (y + h / 2.0) as i64))}@@ -104,7 +122,10 @@ fn increment_center(snap: &serde_json::Value) -> Option<(i64, i64)> {fn counter(snap: &serde_json::Value) -> Option<u64> {let tree = snap["tree_markdown"].as_str()?;let idx = tree.find("counter=")? + "counter=".len();- let digits: String = tree[idx..].chars().take_while(|c| c.is_ascii_digit()).collect();+ let digits: String = tree[idx..]+ .chars()+ .take_while(|c| c.is_ascii_digit())+ .collect();digits.parse().ok()}@@ -135,8 +156,12 @@ fn activate_pid(pid: u32) {#[test]#[ignore]fn desktop_scope_windowless_click_lands_on_control() {- let Some(mut driver) = McpDriver::spawn() else { return };- let Some((pid, wid)) = launch(&mut driver) else { return };+ let Some(mut driver) = McpDriver::spawn_macos_daemon_proxy() else {+ return;+ };+ let Some((pid, wid)) = launch(&mut driver) else {+ return;+ };// Settle for the AppKit AX tree to register the button + its frame.let mut snap = ax_snapshot(&mut driver, pid, wid);@@ -158,8 +183,15 @@ fn desktop_scope_windowless_click_lands_on_control() {activate_pid(pid);// Window-less screen-absolute click — no pid, no window_id; scope per-call.- let clicked = driver.call("click", serde_json::json!({ "x": cx, "y": cy, "scope": "desktop", "session": SESSION }));- assert!(!clicked.is_error(), "desktop-scope click errored: {}", clicked.text());+ let clicked = driver.call(+ "click",+ serde_json::json!({ "x": cx, "y": cy, "scope": "desktop", "session": SESSION }),+ );+ assert!(+ !clicked.is_error(),+ "desktop-scope click errored: {}",+ clicked.text()+ );assert!(clicked.text().to_lowercase().contains("desktop scope"),"click not reported as desktop-scope: {}",@@ -193,9 +225,14 @@ fn desktop_scope_windowless_click_lands_on_control() {#[test]#[ignore]fn window_scope_rejects_windowless_click() {- let Some(mut driver) = McpDriver::spawn() else { return };+ let Some(mut driver) = McpDriver::spawn_macos_daemon_proxy() else {+ return;+ };// Default scope is "window" — a window-less click must be rejected.- let r = driver.call("click", serde_json::json!({ "x": 100, "y": 100, "scope": "window", "session": SESSION }));+ let r = driver.call(+ "click",+ serde_json::json!({ "x": 100, "y": 100, "scope": "window", "session": SESSION }),+ );let txt = r.text().to_lowercase();assert!(r.is_error() || txt.contains("desktop scope") || txt.contains("desktop_scope_disabled"),diff --git a/libs/cua-driver/rust/crates/cua-driver/tests/protocol_media_test.rs b/libs/cua-driver/rust/crates/cua-driver/tests/protocol_media_test.rsindex e79ced7c48..ec8cc34288 100644--- a/libs/cua-driver/rust/crates/cua-driver/tests/protocol_media_test.rs+++ b/libs/cua-driver/rust/crates/cua-driver/tests/protocol_media_test.rs@@ -14,7 +14,9 @@ use cua_driver_testkit::RawDriver;#[test]#[cfg(target_os = "windows")]fn screenshot() {- let Some(mut d) = RawDriver::spawn() else { return; };+ let Some(mut d) = RawDriver::spawn() else {+ return;+ };d.send(&serde_json::json!({"jsonrpc":"2.0","id":1,"method":"initialize","params":{}}));d.recv();@@ -35,14 +37,21 @@ fn screenshot() {"params":{"name":"screenshot","arguments":{"format":"jpeg","quality":70}}}));let resp = d.recv();- assert!(resp["error"].is_null(), "Protocol error from screenshot jpeg: {resp:?}");+ assert!(+ resp["error"].is_null(),+ "Protocol error from screenshot jpeg: {resp:?}"+ );if !resp["result"]["isError"].as_bool().unwrap_or(false) {let content = resp["result"]["content"].as_array().expect("content array");let has_jpeg = content.iter().any(|c| {- c["type"] == "image" && c["mimeType"].as_str().unwrap_or("") == "image/jpeg"+ c["type"] == "image"+ && c["mimeType"].as_str().unwrap_or("") == "image/jpeg"&& c["data"].as_str().map(|s| s.len() > 10).unwrap_or(false)});- assert!(has_jpeg, "Expected image/jpeg in screenshot response: {content:?}");+ assert!(+ has_jpeg,+ "Expected image/jpeg in screenshot response: {content:?}"+ );let sc = &resp["result"]["structuredContent"];assert!(sc["width"].as_f64().unwrap_or(0.0) > 0.0);assert!(sc["height"].as_f64().unwrap_or(0.0) > 0.0);@@ -54,7 +63,9 @@ fn screenshot() {#[cfg(target_os = "macos")]fn screenshot_no_window_id() {//! Call screenshot without window_id — should capture the full display and return image content.- let Some(mut d) = RawDriver::spawn() else { return; };+ let Some(mut d) = RawDriver::spawn() else {+ return;+ };d.send(&serde_json::json!({"jsonrpc":"2.0","id":1,"method":"initialize","params":{}}));d.recv();@@ -64,16 +75,24 @@ fn screenshot_no_window_id() {"params":{"name":"screenshot","arguments":{}}}));let resp = d.recv();- assert!(resp["error"].is_null(), "Protocol error from screenshot: {resp:?}");+ assert!(+ resp["error"].is_null(),+ "Protocol error from screenshot: {resp:?}"+ );let content = resp["result"]["content"].as_array().expect("content array");- assert!(!content.is_empty(), "screenshot should return at least one content item");+ assert!(+ !content.is_empty(),+ "screenshot should return at least one content item"+ );}#[test]#[cfg(target_os = "macos")]fn screenshot_jpeg_format() {//! Call screenshot with format=jpeg — should return a JPEG image.- let Some(mut d) = RawDriver::spawn() else { return; };+ let Some(mut d) = RawDriver::spawn() else {+ return;+ };d.send(&serde_json::json!({"jsonrpc":"2.0","id":1,"method":"initialize","params":{}}));d.recv();@@ -83,20 +102,37 @@ fn screenshot_jpeg_format() {"params":{"name":"screenshot","arguments":{"format":"jpeg","quality":70}}}));let resp = d.recv();- assert!(resp["error"].is_null(), "Protocol error from screenshot: {resp:?}");+ assert!(+ resp["error"].is_null(),+ "Protocol error from screenshot: {resp:?}"+ );let content = resp["result"]["content"].as_array().expect("content array");let is_error = resp["result"]["isError"].as_bool().unwrap_or(false);if !is_error {let has_jpeg = content.iter().any(|c| {- c["type"] == "image" && c["mimeType"].as_str().unwrap_or("") == "image/jpeg"+ c["type"] == "image"+ && c["mimeType"].as_str().unwrap_or("") == "image/jpeg"&& c["data"].as_str().map(|s| s.len() > 10).unwrap_or(false)});- assert!(has_jpeg, "Expected image/jpeg in screenshot response, got: {content:?}");+ assert!(+ has_jpeg,+ "Expected image/jpeg in screenshot response, got: {content:?}"+ );// Verify structured content has width and height.let sc = &resp["result"]["structuredContent"];- assert!(sc["width"].as_f64().unwrap_or(0.0) > 0.0, "Expected positive width in structuredContent");- assert!(sc["height"].as_f64().unwrap_or(0.0) > 0.0, "Expected positive height in structuredContent");- assert_eq!(sc["format"].as_str().unwrap_or(""), "jpeg", "Expected format=jpeg");+ assert!(+ sc["width"].as_f64().unwrap_or(0.0) > 0.0,+ "Expected positive width in structuredContent"+ );+ assert!(+ sc["height"].as_f64().unwrap_or(0.0) > 0.0,+ "Expected positive height in structuredContent"+ );+ assert_eq!(+ sc["format"].as_str().unwrap_or(""),+ "jpeg",+ "Expected format=jpeg"+ );}}@@ -105,7 +141,9 @@ fn screenshot_jpeg_format() {fn zoom_tool_returns_jpeg() {//! Call zoom on a visible window and verify the result contains a JPEG image.//! Skips gracefully if no windows are visible (headless environment).- let Some(mut d) = RawDriver::spawn() else { return; };+ let Some(mut d) = RawDriver::spawn() else {+ return;+ };d.send(&serde_json::json!({"jsonrpc":"2.0","id":1,"method":"initialize","params":{}}));d.recv();@@ -124,8 +162,12 @@ fn zoom_tool_returns_jpeg() {// Try windows until one captures successfully.let mut found_jpeg = false;let mut tried = 0usize;+ let mut zoom_errors: Vec<String> = Vec::new();+ let mut successful_without_jpeg = false;for win in windows.iter().take(5) {- let Some(wid) = win["window_id"].as_u64() else { continue; };+ let Some(wid) = win["window_id"].as_u64() else {+ continue;+ };tried += 1;d.send(&serde_json::json!({@@ -138,19 +180,39 @@ fn zoom_tool_returns_jpeg() {let r = d.recv();if r["result"]["isError"].as_bool().unwrap_or(false) {// This window might be off-screen or not capturable — try the next.+ let text = r["result"]["content"]+ .as_array()+ .and_then(|items| items.iter().find_map(|c| c["text"].as_str()))+ .unwrap_or("<no error text>")+ .to_owned();+ zoom_errors.push(format!("window_id={wid}: {text}"));continue;}let content = r["result"]["content"].as_array().expect("content array");found_jpeg = content.iter().any(|c| {- c["type"] == "image" && c["mimeType"].as_str().unwrap_or("") == "image/jpeg"+ c["type"] == "image"+ && c["mimeType"].as_str().unwrap_or("") == "image/jpeg"&& c["data"].as_str().map(|s| s.len() > 10).unwrap_or(false)});+ if !found_jpeg {+ successful_without_jpeg = true;+ }if found_jpeg {// Also verify structuredContent.let sc = &r["result"]["structuredContent"];- assert!(sc["width"].as_f64().unwrap_or(0.0) > 0.0, "Expected positive width: {sc:?}");- assert!(sc["height"].as_f64().unwrap_or(0.0) > 0.0, "Expected positive height: {sc:?}");- assert_eq!(sc["format"].as_str().unwrap_or(""), "jpeg", "Expected format=jpeg: {sc:?}");+ assert!(+ sc["width"].as_f64().unwrap_or(0.0) > 0.0,+ "Expected positive width: {sc:?}"+ );+ assert!(+ sc["height"].as_f64().unwrap_or(0.0) > 0.0,+ "Expected positive height: {sc:?}"+ );+ assert_eq!(+ sc["format"].as_str().unwrap_or(""),+ "jpeg",+ "Expected format=jpeg: {sc:?}"+ );break;}}@@ -159,7 +221,26 @@ fn zoom_tool_returns_jpeg() {eprintln!("No on-screen windows found — skipping zoom test");return;}- assert!(found_jpeg, "Expected at least one window to return a valid JPEG from zoom");+ if !found_jpeg+ && !successful_without_jpeg+ && cfg!(target_os = "macos")+ && !zoom_errors.is_empty()+ && zoom_errors+ .iter()+ .all(|e| e.contains("screencapture failed"))+ {+ eprintln!(+ "No visible windows were capturable by the raw unbundled test process — skipping zoom test. \+ Errors: {}",+ zoom_errors.join(" | ")+ );+ return;+ }+ assert!(+ found_jpeg,+ "Expected at least one window to return a valid JPEG from zoom. Errors: {}",+ zoom_errors.join(" | ")+ );}#[test]@@ -169,7 +250,9 @@ fn zoom_from_zoom_click_round_trip() {//! 1. click(from_zoom=true) with no zoom context returns the expected error.//! 2. zoom() stores context; subsequent click(from_zoom=true) does NOT return//! that error (translation succeeded, even if the click itself may fail).- let Some(mut d) = RawDriver::spawn() else { return; };+ let Some(mut d) = RawDriver::spawn() else {+ return;+ };d.send(&serde_json::json!({"jsonrpc":"2.0","id":1,"method":"initialize","params":{}}));d.recv();@@ -180,7 +263,10 @@ fn zoom_from_zoom_click_round_trip() {"params":{"name":"list_windows","arguments":{"on_screen_only":true}}}));let resp = d.recv();- let wins = resp["result"]["structuredContent"]["windows"].as_array().cloned().unwrap_or_default();+ let wins = resp["result"]["structuredContent"]["windows"]+ .as_array()+ .cloned()+ .unwrap_or_default();let Some(win) = wins.iter().find(|w| w["pid"].as_i64().is_some()) else {eprintln!("No on-screen windows — skipping from_zoom test");return;@@ -196,8 +282,14 @@ fn zoom_from_zoom_click_round_trip() {let resp = d.recv();let is_err = resp["result"]["isError"].as_bool().unwrap_or(false);let err_text = resp["result"]["content"][0]["text"].as_str().unwrap_or("");- assert!(is_err, "Expected error when from_zoom=true with no context, got: {resp:?}");- assert!(err_text.contains("no zoom context"), "Expected 'no zoom context' error, got: {err_text}");+ assert!(+ is_err,+ "Expected error when from_zoom=true with no context, got: {resp:?}"+ );+ assert!(+ err_text.contains("no zoom context"),+ "Expected 'no zoom context' error, got: {err_text}"+ );// Step 2: call zoom on that window to store context.d.send(&serde_json::json!({@@ -213,7 +305,10 @@ fn zoom_from_zoom_click_round_trip() {if !cfg!(target_os = "windows") {// Verify zoom returned structured content with width/height.let sc = &resp["result"]["structuredContent"];- assert!(sc["width"].as_u64().unwrap_or(0) > 0, "zoom should return positive width: {sc:?}");+ assert!(+ sc["width"].as_u64().unwrap_or(0) > 0,+ "zoom should return positive width: {sc:?}"+ );}// Step 3: click with from_zoom=true — should NOT return the "no zoom context" error.@@ -224,17 +319,22 @@ fn zoom_from_zoom_click_round_trip() {let resp = d.recv();let err_text = resp["result"]["content"][0]["text"].as_str().unwrap_or("");// Translation should succeed — if click fails it's for another reason (target app state), not missing context.- assert!(!err_text.contains("no zoom context"),- "After zoom(), from_zoom click should not say 'no zoom context', got: {err_text}");+ assert!(+ !err_text.contains("no zoom context"),+ "After zoom(), from_zoom click should not say 'no zoom context', got: {err_text}"+ );}#[test]#[cfg(any(target_os = "macos", target_os = "windows"))]fn recording_session() {//! Enable recording, invoke a non-read-only tool (recorded), disable, verify action.json written.- let Some(mut d) = RawDriver::spawn() else { return; };+ let Some(mut d) = RawDriver::spawn() else {+ return;+ };- let tmp_dir = std::env::temp_dir().join(format!("cua-driver-rs-rec-test-{}", std::process::id()));+ let tmp_dir =+ std::env::temp_dir().join(format!("cua-driver-rs-rec-test-{}", std::process::id()));let tmp_str = tmp_dir.to_string_lossy().to_string();d.send(&serde_json::json!({"jsonrpc":"2.0","id":1,"method":"initialize","params":{}}));@@ -246,8 +346,13 @@ fn recording_session() {"params":{"name":"start_recording","arguments":{"output_dir":tmp_str,"record_video":false}}}));let resp = d.recv();- assert!(!resp["result"]["isError"].as_bool().unwrap_or(false), "start_recording failed: {resp:?}");- assert!(resp["result"]["structuredContent"]["enabled"].as_bool().unwrap_or(false));+ assert!(+ !resp["result"]["isError"].as_bool().unwrap_or(false),+ "start_recording failed: {resp:?}"+ );+ assert!(resp["result"]["structuredContent"]["enabled"]+ .as_bool()+ .unwrap_or(false));// Invoke a non-read-only tool — should be recorded.if cfg!(target_os = "windows") {@@ -269,7 +374,9 @@ fn recording_session() {"params":{"name":"get_recording_state","arguments":{}}}));let resp = d.recv();- let next_turn = resp["result"]["structuredContent"]["next_turn"].as_u64().unwrap_or(0);+ let next_turn = resp["result"]["structuredContent"]["next_turn"]+ .as_u64()+ .unwrap_or(0);assert!(next_turn >= 2, "expected next_turn >= 2, got {next_turn}");// Disable recording.@@ -278,7 +385,9 @@ fn recording_session() {"params":{"name":"stop_recording","arguments":{}}}));let resp = d.recv();- assert!(!resp["result"]["structuredContent"]["enabled"].as_bool().unwrap_or(true));+ assert!(!resp["result"]["structuredContent"]["enabled"]+ .as_bool()+ .unwrap_or(true));drop(d);// Small sync wait for the write to flush.@@ -287,14 +396,16 @@ fn recording_session() {// Verify turn-00001/action.json was written.let action_path = tmp_dir.join("turn-00001").join("action.json");assert!(action_path.exists(), "Expected {action_path:?} to exist");- let content: serde_json::Value = serde_json::from_str(- &std::fs::read_to_string(&action_path).unwrap()- ).unwrap();+ let content: serde_json::Value =+ serde_json::from_str(&std::fs::read_to_string(&action_path).unwrap()).unwrap();if cfg!(target_os = "windows") {assert_eq!(content["tool"].as_str().unwrap_or(""), "move_cursor");assert_eq!(content["arguments"]["x"].as_f64().unwrap_or(0.0), 100.0);} else {- assert_eq!(content["tool"].as_str().unwrap_or(""), "set_agent_cursor_enabled");+ assert_eq!(+ content["tool"].as_str().unwrap_or(""),+ "set_agent_cursor_enabled"+ );assert_eq!(content["arguments"]["enabled"].as_bool(), Some(false));}@@ -306,7 +417,9 @@ fn recording_session() {fn recording_screenshot_capture() {//! When recording is active and a tool call includes a window_id, a screenshot.png//! should appear alongside action.json in the turn folder.- let Some(mut d) = RawDriver::spawn() else { return; };+ let Some(mut d) = RawDriver::spawn() else {+ return;+ };let tmp_dir = std::env::temp_dir().join(format!("cua-driver-rs-rec-ss-{}", std::process::id()));let tmp_str = tmp_dir.to_string_lossy().to_string();@@ -320,9 +433,13 @@ fn recording_screenshot_capture() {"params":{"name":"list_windows","arguments":{"on_screen_only":true}}}));let resp = d.recv();- let wins = resp["result"]["structuredContent"]["windows"].as_array().cloned().unwrap_or_default();+ let wins = resp["result"]["structuredContent"]["windows"]+ .as_array()+ .cloned()+ .unwrap_or_default();// Prefer windows with non-empty titles (more likely capturable).- let win = wins.iter()+ let win = wins+ .iter().find(|w| w["pid"].as_i64().is_some() && !w["title"].as_str().unwrap_or("").is_empty()).or_else(|| wins.iter().find(|w| w["pid"].as_i64().is_some()));let Some(win) = win else {@@ -338,7 +455,10 @@ fn recording_screenshot_capture() {"params":{"name":"start_recording","arguments":{"output_dir":tmp_str,"record_video":false}}}));let resp = d.recv();- assert!(!resp["result"]["isError"].as_bool().unwrap_or(false), "start_recording failed: {resp:?}");+ assert!(+ !resp["result"]["isError"].as_bool().unwrap_or(false),+ "start_recording failed: {resp:?}"+ );// Invoke click (non-read-only) with window_id + pid — recording should capture screenshot.d.send(&serde_json::json!({@@ -360,10 +480,12 @@ fn recording_screenshot_capture() {// action.json must exist.let turn_dir = tmp_dir.join("turn-00001");let action_path = turn_dir.join("action.json");- assert!(action_path.exists(), "Expected action.json at {action_path:?}");- let content: serde_json::Value = serde_json::from_str(- &std::fs::read_to_string(&action_path).unwrap()- ).unwrap();+ assert!(+ action_path.exists(),+ "Expected action.json at {action_path:?}"+ );+ let content: serde_json::Value =+ serde_json::from_str(&std::fs::read_to_string(&action_path).unwrap()).unwrap();assert_eq!(content["tool"].as_str().unwrap_or(""), "click");// screenshot.png should exist if screencapture succeeded for this window.@@ -373,7 +495,11 @@ fn recording_screenshot_capture() {} else {let ss_bytes = std::fs::read(&ss_path).unwrap();assert!(!ss_bytes.is_empty(), "screenshot.png should not be empty");- assert_eq!(&ss_bytes[..4], b"\x89PNG", "screenshot.png should be a valid PNG");+ assert_eq!(+ &ss_bytes[..4],+ b"\x89PNG",+ "screenshot.png should be a valid PNG"+ );// click.png should also exist (click at 0,0 with crosshair marker).let click_path = turn_dir.join("click.png");@@ -382,7 +508,11 @@ fn recording_screenshot_capture() {} else {let click_bytes = std::fs::read(&click_path).unwrap();assert!(!click_bytes.is_empty(), "click.png should not be empty");- assert_eq!(&click_bytes[..4], b"\x89PNG", "click.png should be a valid PNG");+ assert_eq!(+ &click_bytes[..4],+ b"\x89PNG",+ "click.png should be a valid PNG"+ );}}@@ -399,7 +529,9 @@ fn start_recording_record_video_flag_accepted() {//! session enters the enabled state. Full video lifecycle is//! covered by the manual demo + the per-platform smoke when//! ffmpeg is installed.- let Some(mut d) = RawDriver::spawn() else { return; };+ let Some(mut d) = RawDriver::spawn() else {+ return;+ };let tmp_dir = std::env::temp_dir().join(format!("cua-driver-rs-recvid-{}", std::process::id()));let tmp_str = tmp_dir.to_string_lossy().to_string();@@ -415,9 +547,15 @@ fn start_recording_record_video_flag_accepted() {}}}));let resp = d.recv();- assert!(resp["error"].is_null(), "Expected no JSON-RPC error, got: {resp:?}");+ assert!(+ resp["error"].is_null(),+ "Expected no JSON-RPC error, got: {resp:?}"+ );let is_err = resp["result"]["isError"].as_bool().unwrap_or(false);- assert!(!is_err, "start_recording with record_video:false should succeed: {resp:?}");+ assert!(+ !is_err,+ "start_recording with record_video:false should succeed: {resp:?}"+ );d.send(&serde_json::json!({"jsonrpc":"2.0","id":3,"method":"tools/call",@@ -432,15 +570,19 @@ fn start_recording_record_video_flag_accepted() {#[cfg(any(target_os = "macos", target_os = "windows"))]fn replay_trajectory() {//! Write a minimal trajectory (one move_cursor turn), replay it, verify succeeded=1.- let Some(mut d) = RawDriver::spawn() else { return; };+ let Some(mut d) = RawDriver::spawn() else {+ return;+ };- let traj_dir = std::env::temp_dir().join(format!("cua-driver-rs-replay-{}", std::process::id()));+ let traj_dir =+ std::env::temp_dir().join(format!("cua-driver-rs-replay-{}", std::process::id()));let turn_dir = traj_dir.join("turn-00001");std::fs::create_dir_all(&turn_dir).unwrap();std::fs::write(turn_dir.join("action.json"),r#"{"tool":"move_cursor","arguments":{"x":50.0,"y":60.0}}"#,- ).unwrap();+ )+ .unwrap();d.send(&serde_json::json!({"jsonrpc":"2.0","id":1,"method":"initialize","params":{}}));d.recv();@@ -453,10 +595,21 @@ fn replay_trajectory() {}}}));let resp = d.recv();- assert!(!resp["result"]["isError"].as_bool().unwrap_or(false), "replay error: {resp:?}");+ assert!(+ !resp["result"]["isError"].as_bool().unwrap_or(false),+ "replay error: {resp:?}"+ );let sc = &resp["result"]["structuredContent"];- assert_eq!(sc["attempted"].as_u64().unwrap_or(0), 1, "expected attempted=1");- assert_eq!(sc["succeeded"].as_u64().unwrap_or(0), 1, "expected succeeded=1");+ assert_eq!(+ sc["attempted"].as_u64().unwrap_or(0),+ 1,+ "expected attempted=1"+ );+ assert_eq!(+ sc["succeeded"].as_u64().unwrap_or(0),+ 1,+ "expected succeeded=1"+ );assert_eq!(sc["failed"].as_u64().unwrap_or(99), 0, "expected failed=0");drop(d);@@ -468,7 +621,9 @@ fn replay_trajectory() {fn click_debug_image_out() {//! click with debug_image_out writes a PNG crosshair file and then proceeds.//! Verifies the debug capture path works end-to-end.- let Some(mut d) = RawDriver::spawn() else { return; };+ let Some(mut d) = RawDriver::spawn() else {+ return;+ };d.send(&serde_json::json!({"jsonrpc":"2.0","id":1,"method":"initialize","params":{}}));d.recv();@@ -480,16 +635,19 @@ fn click_debug_image_out() {let resp = d.recv();let wins = resp["result"]["structuredContent"]["windows"].as_array();// Prefer a window with a non-empty title to avoid uncapturable overlay/system windows.- let Some(win) = wins.and_then(|a| a.iter().find(|w| {- w["pid"].as_i64().is_some() && !w["title"].as_str().unwrap_or("").is_empty()- }).or_else(|| a.iter().find(|w| w["pid"].as_i64().is_some()))) else {+ let Some(win) = wins.and_then(|a| {+ a.iter()+ .find(|w| w["pid"].as_i64().is_some() && !w["title"].as_str().unwrap_or("").is_empty())+ .or_else(|| a.iter().find(|w| w["pid"].as_i64().is_some()))+ }) else {eprintln!("No on-screen windows — skipping debug_image_out test");return;};let pid = win["pid"].as_i64().unwrap();let wid = win["window_id"].as_u64().unwrap();- let dbg_path = std::env::temp_dir().join(format!("cua-driver-rs-dbg-{}.png", std::process::id()));+ let dbg_path =+ std::env::temp_dir().join(format!("cua-driver-rs-dbg-{}.png", std::process::id()));let dbg_path_str = dbg_path.to_string_lossy().to_string();// Click with debug_image_out — should write the PNG and then proceed.@@ -502,7 +660,10 @@ fn click_debug_image_out() {}}}));let resp = d.recv();- assert!(resp["error"].is_null(), "Protocol error from click+debug_image_out: {resp:?}");+ assert!(+ resp["error"].is_null(),+ "Protocol error from click+debug_image_out: {resp:?}"+ );// The tool may return isError if screencapture can't capture this particular window// (e.g., overlay windows, system windows). If that happens, skip the PNG check.@@ -514,9 +675,15 @@ fn click_debug_image_out() {}// Verify the PNG was actually written.- assert!(dbg_path.exists(), "debug_image_out PNG was not written to {dbg_path:?}");+ assert!(+ dbg_path.exists(),+ "debug_image_out PNG was not written to {dbg_path:?}"+ );let meta = std::fs::metadata(&dbg_path).expect("metadata");- assert!(meta.len() > 100, "debug_image_out PNG is suspiciously small");+ assert!(+ meta.len() > 100,+ "debug_image_out PNG is suspiciously small"+ );let _ = std::fs::remove_file(&dbg_path);}@@ -526,7 +693,9 @@ fn click_debug_image_out() {fn set_config_screenshot_resize() {//! set_config(max_image_dimension=200) then screenshot — returned image must//! have both dimensions ≤ 200. Verifies the ResizeRegistry pipeline end-to-end.- let Some(mut d) = RawDriver::spawn() else { return; };+ let Some(mut d) = RawDriver::spawn() else {+ return;+ };d.send(&serde_json::json!({"jsonrpc":"2.0","id":1,"method":"initialize","params":{}}));d.recv();@@ -538,7 +707,10 @@ fn set_config_screenshot_resize() {}));let resp = d.recv();assert!(resp["error"].is_null(), "set_config failed: {resp:?}");- assert!(!resp["result"]["isError"].as_bool().unwrap_or(false), "set_config returned error: {resp:?}");+ assert!(+ !resp["result"]["isError"].as_bool().unwrap_or(false),+ "set_config returned error: {resp:?}"+ );// Capture full display; must be ≤ 200 px in both dimensions.d.send(&serde_json::json!({@@ -562,8 +734,14 @@ fn set_config_screenshot_resize() {eprintln!("screenshot returned no dimensions — skipping resize assertion: {resp:?}");return;};- assert!(w <= 200, "screenshot width {w} should be ≤ 200 after max_image_dimension=200");- assert!(h <= 200, "screenshot height {h} should be ≤ 200 after max_image_dimension=200");+ assert!(+ w <= 200,+ "screenshot width {w} should be ≤ 200 after max_image_dimension=200"+ );+ assert!(+ h <= 200,+ "screenshot height {h} should be ≤ 200 after max_image_dimension=200"+ );// get_config should reflect the updated value.d.send(&serde_json::json!({@@ -572,5 +750,9 @@ fn set_config_screenshot_resize() {}));let resp = d.recv();let dim = resp["result"]["structuredContent"]["max_image_dimension"].as_u64();- assert_eq!(dim, Some(200), "get_config should reflect max_image_dimension=200");+ assert_eq!(+ dim,+ Some(200),+ "get_config should reflect max_image_dimension=200"+ );}diff --git a/libs/cua-driver/rust/crates/platform-macos/src/capture.rs b/libs/cua-driver/rust/crates/platform-macos/src/capture.rsindex 22e507a148..80944336fc 100644--- a/libs/cua-driver/rust/crates/platform-macos/src/capture.rs+++ b/libs/cua-driver/rust/crates/platform-macos/src/capture.rs@@ -17,17 +17,28 @@ use std::process::Command;pub fn screenshot_window_bytes(window_id: u32) -> anyhow::Result<Vec<u8>> {let tmp_path = format!("/tmp/cua-driver-rs-capture-{}.png", window_id);- let status = Command::new("screencapture")+ let output = Command::new("screencapture").args([- "-l", &window_id.to_string(),- "-x", // no sound- "-o", // no shadow+ "-l",+ &window_id.to_string(),+ "-x", // no sound+ "-o", // no shadow&tmp_path,])- .status()?;-- if !status.success() {- anyhow::bail!("screencapture failed for window {window_id}");+ .output()?;++ if !output.status.success() {+ let stderr = String::from_utf8_lossy(&output.stderr).trim().to_owned();+ if stderr.is_empty() {+ anyhow::bail!(+ "screencapture failed for window {window_id} with status {}",+ output.status+ );+ }+ anyhow::bail!(+ "screencapture failed for window {window_id} with status {}: {stderr}",+ output.status+ );}let bytes = std::fs::read(&tmp_path)?;@@ -54,12 +65,22 @@ pub fn screenshot_display_bytes() -> anyhow::Result<Vec<u8>> {// Use a pid-unique path so concurrent cua-driver processes don't step on each other.let tmp_path = format!("/tmp/cua-driver-rs-display-{}.png", std::process::id());- let status = Command::new("screencapture")+ let output = Command::new("screencapture").args(["-x", &*tmp_path])- .status()?;-- if !status.success() {- anyhow::bail!("screencapture failed for main display");+ .output()?;++ if !output.status.success() {+ let stderr = String::from_utf8_lossy(&output.stderr).trim().to_owned();+ if stderr.is_empty() {+ anyhow::bail!(+ "screencapture failed for main display with status {}",+ output.status+ );+ }+ anyhow::bail!(+ "screencapture failed for main display with status {}: {stderr}",+ output.status+ );}let bytes = std::fs::read(&tmp_path)?;@@ -101,12 +122,7 @@ pub fn resize_png_if_needed(png_bytes: &[u8], max_dim: u32) -> anyhow::Result<Ve/// `path`. Used by `click`'s `debug_image_out` param to verify/// coordinate spaces. The crosshair uses top-left-origin coords/// matching the click tool's convention.-pub fn write_crosshair_png(- png_bytes: &[u8],- cx: f64,- cy: f64,- path: &str,-) -> anyhow::Result<()> {+pub fn write_crosshair_png(png_bytes: &[u8], cx: f64, cy: f64, path: &str) -> anyhow::Result<()> {cua_driver_core::image_utils::write_crosshair_png(png_bytes, cx, cy, path)}diff --git a/libs/cua-driver/rust/crates/platform-macos/src/focus_steal.rs b/libs/cua-driver/rust/crates/platform-macos/src/focus_steal.rsindex 65ccf875eb..7e67209e55 100644--- a/libs/cua-driver/rust/crates/platform-macos/src/focus_steal.rs+++ b/libs/cua-driver/rust/crates/platform-macos/src/focus_steal.rs@@ -60,8 +60,8 @@ use std::sync::{Arc, Mutex, OnceLock};use std::time::{Duration, Instant};use objc2_app_kit::{- NSApplicationActivationOptions, NSRunningApplication, NSWorkspace,- NSWorkspaceDidActivateApplicationNotification, NSWorkspaceApplicationKey,+ NSApplicationActivationOptions, NSRunningApplication, NSWorkspace, NSWorkspaceApplicationKey,+ NSWorkspaceDidActivateApplicationNotification,};use objc2_foundation::NSOperationQueue;use uuid::Uuid;@@ -134,9 +134,7 @@ impl FocusStealPreventer {origin: &'static str,) -> SuppressionLease {let shared = Self::shared();- let handle = shared- .dispatcher- .add(target_pid, restore_to, origin);+ let handle = shared.dispatcher.add(target_pid, restore_to, origin);SuppressionLease {handle,dispatcher: Arc::clone(&shared.dispatcher),@@ -160,12 +158,6 @@ impl FocusStealPreventer {let _lease = Self::begin_suppression(target_pid, restore_to, origin);f().await}-- /// For tests. Returns the singleton's dispatcher arc.- #[cfg(test)]- fn dispatcher(&self) -> &Arc<Dispatcher> {- &self.dispatcher- }}/// Begin-suppression convenience that bounces through the singleton.@@ -442,10 +434,7 @@ fn install_observer(dispatcher: &Arc<Dispatcher>) {////// Runs on the observer queue's background thread — safe to call/// blocking system APIs.-fn handle_activation(- dispatcher: &Arc<Dispatcher>,- note: &objc2_foundation::NSNotification,-) {+fn handle_activation(dispatcher: &Arc<Dispatcher>, note: &objc2_foundation::NSNotification) {use objc2::msg_send;use objc2::runtime::AnyObject;@@ -457,8 +446,7 @@ fn handle_activation(// userInfo[NSWorkspaceApplicationKey] -> NSRunningApplication*.// We go through a raw msg_send to avoid Retained generic// bookkeeping for the cross-cast.- let app_ptr: *mut AnyObject =- msg_send![&*info, objectForKey: NSWorkspaceApplicationKey];+ let app_ptr: *mut AnyObject = msg_send![&*info, objectForKey: NSWorkspaceApplicationKey];if app_ptr.is_null() {return;}@@ -476,9 +464,7 @@ fn handle_activation(/// thread — Apple documents `activateWithOptions:` as thread-safe.fn restore_focus(pid: i32) {unsafe {- if let Some(app) =- NSRunningApplication::runningApplicationWithProcessIdentifier(pid)- {+ if let Some(app) = NSRunningApplication::runningApplicationWithProcessIdentifier(pid) {let _ = app.activateWithOptions(NSApplicationActivationOptions(0));}}diff --git a/libs/cua-driver/rust/crates/platform-macos/src/input/mouse.rs b/libs/cua-driver/rust/crates/platform-macos/src/input/mouse.rsindex e838f2bd74..47c30f9775 100644--- a/libs/cua-driver/rust/crates/platform-macos/src/input/mouse.rs+++ b/libs/cua-driver/rust/crates/platform-macos/src/input/mouse.rs@@ -22,7 +22,13 @@ use foreign_types::ForeignType;/// Window-local coordinates for backgrounded targets: if `window_local` is/// `Some((wx, wy))`, stamps a window-local point via `CGEventSetWindowLocation`/// SPI so WindowServer's hit-test uses the local point directly.-pub fn click_at_xy(pid: i32, x: f64, y: f64, count: usize, modifiers: &[&str]) -> anyhow::Result<()> {+pub fn click_at_xy(+ pid: i32,+ x: f64,+ y: f64,+ count: usize,+ modifiers: &[&str],+) -> anyhow::Result<()> {click_at_xy_inner(pid, x, y, None, None, count, modifiers)}@@ -43,9 +49,21 @@ pub fn click_at_xy_desktop(x: f64, y: f64, count: usize, button: &str) -> anyhow.map_err(|_| anyhow::anyhow!("CGEventSource::new failed"))?;let point = CGPoint::new(x, y);let (down_ty, up_ty, btn) = match button {- "right" => (CGEventType::RightMouseDown, CGEventType::RightMouseUp, CGMouseButton::Right),- "middle" => (CGEventType::OtherMouseDown, CGEventType::OtherMouseUp, CGMouseButton::Center),- _ => (CGEventType::LeftMouseDown, CGEventType::LeftMouseUp, CGMouseButton::Left),+ "right" => (+ CGEventType::RightMouseDown,+ CGEventType::RightMouseUp,+ CGMouseButton::Right,+ ),+ "middle" => (+ CGEventType::OtherMouseDown,+ CGEventType::OtherMouseUp,+ CGMouseButton::Center,+ ),+ _ => (+ CGEventType::LeftMouseDown,+ CGEventType::LeftMouseUp,+ CGMouseButton::Left,+ ),};// Warp the REAL cursor to the point first (the macOS peer of Linux XTest's// pointer warp). A synthetic MouseMoved event does not relocate the hardware@@ -80,8 +98,10 @@ extern "C" {/// (f51 / f58 / f91 / f92) for better backgrounded-target delivery.pub fn click_at_xy_with_window_local(pid: i32,- x: f64, y: f64,- wx: f64, wy: f64,+ x: f64,+ y: f64,+ wx: f64,+ wy: f64,wid: u32,count: usize,modifiers: &[&str],@@ -91,7 +111,8 @@ pub fn click_at_xy_with_window_local(fn click_at_xy_inner(pid: i32,- x: f64, y: f64,+ x: f64,+ y: f64,window_local: Option<(f64, f64)>,wid: Option<u32>,count: usize,@@ -128,12 +149,22 @@ fn click_at_xy_inner(CGEventType::LeftMouseDown,point,CGMouseButton::Left,- ).map_err(|_| anyhow::anyhow!("CGEvent::new_mouse_event(down) failed"))?;+ )+ .map_err(|_| anyhow::anyhow!("CGEvent::new_mouse_event(down) failed"))?;if flags != CGEventFlags::CGEventFlagNull {down.set_flags(flags);}- post_mouse_event(pid, &down, window_local, wid, click_group_id, click_state, 0);+ post_mouse_event(+ pid,+ &down,+ window_local,+ wid,+ click_group_id,+ click_state,+ 0,+ 3,+ );// 28 ms down→up gap: an NSButton's mouseDown enters a modal tracking// loop that polls for the matching mouseUp; too tight a gap can race the// loop's first poll and the click is dropped. 16 ms was under that@@ -145,12 +176,22 @@ fn click_at_xy_inner(CGEventType::LeftMouseUp,point,CGMouseButton::Left,- ).map_err(|_| anyhow::anyhow!("CGEvent::new_mouse_event(up) failed"))?;+ )+ .map_err(|_| anyhow::anyhow!("CGEvent::new_mouse_event(up) failed"))?;if flags != CGEventFlags::CGEventFlagNull {up.set_flags(flags);}- post_mouse_event(pid, &up, window_local, wid, click_group_id, click_state, 0);+ post_mouse_event(+ pid,+ &up,+ window_local,+ wid,+ click_group_id,+ click_state,+ 0,+ 3,+ );if count > 1 {std::thread::sleep(std::time::Duration::from_millis(80));@@ -161,9 +202,9 @@ fn click_at_xy_inner(/// Full Chromium-compatible left-click recipe matching Swift's `clickViaAuthSignedPost`.///-/// Sequence (Swift reference minus the `activate_without_raise` prologue — that step-/// causes window-z side effects under Rust's overlay; it is reserved for a future-/// explicit opt-in once the overlay re-pins after focus changes):+/// The focus-without-raise prologue makes the target window key without changing+/// its z-order, which Chromium requires before it accepts a background pixel+/// mouseDown. The cursor overlay is re-pinned by the click tool after dispatch./// 1. Stamped `mouseMoved` at target coords (f0=2, cursor-state primer)./// 2. Off-screen primer down/up at (-1, -1) (f0=1/2) — satisfies Chromium's/// user-activation gate without hitting any DOM element.@@ -193,15 +234,22 @@ pub fn click_at_xy_chromium() -> anyhow::Result<()> {use std::time::{SystemTime, UNIX_EPOCH};+ // Chromium's first-mouse handling rejects a background click delivered to+ // a non-key window. This SkyLight focus record keys the requested window+ // without raising it or moving the user's cursor.+ if crate::input::skylight::activate_without_raise(pid as libc::pid_t, wid) {+ std::thread::sleep(std::time::Duration::from_millis(50));+ }+let source = CGEventSource::new(CGEventSourceStateID::HIDSystemState).map_err(|_| anyhow::anyhow!("CGEventSource::new failed"))?;- let target = CGPoint::new(screen_x, screen_y);+ let target = CGPoint::new(screen_x, screen_y);let off_screen = CGPoint::new(-1.0, -1.0);- let win_local = (win_local_x, win_local_y);- let off_local = (-1.0_f64, -1.0_f64);- let flags = parse_modifier_flags(modifiers);+ let win_local = (win_local_x, win_local_y);+ let off_local = (-1.0_f64, -1.0_f64);+ let flags = parse_modifier_flags(modifiers);let click_pairs = count.max(1).min(2);- let window_id = wid as i64;+ let window_id = wid as i64;// All 5 events share the same click-group ID so WindowServer / Chromium// treat the sequence as one gesture (Swift: field 58).@@ -214,18 +262,20 @@ pub fn click_at_xy_chromium(// this closure is Fn (callable multiple times).let stamp = |event: &CGEvent, local: (f64, f64), click_state: i64, phase: i64| {let ptr = event.as_ptr() as *mut std::ffi::c_void;- let set = |f: u32, v: i64| { crate::input::skylight::set_integer_field(ptr, f, v); };- set(0, phase); // kCGMouseEventNumber (gesture phase)- set(1, click_state); // kCGMouseEventClickState- set(3, 0); // kCGMouseEventButtonNumber (left)- set(7, 3); // kCGMouseEventSubtype (NSEventSubtypeTouch)- set(40, pid as i64); // Chromium synthetic-event filter+ let set = |f: u32, v: i64| {+ crate::input::skylight::set_integer_field(ptr, f, v);+ };+ set(0, phase); // kCGMouseEventNumber (gesture phase)+ set(1, click_state); // kCGMouseEventClickState+ set(3, 0); // kCGMouseEventButtonNumber (left)+ set(7, 3); // kCGMouseEventSubtype (NSEventSubtypeTouch)+ set(40, pid as i64); // Chromium synthetic-event filterif window_id != 0 {- set(51, window_id); // windowNumber (NSEvent bridge equivalent)- set(91, window_id); // kCGMouseEventWindowUnderMousePointer- set(92, window_id); // kCGMouseEventWindowUnderMousePointerThatCanHandleThisEvent+ set(51, window_id); // windowNumber (NSEvent bridge equivalent)+ set(91, window_id); // kCGMouseEventWindowUnderMousePointer+ set(92, window_id); // kCGMouseEventWindowUnderMousePointerThatCanHandleThisEvent}- set(58, click_group_id); // click-group ID (gesture coalescing)+ set(58, click_group_id); // click-group ID (gesture coalescing)crate::input::skylight::set_window_location(ptr, local.0, local.1);if flags != CGEventFlags::CGEventFlagNull {event.set_flags(flags);@@ -241,8 +291,12 @@ pub fn click_at_xy_chromium(// Step 1: mouseMoved at target (phase=2, clickState=0).let move_event = CGEvent::new_mouse_event(- source.clone(), CGEventType::MouseMoved, target, CGMouseButton::Left,- ).map_err(|_| anyhow::anyhow!("mouseMoved event creation failed"))?;+ source.clone(),+ CGEventType::MouseMoved,+ target,+ CGMouseButton::Left,+ )+ .map_err(|_| anyhow::anyhow!("mouseMoved event creation failed"))?;stamp(&move_event, win_local, 0, 2);post(&move_event);std::thread::sleep(std::time::Duration::from_millis(15));@@ -250,15 +304,23 @@ pub fn click_at_xy_chromium(// Step 2: off-screen primer click — opens Chromium user-activation gate// at an off-screen coordinate that can't hit any DOM element.let primer_down = CGEvent::new_mouse_event(- source.clone(), CGEventType::LeftMouseDown, off_screen, CGMouseButton::Left,- ).map_err(|_| anyhow::anyhow!("primer down event creation failed"))?;+ source.clone(),+ CGEventType::LeftMouseDown,+ off_screen,+ CGMouseButton::Left,+ )+ .map_err(|_| anyhow::anyhow!("primer down event creation failed"))?;stamp(&primer_down, off_local, 1, 1);post(&primer_down);std::thread::sleep(std::time::Duration::from_millis(1));let primer_up = CGEvent::new_mouse_event(- source.clone(), CGEventType::LeftMouseUp, off_screen, CGMouseButton::Left,- ).map_err(|_| anyhow::anyhow!("primer up event creation failed"))?;+ source.clone(),+ CGEventType::LeftMouseUp,+ off_screen,+ CGMouseButton::Left,+ )+ .map_err(|_| anyhow::anyhow!("primer up event creation failed"))?;stamp(&primer_up, off_local, 1, 2);post(&primer_up);// ≥1 frame so Chromium sees primer + target as separate gestures, not run-on.@@ -270,15 +332,23 @@ pub fn click_at_xy_chromium(let click_state = pair_index as i64;let down = CGEvent::new_mouse_event(- source.clone(), CGEventType::LeftMouseDown, target, CGMouseButton::Left,- ).map_err(|_| anyhow::anyhow!("target down event creation failed"))?;+ source.clone(),+ CGEventType::LeftMouseDown,+ target,+ CGMouseButton::Left,+ )+ .map_err(|_| anyhow::anyhow!("target down event creation failed"))?;stamp(&down, win_local, click_state, 3);post(&down);std::thread::sleep(std::time::Duration::from_millis(1));let up = CGEvent::new_mouse_event(- source.clone(), CGEventType::LeftMouseUp, target, CGMouseButton::Left,- ).map_err(|_| anyhow::anyhow!("target up event creation failed"))?;+ source.clone(),+ CGEventType::LeftMouseUp,+ target,+ CGMouseButton::Left,+ )+ .map_err(|_| anyhow::anyhow!("target up event creation failed"))?;stamp(&up, win_local, click_state, 3);post(&up);@@ -314,7 +384,9 @@ pub fn drag_at_xy(steps: usize,modifiers: &[&str],button: DragButton,+ foreground_release: bool,) -> anyhow::Result<()> {+ use core_graphics::event::CGEventTapLocation;use std::time::{SystemTime, UNIX_EPOCH};let source = CGEventSource::new(CGEventSourceStateID::HIDSystemState)@@ -322,12 +394,24 @@ pub fn drag_at_xy(let flags = parse_modifier_flags(modifiers);let (cg_button, down_type, dragged_type, up_type) = match button {- DragButton::Left => (CGMouseButton::Left, CGEventType::LeftMouseDown,- CGEventType::LeftMouseDragged, CGEventType::LeftMouseUp),- DragButton::Right => (CGMouseButton::Right, CGEventType::RightMouseDown,- CGEventType::RightMouseDragged, CGEventType::RightMouseUp),- DragButton::Middle => (CGMouseButton::Center, CGEventType::OtherMouseDown,- CGEventType::OtherMouseDragged, CGEventType::OtherMouseUp),+ DragButton::Left => (+ CGMouseButton::Left,+ CGEventType::LeftMouseDown,+ CGEventType::LeftMouseDragged,+ CGEventType::LeftMouseUp,+ ),+ DragButton::Right => (+ CGMouseButton::Right,+ CGEventType::RightMouseDown,+ CGEventType::RightMouseDragged,+ CGEventType::RightMouseUp,+ ),+ DragButton::Middle => (+ CGMouseButton::Center,+ CGEventType::OtherMouseDown,+ CGEventType::OtherMouseDragged,+ CGEventType::OtherMouseUp,+ ),};// f3 button number must match the dragged button (0=left, 1=right, 2=middle).let button_number: i64 = match button {@@ -344,14 +428,29 @@ pub fn drag_at_xy(});let steps = steps.max(1);- let step_delay_ms = if steps > 1 { duration_ms / steps as u64 } else { duration_ms };+ let step_delay_ms = if steps > 1 {+ duration_ms / steps as u64+ } else {+ duration_ms+ };// MouseDown at start.let from_pt = CGPoint::new(from_x, from_y);let down = CGEvent::new_mouse_event(source.clone(), down_type, from_pt, cg_button).map_err(|_| anyhow::anyhow!("drag mouseDown failed"))?;- if flags != CGEventFlags::CGEventFlagNull { down.set_flags(flags); }- post_mouse_event(pid, &down, from_local, wid, click_group_id, 1, button_number);+ if flags != CGEventFlags::CGEventFlagNull {+ down.set_flags(flags);+ }+ post_mouse_event(+ pid,+ &down,+ from_local,+ wid,+ click_group_id,+ 1,+ button_number,+ 0,+ );std::thread::sleep(std::time::Duration::from_millis(16));// Interpolated drag steps.@@ -359,29 +458,163 @@ pub fn drag_at_xy(let t = i as f64 / steps as f64;let ix = from_x + (to_x - from_x) * t;let iy = from_y + (to_y - from_y) * t;- let il = from_local.zip(to_local).map(|((fx, fy), (tx, ty))| {- (fx + (tx - fx) * t, fy + (ty - fy) * t)- });+ let il = from_local+ .zip(to_local)+ .map(|((fx, fy), (tx, ty))| (fx + (tx - fx) * t, fy + (ty - fy) * t));let drag_pt = CGPoint::new(ix, iy);let drag = CGEvent::new_mouse_event(source.clone(), dragged_type, drag_pt, cg_button).map_err(|_| anyhow::anyhow!("drag mouseDragged failed"))?;- if flags != CGEventFlags::CGEventFlagNull { drag.set_flags(flags); }- post_mouse_event(pid, &drag, il, wid, click_group_id, 1, button_number);+ if flags != CGEventFlags::CGEventFlagNull {+ drag.set_flags(flags);+ }+ post_mouse_event(pid, &drag, il, wid, click_group_id, 1, button_number, 0);if step_delay_ms > 0 {std::thread::sleep(std::time::Duration::from_millis(step_delay_ms));}}// MouseUp at end.+ // Give Chromium one run-loop turn to process the final dragged event at+ // the drop point before releasing pointer capture.+ std::thread::sleep(std::time::Duration::from_millis(50));let to_pt = CGPoint::new(to_x, to_y);let up = CGEvent::new_mouse_event(source.clone(), up_type, to_pt, cg_button).map_err(|_| anyhow::anyhow!("drag mouseUp failed"))?;- if flags != CGEventFlags::CGEventFlagNull { up.set_flags(flags); }- post_mouse_event(pid, &up, to_local, wid, click_group_id, 1, button_number);+ if flags != CGEventFlags::CGEventFlagNull {+ up.set_flags(flags);+ }+ post_mouse_event(pid, &up, to_local, wid, click_group_id, 1, button_number, 0);+ if foreground_release {+ // A frontmost Chromium surface can consume PID-routed down/move+ // events yet filter the synthetic release. Re-post only the release+ // through the HID tap while the foreground assist still holds focus.+ up.post(CGEventTapLocation::HID);+ }+ // Chromium may process the final pointerup on the next run-loop turn. In+ // the foreground rung the caller restores the previous app immediately+ // after this function returns, so let the target consume the release and+ // complete pointer capture before that restore.+ std::thread::sleep(std::time::Duration::from_millis(100));Ok(())}+/// Foreground drag through the global HID event tap.+///+/// A frontmost Chromium/WebKit surface expects a real HID-origin gesture for+/// pointer capture and drag tracking. PID-routed `post_to_pid` events are+/// suitable for background delivery, but they can be silently filtered by the+/// renderer even when the target is frontmost.+pub fn drag_at_xy_foreground(+ from_x: f64,+ from_y: f64,+ to_x: f64,+ to_y: f64,+ duration_ms: u64,+ steps: usize,+ modifiers: &[&str],+ button: DragButton,+) -> anyhow::Result<()> {+ use core_graphics::display::CGDisplay;+ use core_graphics::event::CGEventTapLocation;++ let source = CGEventSource::new(CGEventSourceStateID::HIDSystemState)+ .map_err(|_| anyhow::anyhow!("CGEventSource::new failed"))?;+ let flags = parse_modifier_flags(modifiers);+ let (cg_button, down_type, dragged_type, up_type) = match button {+ DragButton::Left => (+ CGMouseButton::Left,+ CGEventType::LeftMouseDown,+ CGEventType::LeftMouseDragged,+ CGEventType::LeftMouseUp,+ ),+ DragButton::Right => (+ CGMouseButton::Right,+ CGEventType::RightMouseDown,+ CGEventType::RightMouseDragged,+ CGEventType::RightMouseUp,+ ),+ DragButton::Middle => (+ CGMouseButton::Center,+ CGEventType::OtherMouseDown,+ CGEventType::OtherMouseDragged,+ CGEventType::OtherMouseUp,+ ),+ };+ let steps = steps.max(1);+ let step_delay_ms = if steps > 1 {+ duration_ms / steps as u64+ } else {+ duration_ms+ };++ let post = |event: &CGEvent| event.post(CGEventTapLocation::HID);++ // Keep WindowServer's hardware cursor and event stream coupled. AppKit+ // hit-tests some pointer-capture surfaces against the actual cursor even+ // when the HID event carries an explicit location.+ let _ = CGDisplay::warp_mouse_cursor_position(CGPoint::new(from_x, from_y));+ unsafe { CGAssociateMouseAndMouseCursorPosition(true) };+ std::thread::sleep(std::time::Duration::from_millis(40));++ // Prime the renderer's tracking state with a genuine HID mouse move.+ if let Ok(move_event) = CGEvent::new_mouse_event(+ source.clone(),+ CGEventType::MouseMoved,+ CGPoint::new(from_x, from_y),+ cg_button,+ ) {+ post(&move_event);+ }+ std::thread::sleep(std::time::Duration::from_millis(30));++ let down = CGEvent::new_mouse_event(+ source.clone(),+ down_type,+ CGPoint::new(from_x, from_y),+ cg_button,+ )+ .map_err(|_| anyhow::anyhow!("foreground drag mouseDown failed"))?;+ if flags != CGEventFlags::CGEventFlagNull {+ down.set_flags(flags);+ }+ down.set_integer_value_field(core_graphics::event::EventField::MOUSE_EVENT_CLICK_STATE, 1);+ down.post(CGEventTapLocation::HID);+ std::thread::sleep(std::time::Duration::from_millis(16));++ for i in 1..=steps {+ let t = i as f64 / steps as f64;+ let event = CGEvent::new_mouse_event(+ source.clone(),+ dragged_type,+ CGPoint::new(from_x + (to_x - from_x) * t, from_y + (to_y - from_y) * t),+ cg_button,+ )+ .map_err(|_| anyhow::anyhow!("foreground drag mouseDragged failed"))?;+ if flags != CGEventFlags::CGEventFlagNull {+ event.set_flags(flags);+ }+ event.set_integer_value_field(core_graphics::event::EventField::MOUSE_EVENT_CLICK_STATE, 1);+ post(&event);+ if step_delay_ms > 0 {+ std::thread::sleep(std::time::Duration::from_millis(step_delay_ms));+ }+ }++ let up = CGEvent::new_mouse_event(source, up_type, CGPoint::new(to_x, to_y), cg_button)+ .map_err(|_| anyhow::anyhow!("foreground drag mouseUp failed"))?;+ if flags != CGEventFlags::CGEventFlagNull {+ up.set_flags(flags);+ }+ up.set_integer_value_field(core_graphics::event::EventField::MOUSE_EVENT_CLICK_STATE, 1);+ post(&up);+ // The foreground wrapper restores the previous app immediately after this+ // function returns. Let the target's run loop consume the queued HID+ // gesture, including pointer-capture release, before that restore happens.+ std::thread::sleep(std::time::Duration::from_millis(100));+ Ok(())+}+/// Mouse button for drag gestures.#[derive(Clone, Copy, Debug)]pub enum DragButton {@@ -403,8 +636,10 @@ pub fn middle_click_at_xy(pid: i32, x: f64, y: f64, modifiers: &[&str]) -> anyho/// Like `middle_click_at_xy` but stamps the window-local `(wx, wy)` point.pub fn middle_click_at_xy_with_window_local(pid: i32,- x: f64, y: f64,- wx: f64, wy: f64,+ x: f64,+ y: f64,+ wx: f64,+ wy: f64,modifiers: &[&str],) -> anyhow::Result<()> {middle_click_at_xy_inner(pid, x, y, Some((wx, wy)), modifiers)@@ -412,7 +647,8 @@ pub fn middle_click_at_xy_with_window_local(fn middle_click_at_xy_inner(pid: i32,- x: f64, y: f64,+ x: f64,+ y: f64,window_local: Option<(f64, f64)>,modifiers: &[&str],) -> anyhow::Result<()> {@@ -426,11 +662,12 @@ fn middle_click_at_xy_inner(CGEventType::OtherMouseDown,point,CGMouseButton::Center,- ).map_err(|_| anyhow::anyhow!("middle mouse down failed"))?;+ )+ .map_err(|_| anyhow::anyhow!("middle mouse down failed"))?;if flags != CGEventFlags::CGEventFlagNull {down.set_flags(flags);}- post_mouse_event(pid, &down, window_local, None, None, 1, 2);+ post_mouse_event(pid, &down, window_local, None, None, 1, 2, 3);std::thread::sleep(std::time::Duration::from_millis(16));let up = CGEvent::new_mouse_event(@@ -438,11 +675,12 @@ fn middle_click_at_xy_inner(CGEventType::OtherMouseUp,point,CGMouseButton::Center,- ).map_err(|_| anyhow::anyhow!("middle mouse up failed"))?;+ )+ .map_err(|_| anyhow::anyhow!("middle mouse up failed"))?;if flags != CGEventFlags::CGEventFlagNull {up.set_flags(flags);}- post_mouse_event(pid, &up, window_local, None, None, 1, 2);+ post_mouse_event(pid, &up, window_local, None, None, 1, 2, 3);Ok(())}@@ -463,8 +701,10 @@ pub fn right_click_at_xy(pid: i32, x: f64, y: f64, modifiers: &[&str]) -> anyhow/// already threaded `wid`; right-click did not, which is why it broke.pub fn right_click_at_xy_with_window_local(pid: i32,- x: f64, y: f64,- wx: f64, wy: f64,+ x: f64,+ y: f64,+ wx: f64,+ wy: f64,wid: u32,modifiers: &[&str],) -> anyhow::Result<()> {@@ -473,7 +713,8 @@ pub fn right_click_at_xy_with_window_local(fn right_click_at_xy_inner(pid: i32,- x: f64, y: f64,+ x: f64,+ y: f64,window_local: Option<(f64, f64)>,wid: Option<u32>,modifiers: &[&str],@@ -502,13 +743,14 @@ fn right_click_at_xy_inner(CGEventType::RightMouseDown,point,CGMouseButton::Right,- ).map_err(|_| anyhow::anyhow!("right mouse down failed"))?;+ )+ .map_err(|_| anyhow::anyhow!("right mouse down failed"))?;if flags != CGEventFlags::CGEventFlagNull {down.set_flags(flags);}// button_number = 1 (right). Stamping 0 here routes the event as a left// button-number on the receiving side even though the type is rightMouseDown.- post_mouse_event(pid, &down, window_local, wid, click_group_id, 1, 1);+ post_mouse_event(pid, &down, window_local, wid, click_group_id, 1, 1, 3);std::thread::sleep(std::time::Duration::from_millis(28));let up = CGEvent::new_mouse_event(@@ -516,11 +758,12 @@ fn right_click_at_xy_inner(CGEventType::RightMouseUp,point,CGMouseButton::Right,- ).map_err(|_| anyhow::anyhow!("right mouse up failed"))?;+ )+ .map_err(|_| anyhow::anyhow!("right mouse up failed"))?;if flags != CGEventFlags::CGEventFlagNull {up.set_flags(flags);}- post_mouse_event(pid, &up, window_local, wid, click_group_id, 1, 1);+ post_mouse_event(pid, &up, window_local, wid, click_group_id, 1, 1, 3);Ok(())}@@ -557,6 +800,7 @@ fn post_mouse_event(click_group_id: Option<i64>,click_state: i64,button_number: i64,+ subtype: i64,) {let event_ptr = event.as_ptr() as *mut std::ffi::c_void;@@ -568,14 +812,16 @@ fn post_mouse_event(// Chromium / AppKit window-routing fields — stamp when window_id is known.if let (Some(wid), Some(cgid)) = (wid, click_group_id) {let window_id = wid as i64;- let set = |f: u32, v: i64| { crate::input::skylight::set_integer_field(event_ptr, f, v); };- set(1, click_state); // kCGMouseEventClickState- set(3, button_number); // kCGMouseEventButtonNumber (0=left, 1=right, 2=middle)- set(7, 3); // kCGMouseEventSubtype (NSEventSubtypeTouch)- set(51, window_id); // windowNumber- set(58, cgid); // click-group ID (gesture coalescing)- set(91, window_id); // kCGMouseEventWindowUnderMousePointer- set(92, window_id); // kCGMouseEventWindowUnderMousePointerThatCanHandleThisEvent+ let set = |f: u32, v: i64| {+ crate::input::skylight::set_integer_field(event_ptr, f, v);+ };+ set(1, click_state); // kCGMouseEventClickState+ set(3, button_number); // kCGMouseEventButtonNumber (0=left, 1=right, 2=middle)+ set(7, subtype); // kCGMouseEventSubtype (touch for clicks, normal for drags)+ set(51, window_id); // windowNumber+ set(58, cgid); // click-group ID (gesture coalescing)+ set(91, window_id); // kCGMouseEventWindowUnderMousePointer+ set(92, window_id); // kCGMouseEventWindowUnderMousePointerThatCanHandleThisEvent}// Always stamp f40 = target pid (Chromium synthetic-event filter).@@ -615,14 +861,14 @@ fn post_mouse_moved_primer(point,CGMouseButton::Left,) {- post_mouse_event(pid, &mv, window_local, wid, click_group_id, 0, 0);+ post_mouse_event(pid, &mv, window_local, wid, click_group_id, 0, 0, 3);}}-/// Synthesize a **pixel-unit mouse-wheel** scroll at `(screen_x, screen_y)`,+/// Synthesize a targeted mouse-wheel scroll at `(screen_x, screen_y)`,/// posted to `pid`.///-/// This is the pixel-wheel scroll path. Unlike the keystroke path (PageDown /+/// This is the targeted wheel path. Unlike the keystroke path (PageDown //// arrow keys), which only ever drives the *focused* / page scroller, a real/// wheel event is hit-tested by the renderer at the cursor point: whatever/// element sits under `(screen_x, screen_y)` receives the scroll. That is the@@ -630,11 +876,11 @@ fn post_mouse_moved_primer(/// therefore can never take keyboard focus (verified no-op via keystrokes on/// WKWebView's inner `scroll-tall` and WebView2).///-/// `CGEventCreateScrollWheelEvent2` builds the event in `kCGScrollEventUnitPixel`-/// units; we then anchor it at the target point with `CGEventSetLocation` so the-/// renderer routes it correctly, and stamp the same background-delivery fields-/// the click primitives use (window-local point + f40 pid filter + window-routing-/// fields) so it reaches backgrounded Chromium/Catalyst/WKWebView targets.+/// `CGEventCreateScrollWheelEvent2` builds a line-unit event; we then anchor it+/// at the target point with `CGEventSetLocation` so the renderer routes it+/// correctly, and stamp the same background-delivery fields the click+/// primitives use (window-local point + f40 pid filter + window-routing fields)+/// so it reaches backgrounded Chromium/Catalyst/WKWebView targets.////// Sign convention (macOS): a POSITIVE `delta_y_per_tick` scrolls the content/// toward the top (reveals content ABOVE); NEGATIVE reveals content BELOW.@@ -651,7 +897,8 @@ fn post_mouse_moved_primer(/// identical in spirit to `post_mouse_event`.pub fn scroll_wheel_at_xy(pid: i32,- screen_x: f64, screen_y: f64,+ screen_x: f64,+ screen_y: f64,window_local: Option<(f64, f64)>,wid: Option<u32>,delta_y_per_tick: i32,@@ -660,21 +907,45 @@ pub fn scroll_wheel_at_xy() -> anyhow::Result<()> {use core_graphics::event::ScrollEventUnit;+ // Prime AppKit/WebKit's tracking state at the target before the wheel+ // gesture. Background windows can retain a stale hit-test location; a+ // nested overflow scroller then receives the wheel nowhere even though+ // the event is successfully posted to the target pid.+ let primer_source = CGEventSource::new(CGEventSourceStateID::HIDSystemState)+ .map_err(|_| anyhow::anyhow!("CGEventSource::new failed"))?;+ post_mouse_moved_primer(+ pid,+ &primer_source,+ CGPoint::new(screen_x, screen_y),+ window_local,+ wid,+ Some(+ std::time::SystemTime::now()+ .duration_since(std::time::UNIX_EPOCH)+ .unwrap_or_default()+ .subsec_nanos() as i64,+ ),+ );+ std::thread::sleep(std::time::Duration::from_millis(12));+for _ in 0..ticks.max(1) {// Fresh source per event, matching the click primitives.let source = CGEventSource::new(CGEventSourceStateID::HIDSystemState).map_err(|_| anyhow::anyhow!("CGEventSource::new failed"))?;// wheel_count = 2 → both axes carried (vertical = wheel1/axis-1,- // horizontal = wheel2/axis-2). PIXEL units so the deltas are device- // pixels, matching a trackpad/precise wheel rather than coarse notches.+ // horizontal = wheel2/axis-2). Convert the driver's pixel-tuned step+ // into a bounded line delta for the event API.+ let wheel_y = (delta_y_per_tick / 120).clamp(-10, 10);+ let wheel_x = (delta_x_per_tick / 120).clamp(-10, 10);let event = CGEvent::new_scroll_event(source,- ScrollEventUnit::PIXEL,+ ScrollEventUnit::LINE,2,- delta_y_per_tick,- delta_x_per_tick,+ wheel_y,+ wheel_x,0,- ).map_err(|_| anyhow::anyhow!("CGEvent::new_scroll_event failed"))?;+ )+ .map_err(|_| anyhow::anyhow!("CGEvent::new_scroll_event failed"))?;let event_ptr = event.as_ptr() as *mut std::ffi::c_void;@@ -688,7 +959,9 @@ pub fn scroll_wheel_at_xy(}if let Some(wid) = wid {let window_id = wid as i64;- let set = |f: u32, v: i64| { crate::input::skylight::set_integer_field(event_ptr, f, v); };+ let set = |f: u32, v: i64| {+ crate::input::skylight::set_integer_field(event_ptr, f, v);+ };set(51, window_id); // windowNumberset(91, window_id); // kCGMouseEventWindowUnderMousePointerset(92, window_id); // ...ThatCanHandleThisEventdiff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/click.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/click.rsindex 89e9566704..f18b8a36c8 100644--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/click.rs+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/click.rs@@ -14,15 +14,17 @@//! to full-window space using the most recent `zoom` context stored per-pid.use async_trait::async_trait;-use cua_driver_core::{protocol::ToolResult, tool::{Tool, ToolDef}};+use cua_driver_core::{+ protocol::ToolResult,+ tool::{Tool, ToolDef},+};use serde_json::Value;use std::sync::Arc;+use crate::apps;use crate::ax::bindings::{- copy_action_names, copy_children, copy_string_attr, element_screen_rect,- AXUIElementRef,+ copy_action_names, copy_children, copy_string_attr, element_screen_rect, AXUIElementRef,};-use crate::apps;use crate::focus_guard;use crate::window_change_detector::WindowChangeDetector;use core_foundation::base::CFRelease;@@ -34,7 +36,9 @@ pub struct ClickTool {}impl ClickTool {- pub fn new(state: Arc<ToolState>) -> Self { Self { state } }+ pub fn new(state: Arc<ToolState>) -> Self {+ Self { state }+ }}static DEF: std::sync::OnceLock<ToolDef> = std::sync::OnceLock::new();@@ -126,7 +130,9 @@ fn def() -> &'static ToolDef {#[async_trait]impl Tool for ClickTool {- fn def(&self) -> &ToolDef { def() }+ fn def(&self) -> &ToolDef {+ def()+ }async fn invoke(&self, args: Value) -> ToolResult {use cua_driver_core::tool_args::ArgsExt;@@ -160,8 +166,14 @@ impl Tool for ClickTool {"suggestion": "pass scope=\"desktop\"",}));}- let sx_shot = args.opt_f64("x").or_else(|| args.opt_i64("x").map(|i| i as f64)).unwrap_or(0.0);- let sy_shot = args.opt_f64("y").or_else(|| args.opt_i64("y").map(|i| i as f64)).unwrap_or(0.0);+ let sx_shot = args+ .opt_f64("x")+ .or_else(|| args.opt_i64("x").map(|i| i as f64))+ .unwrap_or(0.0);+ let sy_shot = args+ .opt_f64("y")+ .or_else(|| args.opt_i64("y").map(|i| i as f64))+ .unwrap_or(0.0);// ── Desktop-screenshot pixels → logical screen points ──────────────// The vision invariant: the pixel an agent reads off the screenshot it// was handed is the pixel that gets clicked. `get_desktop_state`@@ -178,7 +190,8 @@ impl Tool for ClickTool {// CGDisplayPixelsWide under-reports the backing scale (it returns the// scaled-mode point width on some Retina configs → a bogus 1.0).let desktop_ratio = tokio::task::spawn_blocking(|| {- let logical_w = super::get_screen_size::main_screen_size().map(|(w, _, _)| w as f64);+ let logical_w =+ super::get_screen_size::main_screen_size().map(|(w, _, _)| w as f64);let shot_w = crate::capture::screenshot_display_bytes().ok().and_then(|png| crate::capture::png_dimensions(&png).ok())@@ -207,7 +220,9 @@ impl Tool for ClickTool {// Glide the session's agent cursor to the screen point for visibility.let cursor_key = super::cursor_tools::resolve_cursor_key(&args);crate::cursor::overlay::animate_cursor_to(cursor_key.clone(), sx, sy).await;- self.state.cursor_registry.update_position(&cursor_key, sx, sy);+ self.state+ .cursor_registry+ .update_position(&cursor_key, sx, sy);// Resolve the frontmost on-screen window under the point (the macOS// peer of Windows' WindowFromPoint). When found, click THAT pid via@@ -231,14 +246,16 @@ impl Tool for ClickTool {// (Ascending picked the BACKMOST occluded window — a real miss when// windows overlap, e.g. resolving a click to a buried app.)wins.sort_by(|a, b| b.z_index.cmp(&a.z_index)); // front-to-back- wins.into_iter().find(|w| {- w.layer == 0- && w.pid != own_pid- && sx >= w.bounds.x- && sx < w.bounds.x + w.bounds.width- && sy >= w.bounds.y- && sy < w.bounds.y + w.bounds.height- }).map(|w| (w.pid, w.window_id, w.bounds.x, w.bounds.y))+ wins.into_iter()+ .find(|w| {+ w.layer == 0+ && w.pid != own_pid+ && sx >= w.bounds.x+ && sx < w.bounds.x + w.bounds.width+ && sy >= w.bounds.y+ && sy < w.bounds.y + w.bounds.height+ })+ .map(|w| (w.pid, w.window_id, w.bounds.x, w.bounds.y))};let btn = button.clone();let result = tokio::task::spawn_blocking(move || -> anyhow::Result<Option<i32>> {@@ -252,13 +269,31 @@ impl Tool for ClickTool {// as the pixel path); `count` only repeats on the left path.match btn.as_str() {"right" => crate::input::mouse::right_click_at_xy_with_window_local(- pid, sx, sy, wx, wy, wid, &[],+ pid,+ sx,+ sy,+ wx,+ wy,+ wid,+ &[],)?,"middle" => crate::input::mouse::middle_click_at_xy_with_window_local(- pid, sx, sy, wx, wy, &[],+ pid,+ sx,+ sy,+ wx,+ wy,+ &[],)?,_ => crate::input::mouse::click_at_xy_with_window_local(- pid, sx, sy, wx, wy, wid, count, &[],+ pid,+ sx,+ sy,+ wx,+ wy,+ wid,+ count,+ &[],)?,}Ok(Some(pid))@@ -293,7 +328,10 @@ impl Tool for ClickTool {};}- let pid = match args.require_i32("pid") { Ok(v) => v, Err(e) => return e };+ let pid = match args.require_i32("pid") {+ Ok(v) => v,+ Err(e) => return e,+ };// Resolve this action's cursor key so its click-pulse / glide land on// the calling session's cursor, not the shared "default" one.let cursor_key = super::cursor_tools::resolve_cursor_key(&args);@@ -303,7 +341,7 @@ impl Tool for ClickTool {// stale token returns an explicit error instead of silently// falling back to the integer (Surface 6 hard constraint).let element_token_arg = args.opt_str("element_token");- let window_id_arg = args.opt_u64("window_id").map(|v| v as u32);+ let window_id_arg = args.opt_u64("window_id").map(|v| v as u32);let element_index_arg = args.opt_u64("element_index").map(|v| v as usize);let resolved = match cua_driver_core::element_token::resolve_element_args(pid,@@ -318,18 +356,24 @@ impl Tool for ClickTool {let (element_index, window_id, _via_token) = match resolved {cua_driver_core::element_token::ResolvedElement::None => (None, window_id_arg, false),cua_driver_core::element_token::ResolvedElement::Element {- window_id: wid, element_index: idx, via_token,+ window_id: wid,+ element_index: idx,+ via_token,} => (Some(idx), wid, via_token),};- let x = args.opt_f64("x").or_else(|| args.opt_i64("x").map(|i| i as f64));- let y = args.opt_f64("y").or_else(|| args.opt_i64("y").map(|i| i as f64));- let action = args.str_or("action", "press");+ let x = args+ .opt_f64("x")+ .or_else(|| args.opt_i64("x").map(|i| i as f64));+ let y = args+ .opt_f64("y")+ .or_else(|| args.opt_i64("y").map(|i| i as f64));+ let action = args.str_or("action", "press");// Surface 5: optional `button` arg, default "left" preserves legacy behaviour.// Pixel path: routes to left/right/middle CGEvent primitives.// AX path: "right" delegates to AXShowMenu (same surface as right_click);// "middle" has no AX equivalent and falls back to a pixel middle-click// at the element's screen-space center.- let button_str = args.str_or("button", "left").to_lowercase();+ let button_str = args.str_or("button", "left").to_lowercase();// delivery_mode: per-call ladder rung. foreground only applies to the// pixel path and needs a window_id to front (else it degrades to// background). A click is never driver-verifiable either way.@@ -341,9 +385,13 @@ impl Tool for ClickTool {"click: unknown button \"{button_str}\" — expected one of left, right, middle."));}- let button_str = if button_str.is_empty() { "left".to_string() } else { button_str };- let count = args.u64_or("count", 1) as usize;- let from_zoom = args.bool_or("from_zoom", false);+ let button_str = if button_str.is_empty() {+ "left".to_string()+ } else {+ button_str+ };+ let count = args.u64_or("count", 1) as usize;+ let from_zoom = args.bool_or("from_zoom", false);let debug_image_out = args.opt_str("debug_image_out");let modifiers: Vec<String> = args.str_array("modifier");@@ -355,10 +403,12 @@ impl Tool for ClickTool {// guard lives to the end of this method, past the AX action below.let element_guard = match self.state.element_cache.get_element_retained(pid, wid, idx) {Some(e) => e,- None => return ToolResult::error(format!(- "Element index {idx} not found in cache for pid={pid} window_id={wid}. \+ None => {+ return ToolResult::error(format!(+ "Element index {idx} not found in cache for pid={pid} window_id={wid}. \Call get_window_state first."- )),+ ))+ }};let element_ptr = element_guard.as_ptr();@@ -376,7 +426,10 @@ impl Tool for ClickTool {let center_ptr = element_ptr;let center = tokio::task::spawn_blocking(move || unsafe {crate::ax::bindings::element_screen_center(center_ptr as AXUIElementRef)- }).await.ok().flatten();+ })+ .await+ .ok()+ .flatten();// Surface 5: button=middle on the AX path has no AX equivalent.// Fall back to a pixel middle-click at the element's screen-space center@@ -386,23 +439,28 @@ impl Tool for ClickTool {if button_str == "middle" {let (cx, cy) = match center {Some(c) => c,- None => return ToolResult::error(- "click(button=middle) on element_index: could not resolve element \- center for the pixel-middle-click fallback. Pass x, y directly."- ),+ None => {+ return ToolResult::error(+ "click(button=middle) on element_index: could not resolve element \+ center for the pixel-middle-click fallback. Pass x, y directly.",+ )+ }};crate::cursor::overlay::send_command(cursor_key.clone(),cursor_overlay::OverlayCommand::PinAbove(wid as u64),);crate::cursor::overlay::animate_cursor_to(cursor_key.clone(), cx, cy).await;- self.state.cursor_registry.update_position(&cursor_key, cx, cy);+ self.state+ .cursor_registry+ .update_position(&cursor_key, cx, cy);let mods_owned = modifiers.clone();let result = tokio::task::spawn_blocking(move || {let m: Vec<&str> = mods_owned.iter().map(String::as_str).collect();crate::input::mouse::middle_click_at_xy(pid, cx, cy, &m)- }).await;+ })+ .await;return match result {Ok(Ok(())) => ToolResult::text(format!("✅ Posted middle-click to pid {pid} at element [{idx}] center \@@ -424,7 +482,9 @@ impl Tool for ClickTool {// Keep the registry in sync with the overlay so// get_agent_cursor_state reports a truthful position even when// the click was dispatched via the AX path (no pixel coords).- self.state.cursor_registry.update_position(&cursor_key, cx, cy);+ self.state+ .cursor_registry+ .update_position(&cursor_key, cx, cy);}// ── Focus-suppression wrap (Swift WindowChangeDetector + FocusGuard) ──@@ -494,7 +554,7 @@ impl Tool for ClickTool {ToolResult::text(msg).with_structured(structured)}Ok(Err(e)) => ToolResult::error(format!("AX action failed: {e}")),- Err(e) => ToolResult::error(format!("Task error: {e}")),+ Err(e) => ToolResult::error(format!("Task error: {e}")),}} else if let (Some(mut cx), Some(mut cy)) = (x, y) {// ── Pixel path ─────────────────────────────────────────────────@@ -506,35 +566,37 @@ impl Tool for ClickTool {if from_zoom {return ToolResult::error("debug_image_out is incompatible with from_zoom — \- received (x, y) would be in zoom-crop space, not window-local."+ received (x, y) would be in zoom-crop space, not window-local.",);}match window_id {- None => return ToolResult::error(- "debug_image_out requires window_id."- ),+ None => return ToolResult::error("debug_image_out requires window_id."),Some(wid) => {// Session-effective max dimension so debug_image_out// matches the resize the calling session sees in// get_window_state (precedence: session override > global).- let max_dim = self.state.session_config- .effective_max_image_dimension(- args.opt_str("_session_id").as_deref(),- &self.state.config.read().unwrap(),- );+ let max_dim = self.state.session_config.effective_max_image_dimension(+ args.opt_str("_session_id").as_deref(),+ &self.state.config.read().unwrap(),+ );let dbg_path_c = dbg_path.clone();let dbg_result = tokio::task::spawn_blocking(move || {let png = crate::capture::screenshot_window_bytes(wid)?;let png = crate::capture::resize_png_if_needed(&png, max_dim)?;crate::capture::write_crosshair_png(&png, cx, cy, &dbg_path_c)- }).await;+ })+ .await;match dbg_result {- Err(e) => return ToolResult::error(format!(- "debug_image_out task failed: {e}. Not dispatching click."- )),- Ok(Err(e)) => return ToolResult::error(format!(- "debug_image_out write failed: {e}. Not dispatching click."- )),+ Err(e) => {+ return ToolResult::error(format!(+ "debug_image_out task failed: {e}. Not dispatching click."+ ))+ }+ Ok(Err(e)) => {+ return ToolResult::error(format!(+ "debug_image_out write failed: {e}. Not dispatching click."+ ))+ }Ok(Ok(())) => {}}}@@ -548,9 +610,11 @@ impl Tool for ClickTool {cx = wx;cy = wy;}- None => return ToolResult::error(format!(- "from_zoom=true but no zoom context for pid {pid}. Call zoom first."- )),+ None => {+ return ToolResult::error(format!(+ "from_zoom=true but no zoom context for pid {pid}. Call zoom first."+ ))+ }}} else if let Some(ratio) = self.state.resize_registry.ratio(pid) {// Coordinates are in the downscaled image space; scale back to native pixels.@@ -579,14 +643,27 @@ impl Tool for ClickTool {// against the logical bounds.if let Ok(png) = crate::capture::screenshot_window_bytes(wid) {if png.len() >= 24 {- let pw = u32::from_be_bytes([png[16], png[17], png[18], png[19]]) as f64;+ let pw =+ u32::from_be_bytes([png[16], png[17], png[18], png[19]]) as f64;let lw = b.width;- if lw > 0.0 && pw > lw { pw / lw } else { 1.0 }- } else { 1.0 }- } else { 1.0 }- } else { 1.0 };+ if lw > 0.0 && pw > lw {+ pw / lw+ } else {+ 1.0+ }+ } else {+ 1.0+ }+ } else {+ 1.0+ }+ } else {+ 1.0+ };(bounds, scale)- }).await.unwrap_or((None, 1.0));+ })+ .await+ .unwrap_or((None, 1.0));if let (Some(b), scale) = result {let wx = cx / scale;let wy = cy / scale;@@ -612,11 +689,16 @@ impl Tool for ClickTool {// arrive — mirrors Swift's `AgentCursor.shared.animateAndWait(to:)`.crate::cursor::overlay::animate_cursor_to(cursor_key.clone(), screen_x, screen_y).await;// Keep the registry in sync with the overlay (see AX path above).- self.state.cursor_registry.update_position(&cursor_key, screen_x, screen_y);+ self.state+ .cursor_registry+ .update_position(&cursor_key, screen_x, screen_y);// Show click-pulse on the agent cursor overlay.crate::cursor::overlay::send_command(cursor_key.clone(),- cursor_overlay::OverlayCommand::ClickPulse { x: screen_x, y: screen_y },+ cursor_overlay::OverlayCommand::ClickPulse {+ x: screen_x,+ y: screen_y,+ },);// ── Focus-suppression wrap (Swift WindowChangeDetector + FocusGuard) ──@@ -636,6 +718,7 @@ impl Tool for ClickTool {// clicks. Needs window_id to front; without one it degrades to// background (and is labelled as such).let fg = delivery_mode.is_foreground() && window_id.is_some();+let result = focus_guard::with_focus_suppressed(Some(pid),prior_front,@@ -668,7 +751,14 @@ impl Tool for ClickTool {// and Chromium-specific fields (f40, f51, f58, f91, f92) onto events// for better backgrounded-target delivery.if let Some(wid) = window_id {- return crate::input::mouse::click_at_xy_with_window_local(+ if fg {+ return crate::input::mouse::click_at_xy_with_window_local(+ pid, screen_x, screen_y,+ win_local_x, win_local_y,+ wid, count, &m,+ );+ }+ return crate::input::mouse::click_at_xy_chromium(pid, screen_x, screen_y,win_local_x, win_local_y,wid, count, &m,@@ -696,9 +786,9 @@ impl Tool for ClickTool {let changes = snapshot.detect_async().await;let button_label = match button_str.as_str() {- "right" => "right-click",+ "right" => "right-click","middle" => "middle-click",- _ => "click",+ _ => "click",};match result {Ok(Ok(fronted)) => {@@ -718,11 +808,11 @@ impl Tool for ClickTool {.with_structured(serde_json::json!({ "path": path, "verified": false, "effect": "unverifiable" }))}Ok(Err(e)) => ToolResult::error(format!("{button_label} failed: {e}")),- Err(e) => ToolResult::error(format!("Task error: {e}")),+ Err(e) => ToolResult::error(format!("Task error: {e}")),}} else {ToolResult::error(- "Provide either (element_index + window_id) or (x + y). pid is always required."+ "Provide either (element_index + window_id) or (x + y). pid is always required.",)}}@@ -757,7 +847,7 @@ fn perform_ax_click(anyhow::bail!("AXUIElementPerformAction({ax_action}) returned {err}");}- let role = unsafe { copy_string_attr(element, "AXRole") }.unwrap_or_default();+ let role = unsafe { copy_string_attr(element, "AXRole") }.unwrap_or_default();let title = unsafe { copy_string_attr(element, "AXTitle") }.unwrap_or_default();let mut summary = format!("✅ Performed {ax_action} on [{idx}] {role} \"{title}\".");@@ -766,11 +856,14 @@ fn perform_ax_click(if role == "AXPopUpButton" {let children = unsafe { copy_children(element) };if !children.is_empty() {- let options: Vec<String> = children.iter()+ let options: Vec<String> = children+ .iter().filter_map(|&child| {let t = unsafe { copy_string_attr(child, "AXTitle") }.unwrap_or_default();let v = unsafe { copy_string_attr(child, "AXValue") }.unwrap_or_default();- if t.is_empty() && v.is_empty() { return None; }+ if t.is_empty() && v.is_empty() {+ return None;+ }Some(if v.is_empty() || v == t {format!("\"{t}\"")} else {@@ -778,7 +871,11 @@ fn perform_ax_click(})}).collect();- for &child in &children { unsafe { CFRelease(child as _); } }+ for &child in &children {+ unsafe {+ CFRelease(child as _);+ }+ }if !options.is_empty() {let opt_list = options.join(", ");@@ -786,7 +883,7 @@ fn perform_ax_click("\n\n⚠️ This is a popup/select button. The native macOS menu closes \immediately when the window is in the background. Do NOT use click \again — instead, use:\n set_value(pid, window_id, element_index, value)\n\- Available options: ["+ Available options: [",);summary.push_str(&opt_list);summary.push(']');@@ -798,7 +895,11 @@ fn perform_ax_click(// machine-readable `suspected_noop` signal returned to the caller.let suspected_noop = !advertised.contains(&ax_action.to_string());if suspected_noop {- let adv_list = if advertised.is_empty() { "none".into() } else { advertised.join(", ") };+ let adv_list = if advertised.is_empty() {+ "none".into()+ } else {+ advertised.join(", ")+ };summary.push_str(&format!("\n⚠️ Element does not advertise {ax_action} (actions: {adv_list}). \Action may have been a no-op."@@ -806,8 +907,8 @@ fn perform_ax_click(}// WebKit DOM focus settle: 800 ms for text inputs (returned to async caller).- let needs_webkit_delay = ax_action == "AXPress"- && (role == "AXTextField" || role == "AXTextArea");+ let needs_webkit_delay =+ ax_action == "AXPress" && (role == "AXTextField" || role == "AXTextArea");// Show focus-rect highlight around the element (matches Swift showFocusRect).// Also move the cursor to the element center so the glide animation plays.@@ -827,20 +928,21 @@ fn perform_ax_click(cursor_overlay::OverlayCommand::ClickPulse { x: cx, y: cy },);}- let _ = pid; let _ = window_id; // used by caller context+ let _ = pid;+ let _ = window_id; // used by caller contextOk((summary, needs_webkit_delay, suspected_noop))}fn map_action(action: &str) -> &'static str {match action.to_lowercase().as_str() {- "press" | "click" => "AXPress",+ "press" | "click" => "AXPress","show_menu" | "right_click" => "AXShowMenu",- "pick" => "AXPick",- "confirm" => "AXConfirm",- "cancel" => "AXCancel",- "open" => "AXOpen",- _ => "AXPress",+ "pick" => "AXPick",+ "confirm" => "AXConfirm",+ "cancel" => "AXCancel",+ "open" => "AXOpen",+ _ => "AXPress",}}@@ -879,9 +981,18 @@ mod tests {fn description_mentions_button_default() {let d = def();let desc = d.description.to_ascii_lowercase();- assert!(desc.contains("button"), "description should mention button arg");- assert!(desc.contains("left"), "description should mention left default");- assert!(desc.contains("middle"), "description should mention middle button");+ assert!(+ desc.contains("button"),+ "description should mention button arg"+ );+ assert!(+ desc.contains("left"),+ "description should mention left default"+ );+ assert!(+ desc.contains("middle"),+ "description should mention middle button"+ );}/// Existing default behaviour preserved: no `button` field on the call →@@ -893,7 +1004,11 @@ mod tests {use cua_driver_core::tool_args::ArgsExt;let args = serde_json::json!({ "pid": 1234 });let button_str_raw = args.str_or("button", "left").to_lowercase();- let resolved = if button_str_raw.is_empty() { "left".to_string() } else { button_str_raw };+ let resolved = if button_str_raw.is_empty() {+ "left".to_string()+ } else {+ button_str_raw+ };assert_eq!(resolved, "left");}diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/drag.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/drag.rsindex 09c157ff51..8deff2249d 100644--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/drag.rs+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/drag.rs@@ -8,22 +8,27 @@//! recent zoom crop context, same as click/double_click.use async_trait::async_trait;-use cua_driver_core::{protocol::ToolResult, tool::{Tool, ToolDef}};+use cua_driver_core::{+ protocol::ToolResult,+ tool::{Tool, ToolDef},+};use serde_json::Value;use std::sync::Arc;+use super::ToolState;use crate::apps;use crate::focus_guard;use crate::input::mouse::DragButton;use crate::window_change_detector::WindowChangeDetector;-use super::ToolState;pub struct DragTool {pub state: Arc<ToolState>,}impl DragTool {- pub fn new(state: Arc<ToolState>) -> Self { Self { state } }+ pub fn new(state: Arc<ToolState>) -> Self {+ Self { state }+ }}static DEF: std::sync::OnceLock<ToolDef> = std::sync::OnceLock::new();@@ -98,11 +103,16 @@ fn def() -> &'static ToolDef {#[async_trait]impl Tool for DragTool {- fn def(&self) -> &ToolDef { def() }+ fn def(&self) -> &ToolDef {+ def()+ }async fn invoke(&self, args: Value) -> ToolResult {use cua_driver_core::tool_args::ArgsExt;- let pid = match args.require_i32("pid") { Ok(v) => v, Err(e) => return e };+ let pid = match args.require_i32("pid") {+ Ok(v) => v,+ Err(e) => return e,+ };// delivery_mode: foreground briefly fronts the window before the// press-drag-release gesture (the explicit last resort for surfaces// that drop background CGEvents), via the same skylight assist click@@ -112,7 +122,8 @@ impl Tool for DragTool {// Coerce integer or float from JSON for coordinate fields.let coerce = |key: &str| -> Option<f64> {- args.opt_f64(key).or_else(|| args.opt_i64(key).map(|i| i as f64))+ args.opt_f64(key)+ .or_else(|| args.opt_i64(key).map(|i| i as f64))};let mut from_x = match coerce("from_x") {@@ -132,20 +143,22 @@ impl Tool for DragTool {None => return ToolResult::error("Missing required parameter: to_y"),};- let window_id = args.opt_u64("window_id").map(|v| v as u32);+ let window_id = args.opt_u64("window_id").map(|v| v as u32);let duration_ms = args.u64_or("duration_ms", 500);- let steps = args.u64_or("steps", 20) as usize;- let from_zoom = args.bool_or("from_zoom", false);- let button_str = args.str_or("button", "left");+ let steps = args.u64_or("steps", 20) as usize;+ let from_zoom = args.bool_or("from_zoom", false);+ let button_str = args.str_or("button", "left");let modifiers: Vec<String> = args.str_array("modifier");let button = match button_str.to_lowercase().as_str() {- "left" => DragButton::Left,- "right" => DragButton::Right,+ "left" => DragButton::Left,+ "right" => DragButton::Right,"middle" => DragButton::Middle,- other => return ToolResult::error(format!(- "Unknown button \"{other}\" — expected left, right, or middle."- )),+ other => {+ return ToolResult::error(format!(+ "Unknown button \"{other}\" — expected left, right, or middle."+ ))+ }};// from_zoom: translate from last zoom crop context.@@ -154,47 +167,76 @@ impl Tool for DragTool {Some(ctx) => {let (wx, wy) = ctx.zoom_to_window(from_x, from_y);let (wx2, wy2) = ctx.zoom_to_window(to_x, to_y);- from_x = wx; from_y = wy;- to_x = wx2; to_y = wy2;+ from_x = wx;+ from_y = wy;+ to_x = wx2;+ to_y = wy2;+ }+ None => {+ return ToolResult::error(format!(+ "from_zoom=true but no zoom context for pid {pid}. Call zoom first."+ ))}- None => return ToolResult::error(format!(- "from_zoom=true but no zoom context for pid {pid}. Call zoom first."- )),}} else if let Some(ratio) = self.state.resize_registry.ratio(pid) {- from_x *= ratio; from_y *= ratio;- to_x *= ratio; to_y *= ratio;+ from_x *= ratio;+ from_y *= ratio;+ to_x *= ratio;+ to_y *= ratio;}// Translate window-local screenshot pixels → screen coordinates.// Also compute window-local logical coords for CGEventSetWindowLocation.- let (from_sx, from_sy, from_lx, from_ly,- to_sx, to_sy, to_lx, to_ly) = if let Some(wid) = window_id {- let result = tokio::task::spawn_blocking(move || {- let bounds = crate::windows::window_bounds_by_id(wid);- let scale: f64 = if let Some(ref b) = bounds {- if let Ok(png) = crate::capture::screenshot_window_bytes(wid) {- if png.len() >= 24 {- let pw = u32::from_be_bytes([png[16], png[17], png[18], png[19]]) as f64;- let lw = b.width;- if lw > 0.0 && pw > lw { pw / lw } else { 1.0 }- } else { 1.0 }- } else { 1.0 }- } else { 1.0 };- (bounds, scale)- }).await.unwrap_or((None, 1.0));+ let (from_sx, from_sy, from_lx, from_ly, to_sx, to_sy, to_lx, to_ly) =+ if let Some(wid) = window_id {+ let result = tokio::task::spawn_blocking(move || {+ let bounds = crate::windows::window_bounds_by_id(wid);+ let scale: f64 = if let Some(ref b) = bounds {+ if let Ok(png) = crate::capture::screenshot_window_bytes(wid) {+ if png.len() >= 24 {+ let pw =+ u32::from_be_bytes([png[16], png[17], png[18], png[19]]) as f64;+ let lw = b.width;+ if lw > 0.0 && pw > lw {+ pw / lw+ } else {+ 1.0+ }+ } else {+ 1.0+ }+ } else {+ 1.0+ }+ } else {+ 1.0+ };+ (bounds, scale)+ })+ .await+ .unwrap_or((None, 1.0));- if let (Some(b), scale) = result {- let flx = from_x / scale; let fly = from_y / scale;- let tlx = to_x / scale; let tly = to_y / scale;- (b.x + flx, b.y + fly, flx, fly,- b.x + tlx, b.y + tly, tlx, tly)+ if let (Some(b), scale) = result {+ let flx = from_x / scale;+ let fly = from_y / scale;+ let tlx = to_x / scale;+ let tly = to_y / scale;+ (+ b.x + flx,+ b.y + fly,+ flx,+ fly,+ b.x + tlx,+ b.y + tly,+ tlx,+ tly,+ )+ } else {+ (from_x, from_y, from_x, from_y, to_x, to_y, to_x, to_y)+ }} else {(from_x, from_y, from_x, from_y, to_x, to_y, to_x, to_y)- }- } else {- (from_x, from_y, from_x, from_y, to_x, to_y, to_x, to_y)- };+ };// Animate agent cursor along drag path (start → end).if let Some(wid) = window_id {@@ -217,30 +259,64 @@ impl Tool for DragTool {let mods_owned = modifiers.clone();let fg = delivery_mode.is_foreground() && window_id.is_some();let result = focus_guard::with_focus_suppressed(- Some(pid),+ // Foreground drag deliberately activates the target so the global+ // HID stream carries the pressed-button state. A suppression lease+ // here would race that activation and restore the prior app before+ // Chromium receives the gesture.+ if fg { None } else { Some(pid) },prior_front,"drag.CGEvent",|| async move {tokio::task::spawn_blocking(move || -> anyhow::Result<()> {let do_it = move || -> anyhow::Result<()> {let m: Vec<&str> = mods_owned.iter().map(String::as_str).collect();+ if fg {+ // HID delivery is global, so foreground mode must+ // establish a real active application before the+ // gesture begins. The SkyLight flash can be+ // unavailable for Electron child windows; the+ // documented Cocoa activation is the fallback.+ apps::activate_pid(pid);+ std::thread::sleep(std::time::Duration::from_millis(40));+ return crate::input::mouse::drag_at_xy_foreground(+ from_sx,+ from_sy,+ to_sx,+ to_sy,+ duration_ms,+ steps,+ &m,+ button,+ );+ }crate::input::mouse::drag_at_xy(pid,- from_sx, from_sy,- to_sx, to_sy,+ from_sx,+ from_sy,+ to_sx,+ to_sy,Some((from_lx, from_ly)),- Some((to_lx, to_ly)),+ Some((to_lx, to_ly)),window_id,duration_ms,steps,&m,button,+ fg,)};- // Foreground rung: brief front → drag → restore prior frontmost.+ // Foreground rung: activate for the complete HID gesture,+ // then restore the prior app after pointer capture settles.match (fg, window_id) {- (true, Some(wid)) => {- crate::input::skylight::with_foreground_assist(pid as libc::pid_t, wid, do_it)?;+ (true, Some(_wid)) => {+ let result = do_it();+ std::thread::sleep(std::time::Duration::from_millis(100));+ if let Some(previous_pid) = prior_front {+ if previous_pid != pid {+ apps::activate_pid(previous_pid);+ }+ }+ result?;Ok(())}_ => do_it(),@@ -267,11 +343,17 @@ impl Tool for DragTool {} else {format!(" with {}", modifiers.join("+"))};- let btn_suffix = if button_str == "left" { String::new() } else {+ let btn_suffix = if button_str == "left" {+ String::new()+ } else {format!(" ({button_str} button)")};- let mode_label = if fg { " (delivery_mode:foreground)" } else { "" };+ let mode_label = if fg {+ " (delivery_mode:foreground)"+ } else {+ ""+ };match result {Ok(Ok(())) => ToolResult::text(format!("✅ Posted drag{btn_suffix}{mod_suffix} to pid {pid} \diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/scroll.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/scroll.rsindex 070196f4ff..38d8ae59f7 100644--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/scroll.rs+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/scroll.rs@@ -39,10 +39,10 @@ fn def() -> &'static ToolDef {DEF.get_or_init(|| ToolDef {name: "scroll".into(),description: "Scroll the target pid. Two paths, picked by how you address the scroll:\n\n\- • **Pixel-wheel path (targeted)** — when you pass a target, either \+ • **Targeted wheel path** — when you pass a target, either \`element_index`/`element_token` (preferred) or window-local `x, y` pixels: \the driver synthesizes a real mouse-wheel event (CGEventCreateScrollWheelEvent, \- pixel units) AT that screen point. The renderer hit-tests the wheel at the \+ at that screen point. The renderer hit-tests the wheel at the \cursor, so the scroll lands on whatever element is under the point — exactly \like physically rolling the wheel over it. This is the ONLY way to scroll a \nested `overflow:auto` region (e.g. a scrollable <div> with no tabindex): such \@@ -53,7 +53,7 @@ fn def() -> &'static ToolDef {(by='line'); horizontal uses Left/Right arrows. Drives the focused / page \scroller only.\n\n\Mapping: by='page' → larger step; by='line' → smaller step; amount = number of \- wheel notches (pixel path) or keystroke repetitions (keystroke path).".into(),+ wheel notches (targeted path) or keystroke repetitions (keystroke path).".into(),input_schema: serde_json::json!({"type": "object",// `pid` conditionally required (validated in code), not pinned in the@@ -155,7 +155,7 @@ impl Tool for ScrollTool {}}- // ── Pixel-wheel path (targeted scroll) ───────────────────────────────+ // ── Targeted wheel path ─────────────────────────────────────────────// A target — element (preferred) OR window-local x,y — routes the scroll// through a synthesized mouse-wheel event at that screen point, so the// renderer's hit-test delivers it to whatever element is under the@@ -165,7 +165,7 @@ impl Tool for ScrollTool {let x_arg = args.opt_f64("x").or_else(|| args.opt_i64("x").map(|v| v as f64));let y_arg = args.opt_f64("y").or_else(|| args.opt_i64("y").map(|v| v as f64));- // Per-notch pixel step + direction→delta mapping (sign convention lives+ // Per-notch step + direction→delta mapping (sign convention lives// here; the mouse primitive stays sign-agnostic). macOS: +y reveals// content ABOVE, -y reveals BELOW; +x reveals LEFT, -x reveals RIGHT.let step = if by == "page" { WHEEL_STEP_PAGE_PX } else { WHEEL_STEP_LINE_PX };@@ -184,6 +184,18 @@ impl Tool for ScrollTool {// Retina scaling is needed here.let wid = window_id;tokio::task::spawn_blocking(move || {+ // Web content can be present in AX while its frame is below+ // the outer page viewport. Ask the accessibility hierarchy to+ // reveal the target before taking the screen-space center;+ // otherwise the wheel is posted outside the rendered window+ // and nested overflow regions never receive it.+ unsafe {+ let _ = crate::ax::bindings::perform_action(+ element_ptr as AXUIElementRef,+ "AXScrollToVisible",+ );+ }+ std::thread::sleep(std::time::Duration::from_millis(40));let center = unsafe { element_screen_center(element_ptr as AXUIElementRef) };center.map(|(cx, cy)| {let win_local = wid