Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

fix(cua-driver): harden macOS input and capture delivery

Francesco Bonacciec908f74cb9f
diff --git a/libs/cua-driver/rust/crates/cua-driver/tests/harness_swiftui_test.rs b/libs/cua-driver/rust/crates/cua-driver/tests/harness_swiftui_test.rs
index 42fa67a1e1..27435a4f47 100644
--- a/libs/cua-driver/rust/crates/cua-driver/tests/harness_swiftui_test.rs
+++ b/libs/cua-driver/rust/crates/cua-driver/tests/harness_swiftui_test.rs
@@ -26,7 +26,9 @@ use cua_driver_testkit::{harness_app, Driver, McpDriver, ToolResponse};
fn harness_exe() -> PathBuf {
if let Ok(p) = std::env::var("HARNESS_SWIFTUI_APP") {
let pb = PathBuf::from(p).join("Contents/MacOS/CuaTestHarness.SwiftUI");
- if pb.exists() { return pb; }
+ if pb.exists() {
+ return pb;
+ }
}
harness_app(
"harness-swiftui",
@@ -34,7 +36,10 @@ fn harness_exe() -> PathBuf {
)
}
-struct Harness { _app: Child, pid: u32 }
+struct Harness {
+ _app: Child,
+ pid: u32,
+}
impl Harness {
fn launch() -> Option<Self> {
@@ -44,8 +49,10 @@ impl Harness {
return None;
}
let app = Command::new(&exe)
- .stdout(Stdio::null()).stderr(Stdio::null())
- .spawn().ok()?;
+ .stdout(Stdio::null())
+ .stderr(Stdio::null())
+ .spawn()
+ .ok()?;
let pid = app.id();
std::thread::sleep(Duration::from_millis(900));
Some(Self { _app: app, pid })
@@ -76,11 +83,16 @@ fn snapshot_elements(driver: &mut McpDriver, pid: u32, window_id: u64) -> ToolRe
fn harness_swiftui_smoke() {
let harness = match Harness::launch() {
Some(h) => h,
- None => { eprintln!("harness not built — skipping"); return; }
+ None => {
+ eprintln!("harness not built — skipping");
+ return;
+ }
};
println!("harness pid={}", harness.pid);
- let Some(mut driver) = McpDriver::spawn() else { return };
+ let Some(mut driver) = McpDriver::spawn_macos_daemon_proxy() else {
+ return;
+ };
let (wid, title) = driver
.find_window(harness.pid as i64, "CuaTestHarness SwiftUI")
@@ -88,11 +100,11 @@ fn harness_swiftui_smoke() {
println!("main window: id={wid} title={title:?}");
let snap = snapshot_elements(&mut driver, harness.pid, wid);
- if looks_empty(snap.text()) {
+ if looks_empty(snap.tree_text()) {
eprintln!("AX empty — TCC not granted; skipping element-assertions");
return;
}
- let text = snap.text();
+ let text = snap.tree_text();
println!("snapshot:\n{text}");
// SwiftUI Text views render as AXStaticText leaves and don't propagate
@@ -100,17 +112,22 @@ fn harness_swiftui_smoke() {
// quirk as AppKit's NSTextField label mode + WPF's TextBlock). Assert
// on text content for labels, AX-id only for actionable controls.
for aid in [
- "btn-increment", "btn-reset",
+ "btn-increment",
+ "btn-reset",
"txt-input",
"btn-open-popover",
"btn-exit",
] {
- assert!(has_id(snap.text(), aid),
- "missing AX identifier {aid} in SwiftUI snapshot");
+ assert!(
+ has_id(snap.tree_text(), aid),
+ "missing AX identifier {aid} in SwiftUI snapshot"
+ );
}
- assert!(text.contains("HARNESS_TEXT_MARKER_v1"),
- "text_body marker not in SwiftUI snapshot");
+ assert!(
+ text.contains("HARNESS_TEXT_MARKER_v1"),
+ "text_body marker not in SwiftUI snapshot"
+ );
}
/// popover: click the popover trigger, verify the popover body text appears
@@ -120,29 +137,38 @@ fn harness_swiftui_smoke() {
fn harness_swiftui_popover() {
let harness = match Harness::launch() {
Some(h) => h,
- None => { eprintln!("harness not built — skipping"); return; }
+ None => {
+ eprintln!("harness not built — skipping");
+ return;
+ }
};
- let Some(mut driver) = McpDriver::spawn() else { return };
+ let Some(mut driver) = McpDriver::spawn_macos_daemon_proxy() else {
+ return;
+ };
let (wid, _) = driver
.find_window(harness.pid as i64, "CuaTestHarness SwiftUI")
.expect("main window not found");
let snap_pre = snapshot_elements(&mut driver, harness.pid, wid);
- if looks_empty(snap_pre.text()) {
- eprintln!("AX empty — TCC not granted; skipping"); return;
+ if looks_empty(snap_pre.tree_text()) {
+ eprintln!("AX empty — TCC not granted; skipping");
+ return;
}
// Verify popover body is NOT yet in the tree.
- let pre_text = snap_pre.text().to_owned();
- assert!(!pre_text.contains("POPOVER_MARKER_v1"),
- "popover body unexpectedly present BEFORE open");
-
- let trigger_idx: u64 = if let Some(i) = element_index_by_id(snap_pre.text(), "btn-open-popover") {
- i
- } else {
- eprintln!("popover trigger not found, skipping");
- return;
- };
+ let pre_text = snap_pre.tree_text().to_owned();
+ assert!(
+ !pre_text.contains("POPOVER_MARKER_v1"),
+ "popover body unexpectedly present BEFORE open"
+ );
+
+ let trigger_idx: u64 =
+ if let Some(i) = element_index_by_id(snap_pre.tree_text(), "btn-open-popover") {
+ i
+ } else {
+ eprintln!("popover trigger not found, skipping");
+ return;
+ };
let click = driver.call(
"click",
serde_json::json!({
@@ -159,18 +185,23 @@ fn harness_swiftui_popover() {
// Popovers may live in a separate AXWindow on macOS — try the main
// window first, then list_windows for additional candidates.
let snap_post = snapshot_elements(&mut driver, harness.pid, wid);
- let mut found_marker = snap_post.text().contains("POPOVER_MARKER_v1");
+ let mut found_marker = snap_post.tree_text().contains("POPOVER_MARKER_v1");
if !found_marker {
// Walk any new windows for the same pid.
- let resp = driver.call("list_windows",
- serde_json::json!({ "pid": harness.pid as i64 }));
+ let resp = driver.call(
+ "list_windows",
+ serde_json::json!({ "pid": harness.pid as i64 }),
+ );
if let Some(wins) = resp.structured()["windows"].as_array() {
for w in wins {
if let Some(other_wid) = w["window_id"].as_u64() {
- if other_wid == wid { continue; }
+ if other_wid == wid {
+ continue;
+ }
let s = snapshot_elements(&mut driver, harness.pid, other_wid);
- if s.text().contains("POPOVER_MARKER_v1") {
- found_marker = true; break;
+ if s.tree_text().contains("POPOVER_MARKER_v1") {
+ found_marker = true;
+ break;
}
}
}
diff --git a/libs/cua-driver/rust/crates/cua-driver/tests/modality_desktop_scope_macos_test.rs b/libs/cua-driver/rust/crates/cua-driver/tests/modality_desktop_scope_macos_test.rs
index 39db3d248e..d1cef67d7c 100644
--- a/libs/cua-driver/rust/crates/cua-driver/tests/modality_desktop_scope_macos_test.rs
+++ b/libs/cua-driver/rust/crates/cua-driver/tests/modality_desktop_scope_macos_test.rs
@@ -51,14 +51,22 @@ fn launch(driver: &mut McpDriver) -> Option<(u32, u64)> {
}
driver
.reaper()
- .spawn(Command::new(&exe).stdout(Stdio::null()).stderr(Stdio::null()))
+ .spawn(
+ Command::new(&exe)
+ .stdout(Stdio::null())
+ .stderr(Stdio::null()),
+ )
.ok()?;
let deadline = Instant::now() + Duration::from_secs(14);
while Instant::now() < deadline {
let r = driver.call("list_windows", serde_json::json!({}));
if let Some(wins) = r.structured()["windows"].as_array() {
for w in wins {
- if w["title"].as_str().unwrap_or("").contains("CuaTestHarness AppKit") {
+ if w["title"]
+ .as_str()
+ .unwrap_or("")
+ .contains("CuaTestHarness AppKit")
+ {
let pid = w["pid"].as_u64().unwrap_or(0) as u32;
let wid = w["window_id"].as_u64().unwrap_or(0);
if pid != 0 && wid != 0 {
@@ -70,7 +78,9 @@ fn launch(driver: &mut McpDriver) -> Option<(u32, u64)> {
}
std::thread::sleep(Duration::from_millis(400));
}
- eprintln!("[desktop-mac] harness window never appeared — graphical session available? skipping");
+ eprintln!(
+ "[desktop-mac] harness window never appeared — graphical session available? skipping"
+ );
None
}
@@ -91,10 +101,18 @@ fn increment_center(snap: &serde_json::Value) -> Option<(i64, i64)> {
let els = snap["elements"].as_array()?;
let btn = els.iter().find(|e| {
e["role"].as_str() == Some("AXButton")
- && e["label"].as_str().map(|l| l.trim().eq_ignore_ascii_case("increment")).unwrap_or(false)
+ && e["label"]
+ .as_str()
+ .map(|l| l.trim().eq_ignore_ascii_case("increment"))
+ .unwrap_or(false)
})?;
let f = &btn["frame"];
- let (x, y, w, h) = (f["x"].as_f64()?, f["y"].as_f64()?, f["w"].as_f64()?, f["h"].as_f64()?);
+ let (x, y, w, h) = (
+ f["x"].as_f64()?,
+ f["y"].as_f64()?,
+ f["w"].as_f64()?,
+ f["h"].as_f64()?,
+ );
Some(((x + w / 2.0) as i64, (y + h / 2.0) as i64))
}
@@ -104,7 +122,10 @@ fn increment_center(snap: &serde_json::Value) -> Option<(i64, i64)> {
fn counter(snap: &serde_json::Value) -> Option<u64> {
let tree = snap["tree_markdown"].as_str()?;
let idx = tree.find("counter=")? + "counter=".len();
- let digits: String = tree[idx..].chars().take_while(|c| c.is_ascii_digit()).collect();
+ let digits: String = tree[idx..]
+ .chars()
+ .take_while(|c| c.is_ascii_digit())
+ .collect();
digits.parse().ok()
}
@@ -135,8 +156,12 @@ fn activate_pid(pid: u32) {
#[test]
#[ignore]
fn desktop_scope_windowless_click_lands_on_control() {
- let Some(mut driver) = McpDriver::spawn() else { return };
- let Some((pid, wid)) = launch(&mut driver) else { return };
+ let Some(mut driver) = McpDriver::spawn_macos_daemon_proxy() else {
+ return;
+ };
+ let Some((pid, wid)) = launch(&mut driver) else {
+ return;
+ };
// Settle for the AppKit AX tree to register the button + its frame.
let mut snap = ax_snapshot(&mut driver, pid, wid);
@@ -158,8 +183,15 @@ fn desktop_scope_windowless_click_lands_on_control() {
activate_pid(pid);
// Window-less screen-absolute click — no pid, no window_id; scope per-call.
- let clicked = driver.call("click", serde_json::json!({ "x": cx, "y": cy, "scope": "desktop", "session": SESSION }));
- assert!(!clicked.is_error(), "desktop-scope click errored: {}", clicked.text());
+ let clicked = driver.call(
+ "click",
+ serde_json::json!({ "x": cx, "y": cy, "scope": "desktop", "session": SESSION }),
+ );
+ assert!(
+ !clicked.is_error(),
+ "desktop-scope click errored: {}",
+ clicked.text()
+ );
assert!(
clicked.text().to_lowercase().contains("desktop scope"),
"click not reported as desktop-scope: {}",
@@ -193,9 +225,14 @@ fn desktop_scope_windowless_click_lands_on_control() {
#[test]
#[ignore]
fn window_scope_rejects_windowless_click() {
- let Some(mut driver) = McpDriver::spawn() else { return };
+ let Some(mut driver) = McpDriver::spawn_macos_daemon_proxy() else {
+ return;
+ };
// Default scope is "window" — a window-less click must be rejected.
- let r = driver.call("click", serde_json::json!({ "x": 100, "y": 100, "scope": "window", "session": SESSION }));
+ let r = driver.call(
+ "click",
+ serde_json::json!({ "x": 100, "y": 100, "scope": "window", "session": SESSION }),
+ );
let txt = r.text().to_lowercase();
assert!(
r.is_error() || txt.contains("desktop scope") || txt.contains("desktop_scope_disabled"),
diff --git a/libs/cua-driver/rust/crates/cua-driver/tests/protocol_media_test.rs b/libs/cua-driver/rust/crates/cua-driver/tests/protocol_media_test.rs
index e79ced7c48..ec8cc34288 100644
--- a/libs/cua-driver/rust/crates/cua-driver/tests/protocol_media_test.rs
+++ b/libs/cua-driver/rust/crates/cua-driver/tests/protocol_media_test.rs
@@ -14,7 +14,9 @@ use cua_driver_testkit::RawDriver;
#[test]
#[cfg(target_os = "windows")]
fn screenshot() {
- let Some(mut d) = RawDriver::spawn() else { return; };
+ let Some(mut d) = RawDriver::spawn() else {
+ return;
+ };
d.send(&serde_json::json!({"jsonrpc":"2.0","id":1,"method":"initialize","params":{}}));
d.recv();
@@ -35,14 +37,21 @@ fn screenshot() {
"params":{"name":"screenshot","arguments":{"format":"jpeg","quality":70}}
}));
let resp = d.recv();
- assert!(resp["error"].is_null(), "Protocol error from screenshot jpeg: {resp:?}");
+ assert!(
+ resp["error"].is_null(),
+ "Protocol error from screenshot jpeg: {resp:?}"
+ );
if !resp["result"]["isError"].as_bool().unwrap_or(false) {
let content = resp["result"]["content"].as_array().expect("content array");
let has_jpeg = content.iter().any(|c| {
- c["type"] == "image" && c["mimeType"].as_str().unwrap_or("") == "image/jpeg"
+ c["type"] == "image"
+ && c["mimeType"].as_str().unwrap_or("") == "image/jpeg"
&& c["data"].as_str().map(|s| s.len() > 10).unwrap_or(false)
});
- assert!(has_jpeg, "Expected image/jpeg in screenshot response: {content:?}");
+ assert!(
+ has_jpeg,
+ "Expected image/jpeg in screenshot response: {content:?}"
+ );
let sc = &resp["result"]["structuredContent"];
assert!(sc["width"].as_f64().unwrap_or(0.0) > 0.0);
assert!(sc["height"].as_f64().unwrap_or(0.0) > 0.0);
@@ -54,7 +63,9 @@ fn screenshot() {
#[cfg(target_os = "macos")]
fn screenshot_no_window_id() {
//! Call screenshot without window_id — should capture the full display and return image content.
- let Some(mut d) = RawDriver::spawn() else { return; };
+ let Some(mut d) = RawDriver::spawn() else {
+ return;
+ };
d.send(&serde_json::json!({"jsonrpc":"2.0","id":1,"method":"initialize","params":{}}));
d.recv();
@@ -64,16 +75,24 @@ fn screenshot_no_window_id() {
"params":{"name":"screenshot","arguments":{}}
}));
let resp = d.recv();
- assert!(resp["error"].is_null(), "Protocol error from screenshot: {resp:?}");
+ assert!(
+ resp["error"].is_null(),
+ "Protocol error from screenshot: {resp:?}"
+ );
let content = resp["result"]["content"].as_array().expect("content array");
- assert!(!content.is_empty(), "screenshot should return at least one content item");
+ assert!(
+ !content.is_empty(),
+ "screenshot should return at least one content item"
+ );
}
#[test]
#[cfg(target_os = "macos")]
fn screenshot_jpeg_format() {
//! Call screenshot with format=jpeg — should return a JPEG image.
- let Some(mut d) = RawDriver::spawn() else { return; };
+ let Some(mut d) = RawDriver::spawn() else {
+ return;
+ };
d.send(&serde_json::json!({"jsonrpc":"2.0","id":1,"method":"initialize","params":{}}));
d.recv();
@@ -83,20 +102,37 @@ fn screenshot_jpeg_format() {
"params":{"name":"screenshot","arguments":{"format":"jpeg","quality":70}}
}));
let resp = d.recv();
- assert!(resp["error"].is_null(), "Protocol error from screenshot: {resp:?}");
+ assert!(
+ resp["error"].is_null(),
+ "Protocol error from screenshot: {resp:?}"
+ );
let content = resp["result"]["content"].as_array().expect("content array");
let is_error = resp["result"]["isError"].as_bool().unwrap_or(false);
if !is_error {
let has_jpeg = content.iter().any(|c| {
- c["type"] == "image" && c["mimeType"].as_str().unwrap_or("") == "image/jpeg"
+ c["type"] == "image"
+ && c["mimeType"].as_str().unwrap_or("") == "image/jpeg"
&& c["data"].as_str().map(|s| s.len() > 10).unwrap_or(false)
});
- assert!(has_jpeg, "Expected image/jpeg in screenshot response, got: {content:?}");
+ assert!(
+ has_jpeg,
+ "Expected image/jpeg in screenshot response, got: {content:?}"
+ );
// Verify structured content has width and height.
let sc = &resp["result"]["structuredContent"];
- assert!(sc["width"].as_f64().unwrap_or(0.0) > 0.0, "Expected positive width in structuredContent");
- assert!(sc["height"].as_f64().unwrap_or(0.0) > 0.0, "Expected positive height in structuredContent");
- assert_eq!(sc["format"].as_str().unwrap_or(""), "jpeg", "Expected format=jpeg");
+ assert!(
+ sc["width"].as_f64().unwrap_or(0.0) > 0.0,
+ "Expected positive width in structuredContent"
+ );
+ assert!(
+ sc["height"].as_f64().unwrap_or(0.0) > 0.0,
+ "Expected positive height in structuredContent"
+ );
+ assert_eq!(
+ sc["format"].as_str().unwrap_or(""),
+ "jpeg",
+ "Expected format=jpeg"
+ );
}
}
@@ -105,7 +141,9 @@ fn screenshot_jpeg_format() {
fn zoom_tool_returns_jpeg() {
//! Call zoom on a visible window and verify the result contains a JPEG image.
//! Skips gracefully if no windows are visible (headless environment).
- let Some(mut d) = RawDriver::spawn() else { return; };
+ let Some(mut d) = RawDriver::spawn() else {
+ return;
+ };
d.send(&serde_json::json!({"jsonrpc":"2.0","id":1,"method":"initialize","params":{}}));
d.recv();
@@ -124,8 +162,12 @@ fn zoom_tool_returns_jpeg() {
// Try windows until one captures successfully.
let mut found_jpeg = false;
let mut tried = 0usize;
+ let mut zoom_errors: Vec<String> = Vec::new();
+ let mut successful_without_jpeg = false;
for win in windows.iter().take(5) {
- let Some(wid) = win["window_id"].as_u64() else { continue; };
+ let Some(wid) = win["window_id"].as_u64() else {
+ continue;
+ };
tried += 1;
d.send(&serde_json::json!({
@@ -138,19 +180,39 @@ fn zoom_tool_returns_jpeg() {
let r = d.recv();
if r["result"]["isError"].as_bool().unwrap_or(false) {
// This window might be off-screen or not capturable — try the next.
+ let text = r["result"]["content"]
+ .as_array()
+ .and_then(|items| items.iter().find_map(|c| c["text"].as_str()))
+ .unwrap_or("<no error text>")
+ .to_owned();
+ zoom_errors.push(format!("window_id={wid}: {text}"));
continue;
}
let content = r["result"]["content"].as_array().expect("content array");
found_jpeg = content.iter().any(|c| {
- c["type"] == "image" && c["mimeType"].as_str().unwrap_or("") == "image/jpeg"
+ c["type"] == "image"
+ && c["mimeType"].as_str().unwrap_or("") == "image/jpeg"
&& c["data"].as_str().map(|s| s.len() > 10).unwrap_or(false)
});
+ if !found_jpeg {
+ successful_without_jpeg = true;
+ }
if found_jpeg {
// Also verify structuredContent.
let sc = &r["result"]["structuredContent"];
- assert!(sc["width"].as_f64().unwrap_or(0.0) > 0.0, "Expected positive width: {sc:?}");
- assert!(sc["height"].as_f64().unwrap_or(0.0) > 0.0, "Expected positive height: {sc:?}");
- assert_eq!(sc["format"].as_str().unwrap_or(""), "jpeg", "Expected format=jpeg: {sc:?}");
+ assert!(
+ sc["width"].as_f64().unwrap_or(0.0) > 0.0,
+ "Expected positive width: {sc:?}"
+ );
+ assert!(
+ sc["height"].as_f64().unwrap_or(0.0) > 0.0,
+ "Expected positive height: {sc:?}"
+ );
+ assert_eq!(
+ sc["format"].as_str().unwrap_or(""),
+ "jpeg",
+ "Expected format=jpeg: {sc:?}"
+ );
break;
}
}
@@ -159,7 +221,26 @@ fn zoom_tool_returns_jpeg() {
eprintln!("No on-screen windows found — skipping zoom test");
return;
}
- assert!(found_jpeg, "Expected at least one window to return a valid JPEG from zoom");
+ if !found_jpeg
+ && !successful_without_jpeg
+ && cfg!(target_os = "macos")
+ && !zoom_errors.is_empty()
+ && zoom_errors
+ .iter()
+ .all(|e| e.contains("screencapture failed"))
+ {
+ eprintln!(
+ "No visible windows were capturable by the raw unbundled test process — skipping zoom test. \
+ Errors: {}",
+ zoom_errors.join(" | ")
+ );
+ return;
+ }
+ assert!(
+ found_jpeg,
+ "Expected at least one window to return a valid JPEG from zoom. Errors: {}",
+ zoom_errors.join(" | ")
+ );
}
#[test]
@@ -169,7 +250,9 @@ fn zoom_from_zoom_click_round_trip() {
//! 1. click(from_zoom=true) with no zoom context returns the expected error.
//! 2. zoom() stores context; subsequent click(from_zoom=true) does NOT return
//! that error (translation succeeded, even if the click itself may fail).
- let Some(mut d) = RawDriver::spawn() else { return; };
+ let Some(mut d) = RawDriver::spawn() else {
+ return;
+ };
d.send(&serde_json::json!({"jsonrpc":"2.0","id":1,"method":"initialize","params":{}}));
d.recv();
@@ -180,7 +263,10 @@ fn zoom_from_zoom_click_round_trip() {
"params":{"name":"list_windows","arguments":{"on_screen_only":true}}
}));
let resp = d.recv();
- let wins = resp["result"]["structuredContent"]["windows"].as_array().cloned().unwrap_or_default();
+ let wins = resp["result"]["structuredContent"]["windows"]
+ .as_array()
+ .cloned()
+ .unwrap_or_default();
let Some(win) = wins.iter().find(|w| w["pid"].as_i64().is_some()) else {
eprintln!("No on-screen windows — skipping from_zoom test");
return;
@@ -196,8 +282,14 @@ fn zoom_from_zoom_click_round_trip() {
let resp = d.recv();
let is_err = resp["result"]["isError"].as_bool().unwrap_or(false);
let err_text = resp["result"]["content"][0]["text"].as_str().unwrap_or("");
- assert!(is_err, "Expected error when from_zoom=true with no context, got: {resp:?}");
- assert!(err_text.contains("no zoom context"), "Expected 'no zoom context' error, got: {err_text}");
+ assert!(
+ is_err,
+ "Expected error when from_zoom=true with no context, got: {resp:?}"
+ );
+ assert!(
+ err_text.contains("no zoom context"),
+ "Expected 'no zoom context' error, got: {err_text}"
+ );
// Step 2: call zoom on that window to store context.
d.send(&serde_json::json!({
@@ -213,7 +305,10 @@ fn zoom_from_zoom_click_round_trip() {
if !cfg!(target_os = "windows") {
// Verify zoom returned structured content with width/height.
let sc = &resp["result"]["structuredContent"];
- assert!(sc["width"].as_u64().unwrap_or(0) > 0, "zoom should return positive width: {sc:?}");
+ assert!(
+ sc["width"].as_u64().unwrap_or(0) > 0,
+ "zoom should return positive width: {sc:?}"
+ );
}
// Step 3: click with from_zoom=true — should NOT return the "no zoom context" error.
@@ -224,17 +319,22 @@ fn zoom_from_zoom_click_round_trip() {
let resp = d.recv();
let err_text = resp["result"]["content"][0]["text"].as_str().unwrap_or("");
// Translation should succeed — if click fails it's for another reason (target app state), not missing context.
- assert!(!err_text.contains("no zoom context"),
- "After zoom(), from_zoom click should not say 'no zoom context', got: {err_text}");
+ assert!(
+ !err_text.contains("no zoom context"),
+ "After zoom(), from_zoom click should not say 'no zoom context', got: {err_text}"
+ );
}
#[test]
#[cfg(any(target_os = "macos", target_os = "windows"))]
fn recording_session() {
//! Enable recording, invoke a non-read-only tool (recorded), disable, verify action.json written.
- let Some(mut d) = RawDriver::spawn() else { return; };
+ let Some(mut d) = RawDriver::spawn() else {
+ return;
+ };
- let tmp_dir = std::env::temp_dir().join(format!("cua-driver-rs-rec-test-{}", std::process::id()));
+ let tmp_dir =
+ std::env::temp_dir().join(format!("cua-driver-rs-rec-test-{}", std::process::id()));
let tmp_str = tmp_dir.to_string_lossy().to_string();
d.send(&serde_json::json!({"jsonrpc":"2.0","id":1,"method":"initialize","params":{}}));
@@ -246,8 +346,13 @@ fn recording_session() {
"params":{"name":"start_recording","arguments":{"output_dir":tmp_str,"record_video":false}}
}));
let resp = d.recv();
- assert!(!resp["result"]["isError"].as_bool().unwrap_or(false), "start_recording failed: {resp:?}");
- assert!(resp["result"]["structuredContent"]["enabled"].as_bool().unwrap_or(false));
+ assert!(
+ !resp["result"]["isError"].as_bool().unwrap_or(false),
+ "start_recording failed: {resp:?}"
+ );
+ assert!(resp["result"]["structuredContent"]["enabled"]
+ .as_bool()
+ .unwrap_or(false));
// Invoke a non-read-only tool — should be recorded.
if cfg!(target_os = "windows") {
@@ -269,7 +374,9 @@ fn recording_session() {
"params":{"name":"get_recording_state","arguments":{}}
}));
let resp = d.recv();
- let next_turn = resp["result"]["structuredContent"]["next_turn"].as_u64().unwrap_or(0);
+ let next_turn = resp["result"]["structuredContent"]["next_turn"]
+ .as_u64()
+ .unwrap_or(0);
assert!(next_turn >= 2, "expected next_turn >= 2, got {next_turn}");
// Disable recording.
@@ -278,7 +385,9 @@ fn recording_session() {
"params":{"name":"stop_recording","arguments":{}}
}));
let resp = d.recv();
- assert!(!resp["result"]["structuredContent"]["enabled"].as_bool().unwrap_or(true));
+ assert!(!resp["result"]["structuredContent"]["enabled"]
+ .as_bool()
+ .unwrap_or(true));
drop(d);
// Small sync wait for the write to flush.
@@ -287,14 +396,16 @@ fn recording_session() {
// Verify turn-00001/action.json was written.
let action_path = tmp_dir.join("turn-00001").join("action.json");
assert!(action_path.exists(), "Expected {action_path:?} to exist");
- let content: serde_json::Value = serde_json::from_str(
- &std::fs::read_to_string(&action_path).unwrap()
- ).unwrap();
+ let content: serde_json::Value =
+ serde_json::from_str(&std::fs::read_to_string(&action_path).unwrap()).unwrap();
if cfg!(target_os = "windows") {
assert_eq!(content["tool"].as_str().unwrap_or(""), "move_cursor");
assert_eq!(content["arguments"]["x"].as_f64().unwrap_or(0.0), 100.0);
} else {
- assert_eq!(content["tool"].as_str().unwrap_or(""), "set_agent_cursor_enabled");
+ assert_eq!(
+ content["tool"].as_str().unwrap_or(""),
+ "set_agent_cursor_enabled"
+ );
assert_eq!(content["arguments"]["enabled"].as_bool(), Some(false));
}
@@ -306,7 +417,9 @@ fn recording_session() {
fn recording_screenshot_capture() {
//! When recording is active and a tool call includes a window_id, a screenshot.png
//! should appear alongside action.json in the turn folder.
- let Some(mut d) = RawDriver::spawn() else { return; };
+ let Some(mut d) = RawDriver::spawn() else {
+ return;
+ };
let tmp_dir = std::env::temp_dir().join(format!("cua-driver-rs-rec-ss-{}", std::process::id()));
let tmp_str = tmp_dir.to_string_lossy().to_string();
@@ -320,9 +433,13 @@ fn recording_screenshot_capture() {
"params":{"name":"list_windows","arguments":{"on_screen_only":true}}
}));
let resp = d.recv();
- let wins = resp["result"]["structuredContent"]["windows"].as_array().cloned().unwrap_or_default();
+ let wins = resp["result"]["structuredContent"]["windows"]
+ .as_array()
+ .cloned()
+ .unwrap_or_default();
// Prefer windows with non-empty titles (more likely capturable).
- let win = wins.iter()
+ let win = wins
+ .iter()
.find(|w| w["pid"].as_i64().is_some() && !w["title"].as_str().unwrap_or("").is_empty())
.or_else(|| wins.iter().find(|w| w["pid"].as_i64().is_some()));
let Some(win) = win else {
@@ -338,7 +455,10 @@ fn recording_screenshot_capture() {
"params":{"name":"start_recording","arguments":{"output_dir":tmp_str,"record_video":false}}
}));
let resp = d.recv();
- assert!(!resp["result"]["isError"].as_bool().unwrap_or(false), "start_recording failed: {resp:?}");
+ assert!(
+ !resp["result"]["isError"].as_bool().unwrap_or(false),
+ "start_recording failed: {resp:?}"
+ );
// Invoke click (non-read-only) with window_id + pid — recording should capture screenshot.
d.send(&serde_json::json!({
@@ -360,10 +480,12 @@ fn recording_screenshot_capture() {
// action.json must exist.
let turn_dir = tmp_dir.join("turn-00001");
let action_path = turn_dir.join("action.json");
- assert!(action_path.exists(), "Expected action.json at {action_path:?}");
- let content: serde_json::Value = serde_json::from_str(
- &std::fs::read_to_string(&action_path).unwrap()
- ).unwrap();
+ assert!(
+ action_path.exists(),
+ "Expected action.json at {action_path:?}"
+ );
+ let content: serde_json::Value =
+ serde_json::from_str(&std::fs::read_to_string(&action_path).unwrap()).unwrap();
assert_eq!(content["tool"].as_str().unwrap_or(""), "click");
// screenshot.png should exist if screencapture succeeded for this window.
@@ -373,7 +495,11 @@ fn recording_screenshot_capture() {
} else {
let ss_bytes = std::fs::read(&ss_path).unwrap();
assert!(!ss_bytes.is_empty(), "screenshot.png should not be empty");
- assert_eq!(&ss_bytes[..4], b"\x89PNG", "screenshot.png should be a valid PNG");
+ assert_eq!(
+ &ss_bytes[..4],
+ b"\x89PNG",
+ "screenshot.png should be a valid PNG"
+ );
// click.png should also exist (click at 0,0 with crosshair marker).
let click_path = turn_dir.join("click.png");
@@ -382,7 +508,11 @@ fn recording_screenshot_capture() {
} else {
let click_bytes = std::fs::read(&click_path).unwrap();
assert!(!click_bytes.is_empty(), "click.png should not be empty");
- assert_eq!(&click_bytes[..4], b"\x89PNG", "click.png should be a valid PNG");
+ assert_eq!(
+ &click_bytes[..4],
+ b"\x89PNG",
+ "click.png should be a valid PNG"
+ );
}
}
@@ -399,7 +529,9 @@ fn start_recording_record_video_flag_accepted() {
//! session enters the enabled state. Full video lifecycle is
//! covered by the manual demo + the per-platform smoke when
//! ffmpeg is installed.
- let Some(mut d) = RawDriver::spawn() else { return; };
+ let Some(mut d) = RawDriver::spawn() else {
+ return;
+ };
let tmp_dir = std::env::temp_dir().join(format!("cua-driver-rs-recvid-{}", std::process::id()));
let tmp_str = tmp_dir.to_string_lossy().to_string();
@@ -415,9 +547,15 @@ fn start_recording_record_video_flag_accepted() {
}}
}));
let resp = d.recv();
- assert!(resp["error"].is_null(), "Expected no JSON-RPC error, got: {resp:?}");
+ assert!(
+ resp["error"].is_null(),
+ "Expected no JSON-RPC error, got: {resp:?}"
+ );
let is_err = resp["result"]["isError"].as_bool().unwrap_or(false);
- assert!(!is_err, "start_recording with record_video:false should succeed: {resp:?}");
+ assert!(
+ !is_err,
+ "start_recording with record_video:false should succeed: {resp:?}"
+ );
d.send(&serde_json::json!({
"jsonrpc":"2.0","id":3,"method":"tools/call",
@@ -432,15 +570,19 @@ fn start_recording_record_video_flag_accepted() {
#[cfg(any(target_os = "macos", target_os = "windows"))]
fn replay_trajectory() {
//! Write a minimal trajectory (one move_cursor turn), replay it, verify succeeded=1.
- let Some(mut d) = RawDriver::spawn() else { return; };
+ let Some(mut d) = RawDriver::spawn() else {
+ return;
+ };
- let traj_dir = std::env::temp_dir().join(format!("cua-driver-rs-replay-{}", std::process::id()));
+ let traj_dir =
+ std::env::temp_dir().join(format!("cua-driver-rs-replay-{}", std::process::id()));
let turn_dir = traj_dir.join("turn-00001");
std::fs::create_dir_all(&turn_dir).unwrap();
std::fs::write(
turn_dir.join("action.json"),
r#"{"tool":"move_cursor","arguments":{"x":50.0,"y":60.0}}"#,
- ).unwrap();
+ )
+ .unwrap();
d.send(&serde_json::json!({"jsonrpc":"2.0","id":1,"method":"initialize","params":{}}));
d.recv();
@@ -453,10 +595,21 @@ fn replay_trajectory() {
}}
}));
let resp = d.recv();
- assert!(!resp["result"]["isError"].as_bool().unwrap_or(false), "replay error: {resp:?}");
+ assert!(
+ !resp["result"]["isError"].as_bool().unwrap_or(false),
+ "replay error: {resp:?}"
+ );
let sc = &resp["result"]["structuredContent"];
- assert_eq!(sc["attempted"].as_u64().unwrap_or(0), 1, "expected attempted=1");
- assert_eq!(sc["succeeded"].as_u64().unwrap_or(0), 1, "expected succeeded=1");
+ assert_eq!(
+ sc["attempted"].as_u64().unwrap_or(0),
+ 1,
+ "expected attempted=1"
+ );
+ assert_eq!(
+ sc["succeeded"].as_u64().unwrap_or(0),
+ 1,
+ "expected succeeded=1"
+ );
assert_eq!(sc["failed"].as_u64().unwrap_or(99), 0, "expected failed=0");
drop(d);
@@ -468,7 +621,9 @@ fn replay_trajectory() {
fn click_debug_image_out() {
//! click with debug_image_out writes a PNG crosshair file and then proceeds.
//! Verifies the debug capture path works end-to-end.
- let Some(mut d) = RawDriver::spawn() else { return; };
+ let Some(mut d) = RawDriver::spawn() else {
+ return;
+ };
d.send(&serde_json::json!({"jsonrpc":"2.0","id":1,"method":"initialize","params":{}}));
d.recv();
@@ -480,16 +635,19 @@ fn click_debug_image_out() {
let resp = d.recv();
let wins = resp["result"]["structuredContent"]["windows"].as_array();
// Prefer a window with a non-empty title to avoid uncapturable overlay/system windows.
- let Some(win) = wins.and_then(|a| a.iter().find(|w| {
- w["pid"].as_i64().is_some() && !w["title"].as_str().unwrap_or("").is_empty()
- }).or_else(|| a.iter().find(|w| w["pid"].as_i64().is_some()))) else {
+ let Some(win) = wins.and_then(|a| {
+ a.iter()
+ .find(|w| w["pid"].as_i64().is_some() && !w["title"].as_str().unwrap_or("").is_empty())
+ .or_else(|| a.iter().find(|w| w["pid"].as_i64().is_some()))
+ }) else {
eprintln!("No on-screen windows — skipping debug_image_out test");
return;
};
let pid = win["pid"].as_i64().unwrap();
let wid = win["window_id"].as_u64().unwrap();
- let dbg_path = std::env::temp_dir().join(format!("cua-driver-rs-dbg-{}.png", std::process::id()));
+ let dbg_path =
+ std::env::temp_dir().join(format!("cua-driver-rs-dbg-{}.png", std::process::id()));
let dbg_path_str = dbg_path.to_string_lossy().to_string();
// Click with debug_image_out — should write the PNG and then proceed.
@@ -502,7 +660,10 @@ fn click_debug_image_out() {
}}
}));
let resp = d.recv();
- assert!(resp["error"].is_null(), "Protocol error from click+debug_image_out: {resp:?}");
+ assert!(
+ resp["error"].is_null(),
+ "Protocol error from click+debug_image_out: {resp:?}"
+ );
// The tool may return isError if screencapture can't capture this particular window
// (e.g., overlay windows, system windows). If that happens, skip the PNG check.
@@ -514,9 +675,15 @@ fn click_debug_image_out() {
}
// Verify the PNG was actually written.
- assert!(dbg_path.exists(), "debug_image_out PNG was not written to {dbg_path:?}");
+ assert!(
+ dbg_path.exists(),
+ "debug_image_out PNG was not written to {dbg_path:?}"
+ );
let meta = std::fs::metadata(&dbg_path).expect("metadata");
- assert!(meta.len() > 100, "debug_image_out PNG is suspiciously small");
+ assert!(
+ meta.len() > 100,
+ "debug_image_out PNG is suspiciously small"
+ );
let _ = std::fs::remove_file(&dbg_path);
}
@@ -526,7 +693,9 @@ fn click_debug_image_out() {
fn set_config_screenshot_resize() {
//! set_config(max_image_dimension=200) then screenshot — returned image must
//! have both dimensions ≤ 200. Verifies the ResizeRegistry pipeline end-to-end.
- let Some(mut d) = RawDriver::spawn() else { return; };
+ let Some(mut d) = RawDriver::spawn() else {
+ return;
+ };
d.send(&serde_json::json!({"jsonrpc":"2.0","id":1,"method":"initialize","params":{}}));
d.recv();
@@ -538,7 +707,10 @@ fn set_config_screenshot_resize() {
}));
let resp = d.recv();
assert!(resp["error"].is_null(), "set_config failed: {resp:?}");
- assert!(!resp["result"]["isError"].as_bool().unwrap_or(false), "set_config returned error: {resp:?}");
+ assert!(
+ !resp["result"]["isError"].as_bool().unwrap_or(false),
+ "set_config returned error: {resp:?}"
+ );
// Capture full display; must be ≤ 200 px in both dimensions.
d.send(&serde_json::json!({
@@ -562,8 +734,14 @@ fn set_config_screenshot_resize() {
eprintln!("screenshot returned no dimensions — skipping resize assertion: {resp:?}");
return;
};
- assert!(w <= 200, "screenshot width {w} should be ≤ 200 after max_image_dimension=200");
- assert!(h <= 200, "screenshot height {h} should be ≤ 200 after max_image_dimension=200");
+ assert!(
+ w <= 200,
+ "screenshot width {w} should be ≤ 200 after max_image_dimension=200"
+ );
+ assert!(
+ h <= 200,
+ "screenshot height {h} should be ≤ 200 after max_image_dimension=200"
+ );
// get_config should reflect the updated value.
d.send(&serde_json::json!({
@@ -572,5 +750,9 @@ fn set_config_screenshot_resize() {
}));
let resp = d.recv();
let dim = resp["result"]["structuredContent"]["max_image_dimension"].as_u64();
- assert_eq!(dim, Some(200), "get_config should reflect max_image_dimension=200");
+ assert_eq!(
+ dim,
+ Some(200),
+ "get_config should reflect max_image_dimension=200"
+ );
}
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/capture.rs b/libs/cua-driver/rust/crates/platform-macos/src/capture.rs
index 22e507a148..80944336fc 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/capture.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/capture.rs
@@ -17,17 +17,28 @@ use std::process::Command;
pub fn screenshot_window_bytes(window_id: u32) -> anyhow::Result<Vec<u8>> {
let tmp_path = format!("/tmp/cua-driver-rs-capture-{}.png", window_id);
- let status = Command::new("screencapture")
+ let output = Command::new("screencapture")
.args([
- "-l", &window_id.to_string(),
- "-x", // no sound
- "-o", // no shadow
+ "-l",
+ &window_id.to_string(),
+ "-x", // no sound
+ "-o", // no shadow
&tmp_path,
])
- .status()?;
-
- if !status.success() {
- anyhow::bail!("screencapture failed for window {window_id}");
+ .output()?;
+
+ if !output.status.success() {
+ let stderr = String::from_utf8_lossy(&output.stderr).trim().to_owned();
+ if stderr.is_empty() {
+ anyhow::bail!(
+ "screencapture failed for window {window_id} with status {}",
+ output.status
+ );
+ }
+ anyhow::bail!(
+ "screencapture failed for window {window_id} with status {}: {stderr}",
+ output.status
+ );
}
let bytes = std::fs::read(&tmp_path)?;
@@ -54,12 +65,22 @@ pub fn screenshot_display_bytes() -> anyhow::Result<Vec<u8>> {
// Use a pid-unique path so concurrent cua-driver processes don't step on each other.
let tmp_path = format!("/tmp/cua-driver-rs-display-{}.png", std::process::id());
- let status = Command::new("screencapture")
+ let output = Command::new("screencapture")
.args(["-x", &*tmp_path])
- .status()?;
-
- if !status.success() {
- anyhow::bail!("screencapture failed for main display");
+ .output()?;
+
+ if !output.status.success() {
+ let stderr = String::from_utf8_lossy(&output.stderr).trim().to_owned();
+ if stderr.is_empty() {
+ anyhow::bail!(
+ "screencapture failed for main display with status {}",
+ output.status
+ );
+ }
+ anyhow::bail!(
+ "screencapture failed for main display with status {}: {stderr}",
+ output.status
+ );
}
let bytes = std::fs::read(&tmp_path)?;
@@ -101,12 +122,7 @@ pub fn resize_png_if_needed(png_bytes: &[u8], max_dim: u32) -> anyhow::Result<Ve
/// `path`. Used by `click`'s `debug_image_out` param to verify
/// coordinate spaces. The crosshair uses top-left-origin coords
/// matching the click tool's convention.
-pub fn write_crosshair_png(
- png_bytes: &[u8],
- cx: f64,
- cy: f64,
- path: &str,
-) -> anyhow::Result<()> {
+pub fn write_crosshair_png(png_bytes: &[u8], cx: f64, cy: f64, path: &str) -> anyhow::Result<()> {
cua_driver_core::image_utils::write_crosshair_png(png_bytes, cx, cy, path)
}
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/focus_steal.rs b/libs/cua-driver/rust/crates/platform-macos/src/focus_steal.rs
index 65ccf875eb..7e67209e55 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/focus_steal.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/focus_steal.rs
@@ -60,8 +60,8 @@ use std::sync::{Arc, Mutex, OnceLock};
use std::time::{Duration, Instant};
use objc2_app_kit::{
- NSApplicationActivationOptions, NSRunningApplication, NSWorkspace,
- NSWorkspaceDidActivateApplicationNotification, NSWorkspaceApplicationKey,
+ NSApplicationActivationOptions, NSRunningApplication, NSWorkspace, NSWorkspaceApplicationKey,
+ NSWorkspaceDidActivateApplicationNotification,
};
use objc2_foundation::NSOperationQueue;
use uuid::Uuid;
@@ -134,9 +134,7 @@ impl FocusStealPreventer {
origin: &'static str,
) -> SuppressionLease {
let shared = Self::shared();
- let handle = shared
- .dispatcher
- .add(target_pid, restore_to, origin);
+ let handle = shared.dispatcher.add(target_pid, restore_to, origin);
SuppressionLease {
handle,
dispatcher: Arc::clone(&shared.dispatcher),
@@ -160,12 +158,6 @@ impl FocusStealPreventer {
let _lease = Self::begin_suppression(target_pid, restore_to, origin);
f().await
}
-
- /// For tests. Returns the singleton's dispatcher arc.
- #[cfg(test)]
- fn dispatcher(&self) -> &Arc<Dispatcher> {
- &self.dispatcher
- }
}
/// Begin-suppression convenience that bounces through the singleton.
@@ -442,10 +434,7 @@ fn install_observer(dispatcher: &Arc<Dispatcher>) {
///
/// Runs on the observer queue's background thread — safe to call
/// blocking system APIs.
-fn handle_activation(
- dispatcher: &Arc<Dispatcher>,
- note: &objc2_foundation::NSNotification,
-) {
+fn handle_activation(dispatcher: &Arc<Dispatcher>, note: &objc2_foundation::NSNotification) {
use objc2::msg_send;
use objc2::runtime::AnyObject;
@@ -457,8 +446,7 @@ fn handle_activation(
// userInfo[NSWorkspaceApplicationKey] -> NSRunningApplication*.
// We go through a raw msg_send to avoid Retained generic
// bookkeeping for the cross-cast.
- let app_ptr: *mut AnyObject =
- msg_send![&*info, objectForKey: NSWorkspaceApplicationKey];
+ let app_ptr: *mut AnyObject = msg_send![&*info, objectForKey: NSWorkspaceApplicationKey];
if app_ptr.is_null() {
return;
}
@@ -476,9 +464,7 @@ fn handle_activation(
/// thread — Apple documents `activateWithOptions:` as thread-safe.
fn restore_focus(pid: i32) {
unsafe {
- if let Some(app) =
- NSRunningApplication::runningApplicationWithProcessIdentifier(pid)
- {
+ if let Some(app) = NSRunningApplication::runningApplicationWithProcessIdentifier(pid) {
let _ = app.activateWithOptions(NSApplicationActivationOptions(0));
}
}
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/input/mouse.rs b/libs/cua-driver/rust/crates/platform-macos/src/input/mouse.rs
index e838f2bd74..47c30f9775 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/input/mouse.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/input/mouse.rs
@@ -22,7 +22,13 @@ use foreign_types::ForeignType;
/// Window-local coordinates for backgrounded targets: if `window_local` is
/// `Some((wx, wy))`, stamps a window-local point via `CGEventSetWindowLocation`
/// SPI so WindowServer's hit-test uses the local point directly.
-pub fn click_at_xy(pid: i32, x: f64, y: f64, count: usize, modifiers: &[&str]) -> anyhow::Result<()> {
+pub fn click_at_xy(
+ pid: i32,
+ x: f64,
+ y: f64,
+ count: usize,
+ modifiers: &[&str],
+) -> anyhow::Result<()> {
click_at_xy_inner(pid, x, y, None, None, count, modifiers)
}
@@ -43,9 +49,21 @@ pub fn click_at_xy_desktop(x: f64, y: f64, count: usize, button: &str) -> anyhow
.map_err(|_| anyhow::anyhow!("CGEventSource::new failed"))?;
let point = CGPoint::new(x, y);
let (down_ty, up_ty, btn) = match button {
- "right" => (CGEventType::RightMouseDown, CGEventType::RightMouseUp, CGMouseButton::Right),
- "middle" => (CGEventType::OtherMouseDown, CGEventType::OtherMouseUp, CGMouseButton::Center),
- _ => (CGEventType::LeftMouseDown, CGEventType::LeftMouseUp, CGMouseButton::Left),
+ "right" => (
+ CGEventType::RightMouseDown,
+ CGEventType::RightMouseUp,
+ CGMouseButton::Right,
+ ),
+ "middle" => (
+ CGEventType::OtherMouseDown,
+ CGEventType::OtherMouseUp,
+ CGMouseButton::Center,
+ ),
+ _ => (
+ CGEventType::LeftMouseDown,
+ CGEventType::LeftMouseUp,
+ CGMouseButton::Left,
+ ),
};
// Warp the REAL cursor to the point first (the macOS peer of Linux XTest's
// pointer warp). A synthetic MouseMoved event does not relocate the hardware
@@ -80,8 +98,10 @@ extern "C" {
/// (f51 / f58 / f91 / f92) for better backgrounded-target delivery.
pub fn click_at_xy_with_window_local(
pid: i32,
- x: f64, y: f64,
- wx: f64, wy: f64,
+ x: f64,
+ y: f64,
+ wx: f64,
+ wy: f64,
wid: u32,
count: usize,
modifiers: &[&str],
@@ -91,7 +111,8 @@ pub fn click_at_xy_with_window_local(
fn click_at_xy_inner(
pid: i32,
- x: f64, y: f64,
+ x: f64,
+ y: f64,
window_local: Option<(f64, f64)>,
wid: Option<u32>,
count: usize,
@@ -128,12 +149,22 @@ fn click_at_xy_inner(
CGEventType::LeftMouseDown,
point,
CGMouseButton::Left,
- ).map_err(|_| anyhow::anyhow!("CGEvent::new_mouse_event(down) failed"))?;
+ )
+ .map_err(|_| anyhow::anyhow!("CGEvent::new_mouse_event(down) failed"))?;
if flags != CGEventFlags::CGEventFlagNull {
down.set_flags(flags);
}
- post_mouse_event(pid, &down, window_local, wid, click_group_id, click_state, 0);
+ post_mouse_event(
+ pid,
+ &down,
+ window_local,
+ wid,
+ click_group_id,
+ click_state,
+ 0,
+ 3,
+ );
// 28 ms down→up gap: an NSButton's mouseDown enters a modal tracking
// loop that polls for the matching mouseUp; too tight a gap can race the
// loop's first poll and the click is dropped. 16 ms was under that
@@ -145,12 +176,22 @@ fn click_at_xy_inner(
CGEventType::LeftMouseUp,
point,
CGMouseButton::Left,
- ).map_err(|_| anyhow::anyhow!("CGEvent::new_mouse_event(up) failed"))?;
+ )
+ .map_err(|_| anyhow::anyhow!("CGEvent::new_mouse_event(up) failed"))?;
if flags != CGEventFlags::CGEventFlagNull {
up.set_flags(flags);
}
- post_mouse_event(pid, &up, window_local, wid, click_group_id, click_state, 0);
+ post_mouse_event(
+ pid,
+ &up,
+ window_local,
+ wid,
+ click_group_id,
+ click_state,
+ 0,
+ 3,
+ );
if count > 1 {
std::thread::sleep(std::time::Duration::from_millis(80));
@@ -161,9 +202,9 @@ fn click_at_xy_inner(
/// Full Chromium-compatible left-click recipe matching Swift's `clickViaAuthSignedPost`.
///
-/// Sequence (Swift reference minus the `activate_without_raise` prologue — that step
-/// causes window-z side effects under Rust's overlay; it is reserved for a future
-/// explicit opt-in once the overlay re-pins after focus changes):
+/// The focus-without-raise prologue makes the target window key without changing
+/// its z-order, which Chromium requires before it accepts a background pixel
+/// mouseDown. The cursor overlay is re-pinned by the click tool after dispatch.
/// 1. Stamped `mouseMoved` at target coords (f0=2, cursor-state primer).
/// 2. Off-screen primer down/up at (-1, -1) (f0=1/2) — satisfies Chromium's
/// user-activation gate without hitting any DOM element.
@@ -193,15 +234,22 @@ pub fn click_at_xy_chromium(
) -> anyhow::Result<()> {
use std::time::{SystemTime, UNIX_EPOCH};
+ // Chromium's first-mouse handling rejects a background click delivered to
+ // a non-key window. This SkyLight focus record keys the requested window
+ // without raising it or moving the user's cursor.
+ if crate::input::skylight::activate_without_raise(pid as libc::pid_t, wid) {
+ std::thread::sleep(std::time::Duration::from_millis(50));
+ }
+
let source = CGEventSource::new(CGEventSourceStateID::HIDSystemState)
.map_err(|_| anyhow::anyhow!("CGEventSource::new failed"))?;
- let target = CGPoint::new(screen_x, screen_y);
+ let target = CGPoint::new(screen_x, screen_y);
let off_screen = CGPoint::new(-1.0, -1.0);
- let win_local = (win_local_x, win_local_y);
- let off_local = (-1.0_f64, -1.0_f64);
- let flags = parse_modifier_flags(modifiers);
+ let win_local = (win_local_x, win_local_y);
+ let off_local = (-1.0_f64, -1.0_f64);
+ let flags = parse_modifier_flags(modifiers);
let click_pairs = count.max(1).min(2);
- let window_id = wid as i64;
+ let window_id = wid as i64;
// All 5 events share the same click-group ID so WindowServer / Chromium
// treat the sequence as one gesture (Swift: field 58).
@@ -214,18 +262,20 @@ pub fn click_at_xy_chromium(
// this closure is Fn (callable multiple times).
let stamp = |event: &CGEvent, local: (f64, f64), click_state: i64, phase: i64| {
let ptr = event.as_ptr() as *mut std::ffi::c_void;
- let set = |f: u32, v: i64| { crate::input::skylight::set_integer_field(ptr, f, v); };
- set(0, phase); // kCGMouseEventNumber (gesture phase)
- set(1, click_state); // kCGMouseEventClickState
- set(3, 0); // kCGMouseEventButtonNumber (left)
- set(7, 3); // kCGMouseEventSubtype (NSEventSubtypeTouch)
- set(40, pid as i64); // Chromium synthetic-event filter
+ let set = |f: u32, v: i64| {
+ crate::input::skylight::set_integer_field(ptr, f, v);
+ };
+ set(0, phase); // kCGMouseEventNumber (gesture phase)
+ set(1, click_state); // kCGMouseEventClickState
+ set(3, 0); // kCGMouseEventButtonNumber (left)
+ set(7, 3); // kCGMouseEventSubtype (NSEventSubtypeTouch)
+ set(40, pid as i64); // Chromium synthetic-event filter
if window_id != 0 {
- set(51, window_id); // windowNumber (NSEvent bridge equivalent)
- set(91, window_id); // kCGMouseEventWindowUnderMousePointer
- set(92, window_id); // kCGMouseEventWindowUnderMousePointerThatCanHandleThisEvent
+ set(51, window_id); // windowNumber (NSEvent bridge equivalent)
+ set(91, window_id); // kCGMouseEventWindowUnderMousePointer
+ set(92, window_id); // kCGMouseEventWindowUnderMousePointerThatCanHandleThisEvent
}
- set(58, click_group_id); // click-group ID (gesture coalescing)
+ set(58, click_group_id); // click-group ID (gesture coalescing)
crate::input::skylight::set_window_location(ptr, local.0, local.1);
if flags != CGEventFlags::CGEventFlagNull {
event.set_flags(flags);
@@ -241,8 +291,12 @@ pub fn click_at_xy_chromium(
// Step 1: mouseMoved at target (phase=2, clickState=0).
let move_event = CGEvent::new_mouse_event(
- source.clone(), CGEventType::MouseMoved, target, CGMouseButton::Left,
- ).map_err(|_| anyhow::anyhow!("mouseMoved event creation failed"))?;
+ source.clone(),
+ CGEventType::MouseMoved,
+ target,
+ CGMouseButton::Left,
+ )
+ .map_err(|_| anyhow::anyhow!("mouseMoved event creation failed"))?;
stamp(&move_event, win_local, 0, 2);
post(&move_event);
std::thread::sleep(std::time::Duration::from_millis(15));
@@ -250,15 +304,23 @@ pub fn click_at_xy_chromium(
// Step 2: off-screen primer click — opens Chromium user-activation gate
// at an off-screen coordinate that can't hit any DOM element.
let primer_down = CGEvent::new_mouse_event(
- source.clone(), CGEventType::LeftMouseDown, off_screen, CGMouseButton::Left,
- ).map_err(|_| anyhow::anyhow!("primer down event creation failed"))?;
+ source.clone(),
+ CGEventType::LeftMouseDown,
+ off_screen,
+ CGMouseButton::Left,
+ )
+ .map_err(|_| anyhow::anyhow!("primer down event creation failed"))?;
stamp(&primer_down, off_local, 1, 1);
post(&primer_down);
std::thread::sleep(std::time::Duration::from_millis(1));
let primer_up = CGEvent::new_mouse_event(
- source.clone(), CGEventType::LeftMouseUp, off_screen, CGMouseButton::Left,
- ).map_err(|_| anyhow::anyhow!("primer up event creation failed"))?;
+ source.clone(),
+ CGEventType::LeftMouseUp,
+ off_screen,
+ CGMouseButton::Left,
+ )
+ .map_err(|_| anyhow::anyhow!("primer up event creation failed"))?;
stamp(&primer_up, off_local, 1, 2);
post(&primer_up);
// ≥1 frame so Chromium sees primer + target as separate gestures, not run-on.
@@ -270,15 +332,23 @@ pub fn click_at_xy_chromium(
let click_state = pair_index as i64;
let down = CGEvent::new_mouse_event(
- source.clone(), CGEventType::LeftMouseDown, target, CGMouseButton::Left,
- ).map_err(|_| anyhow::anyhow!("target down event creation failed"))?;
+ source.clone(),
+ CGEventType::LeftMouseDown,
+ target,
+ CGMouseButton::Left,
+ )
+ .map_err(|_| anyhow::anyhow!("target down event creation failed"))?;
stamp(&down, win_local, click_state, 3);
post(&down);
std::thread::sleep(std::time::Duration::from_millis(1));
let up = CGEvent::new_mouse_event(
- source.clone(), CGEventType::LeftMouseUp, target, CGMouseButton::Left,
- ).map_err(|_| anyhow::anyhow!("target up event creation failed"))?;
+ source.clone(),
+ CGEventType::LeftMouseUp,
+ target,
+ CGMouseButton::Left,
+ )
+ .map_err(|_| anyhow::anyhow!("target up event creation failed"))?;
stamp(&up, win_local, click_state, 3);
post(&up);
@@ -314,7 +384,9 @@ pub fn drag_at_xy(
steps: usize,
modifiers: &[&str],
button: DragButton,
+ foreground_release: bool,
) -> anyhow::Result<()> {
+ use core_graphics::event::CGEventTapLocation;
use std::time::{SystemTime, UNIX_EPOCH};
let source = CGEventSource::new(CGEventSourceStateID::HIDSystemState)
@@ -322,12 +394,24 @@ pub fn drag_at_xy(
let flags = parse_modifier_flags(modifiers);
let (cg_button, down_type, dragged_type, up_type) = match button {
- DragButton::Left => (CGMouseButton::Left, CGEventType::LeftMouseDown,
- CGEventType::LeftMouseDragged, CGEventType::LeftMouseUp),
- DragButton::Right => (CGMouseButton::Right, CGEventType::RightMouseDown,
- CGEventType::RightMouseDragged, CGEventType::RightMouseUp),
- DragButton::Middle => (CGMouseButton::Center, CGEventType::OtherMouseDown,
- CGEventType::OtherMouseDragged, CGEventType::OtherMouseUp),
+ DragButton::Left => (
+ CGMouseButton::Left,
+ CGEventType::LeftMouseDown,
+ CGEventType::LeftMouseDragged,
+ CGEventType::LeftMouseUp,
+ ),
+ DragButton::Right => (
+ CGMouseButton::Right,
+ CGEventType::RightMouseDown,
+ CGEventType::RightMouseDragged,
+ CGEventType::RightMouseUp,
+ ),
+ DragButton::Middle => (
+ CGMouseButton::Center,
+ CGEventType::OtherMouseDown,
+ CGEventType::OtherMouseDragged,
+ CGEventType::OtherMouseUp,
+ ),
};
// f3 button number must match the dragged button (0=left, 1=right, 2=middle).
let button_number: i64 = match button {
@@ -344,14 +428,29 @@ pub fn drag_at_xy(
});
let steps = steps.max(1);
- let step_delay_ms = if steps > 1 { duration_ms / steps as u64 } else { duration_ms };
+ let step_delay_ms = if steps > 1 {
+ duration_ms / steps as u64
+ } else {
+ duration_ms
+ };
// MouseDown at start.
let from_pt = CGPoint::new(from_x, from_y);
let down = CGEvent::new_mouse_event(source.clone(), down_type, from_pt, cg_button)
.map_err(|_| anyhow::anyhow!("drag mouseDown failed"))?;
- if flags != CGEventFlags::CGEventFlagNull { down.set_flags(flags); }
- post_mouse_event(pid, &down, from_local, wid, click_group_id, 1, button_number);
+ if flags != CGEventFlags::CGEventFlagNull {
+ down.set_flags(flags);
+ }
+ post_mouse_event(
+ pid,
+ &down,
+ from_local,
+ wid,
+ click_group_id,
+ 1,
+ button_number,
+ 0,
+ );
std::thread::sleep(std::time::Duration::from_millis(16));
// Interpolated drag steps.
@@ -359,29 +458,163 @@ pub fn drag_at_xy(
let t = i as f64 / steps as f64;
let ix = from_x + (to_x - from_x) * t;
let iy = from_y + (to_y - from_y) * t;
- let il = from_local.zip(to_local).map(|((fx, fy), (tx, ty))| {
- (fx + (tx - fx) * t, fy + (ty - fy) * t)
- });
+ let il = from_local
+ .zip(to_local)
+ .map(|((fx, fy), (tx, ty))| (fx + (tx - fx) * t, fy + (ty - fy) * t));
let drag_pt = CGPoint::new(ix, iy);
let drag = CGEvent::new_mouse_event(source.clone(), dragged_type, drag_pt, cg_button)
.map_err(|_| anyhow::anyhow!("drag mouseDragged failed"))?;
- if flags != CGEventFlags::CGEventFlagNull { drag.set_flags(flags); }
- post_mouse_event(pid, &drag, il, wid, click_group_id, 1, button_number);
+ if flags != CGEventFlags::CGEventFlagNull {
+ drag.set_flags(flags);
+ }
+ post_mouse_event(pid, &drag, il, wid, click_group_id, 1, button_number, 0);
if step_delay_ms > 0 {
std::thread::sleep(std::time::Duration::from_millis(step_delay_ms));
}
}
// MouseUp at end.
+ // Give Chromium one run-loop turn to process the final dragged event at
+ // the drop point before releasing pointer capture.
+ std::thread::sleep(std::time::Duration::from_millis(50));
let to_pt = CGPoint::new(to_x, to_y);
let up = CGEvent::new_mouse_event(source.clone(), up_type, to_pt, cg_button)
.map_err(|_| anyhow::anyhow!("drag mouseUp failed"))?;
- if flags != CGEventFlags::CGEventFlagNull { up.set_flags(flags); }
- post_mouse_event(pid, &up, to_local, wid, click_group_id, 1, button_number);
+ if flags != CGEventFlags::CGEventFlagNull {
+ up.set_flags(flags);
+ }
+ post_mouse_event(pid, &up, to_local, wid, click_group_id, 1, button_number, 0);
+ if foreground_release {
+ // A frontmost Chromium surface can consume PID-routed down/move
+ // events yet filter the synthetic release. Re-post only the release
+ // through the HID tap while the foreground assist still holds focus.
+ up.post(CGEventTapLocation::HID);
+ }
+ // Chromium may process the final pointerup on the next run-loop turn. In
+ // the foreground rung the caller restores the previous app immediately
+ // after this function returns, so let the target consume the release and
+ // complete pointer capture before that restore.
+ std::thread::sleep(std::time::Duration::from_millis(100));
Ok(())
}
+/// Foreground drag through the global HID event tap.
+///
+/// A frontmost Chromium/WebKit surface expects a real HID-origin gesture for
+/// pointer capture and drag tracking. PID-routed `post_to_pid` events are
+/// suitable for background delivery, but they can be silently filtered by the
+/// renderer even when the target is frontmost.
+pub fn drag_at_xy_foreground(
+ from_x: f64,
+ from_y: f64,
+ to_x: f64,
+ to_y: f64,
+ duration_ms: u64,
+ steps: usize,
+ modifiers: &[&str],
+ button: DragButton,
+) -> anyhow::Result<()> {
+ use core_graphics::display::CGDisplay;
+ use core_graphics::event::CGEventTapLocation;
+
+ let source = CGEventSource::new(CGEventSourceStateID::HIDSystemState)
+ .map_err(|_| anyhow::anyhow!("CGEventSource::new failed"))?;
+ let flags = parse_modifier_flags(modifiers);
+ let (cg_button, down_type, dragged_type, up_type) = match button {
+ DragButton::Left => (
+ CGMouseButton::Left,
+ CGEventType::LeftMouseDown,
+ CGEventType::LeftMouseDragged,
+ CGEventType::LeftMouseUp,
+ ),
+ DragButton::Right => (
+ CGMouseButton::Right,
+ CGEventType::RightMouseDown,
+ CGEventType::RightMouseDragged,
+ CGEventType::RightMouseUp,
+ ),
+ DragButton::Middle => (
+ CGMouseButton::Center,
+ CGEventType::OtherMouseDown,
+ CGEventType::OtherMouseDragged,
+ CGEventType::OtherMouseUp,
+ ),
+ };
+ let steps = steps.max(1);
+ let step_delay_ms = if steps > 1 {
+ duration_ms / steps as u64
+ } else {
+ duration_ms
+ };
+
+ let post = |event: &CGEvent| event.post(CGEventTapLocation::HID);
+
+ // Keep WindowServer's hardware cursor and event stream coupled. AppKit
+ // hit-tests some pointer-capture surfaces against the actual cursor even
+ // when the HID event carries an explicit location.
+ let _ = CGDisplay::warp_mouse_cursor_position(CGPoint::new(from_x, from_y));
+ unsafe { CGAssociateMouseAndMouseCursorPosition(true) };
+ std::thread::sleep(std::time::Duration::from_millis(40));
+
+ // Prime the renderer's tracking state with a genuine HID mouse move.
+ if let Ok(move_event) = CGEvent::new_mouse_event(
+ source.clone(),
+ CGEventType::MouseMoved,
+ CGPoint::new(from_x, from_y),
+ cg_button,
+ ) {
+ post(&move_event);
+ }
+ std::thread::sleep(std::time::Duration::from_millis(30));
+
+ let down = CGEvent::new_mouse_event(
+ source.clone(),
+ down_type,
+ CGPoint::new(from_x, from_y),
+ cg_button,
+ )
+ .map_err(|_| anyhow::anyhow!("foreground drag mouseDown failed"))?;
+ if flags != CGEventFlags::CGEventFlagNull {
+ down.set_flags(flags);
+ }
+ down.set_integer_value_field(core_graphics::event::EventField::MOUSE_EVENT_CLICK_STATE, 1);
+ down.post(CGEventTapLocation::HID);
+ std::thread::sleep(std::time::Duration::from_millis(16));
+
+ for i in 1..=steps {
+ let t = i as f64 / steps as f64;
+ let event = CGEvent::new_mouse_event(
+ source.clone(),
+ dragged_type,
+ CGPoint::new(from_x + (to_x - from_x) * t, from_y + (to_y - from_y) * t),
+ cg_button,
+ )
+ .map_err(|_| anyhow::anyhow!("foreground drag mouseDragged failed"))?;
+ if flags != CGEventFlags::CGEventFlagNull {
+ event.set_flags(flags);
+ }
+ event.set_integer_value_field(core_graphics::event::EventField::MOUSE_EVENT_CLICK_STATE, 1);
+ post(&event);
+ if step_delay_ms > 0 {
+ std::thread::sleep(std::time::Duration::from_millis(step_delay_ms));
+ }
+ }
+
+ let up = CGEvent::new_mouse_event(source, up_type, CGPoint::new(to_x, to_y), cg_button)
+ .map_err(|_| anyhow::anyhow!("foreground drag mouseUp failed"))?;
+ if flags != CGEventFlags::CGEventFlagNull {
+ up.set_flags(flags);
+ }
+ up.set_integer_value_field(core_graphics::event::EventField::MOUSE_EVENT_CLICK_STATE, 1);
+ post(&up);
+ // The foreground wrapper restores the previous app immediately after this
+ // function returns. Let the target's run loop consume the queued HID
+ // gesture, including pointer-capture release, before that restore happens.
+ std::thread::sleep(std::time::Duration::from_millis(100));
+ Ok(())
+}
+
/// Mouse button for drag gestures.
#[derive(Clone, Copy, Debug)]
pub enum DragButton {
@@ -403,8 +636,10 @@ pub fn middle_click_at_xy(pid: i32, x: f64, y: f64, modifiers: &[&str]) -> anyho
/// Like `middle_click_at_xy` but stamps the window-local `(wx, wy)` point.
pub fn middle_click_at_xy_with_window_local(
pid: i32,
- x: f64, y: f64,
- wx: f64, wy: f64,
+ x: f64,
+ y: f64,
+ wx: f64,
+ wy: f64,
modifiers: &[&str],
) -> anyhow::Result<()> {
middle_click_at_xy_inner(pid, x, y, Some((wx, wy)), modifiers)
@@ -412,7 +647,8 @@ pub fn middle_click_at_xy_with_window_local(
fn middle_click_at_xy_inner(
pid: i32,
- x: f64, y: f64,
+ x: f64,
+ y: f64,
window_local: Option<(f64, f64)>,
modifiers: &[&str],
) -> anyhow::Result<()> {
@@ -426,11 +662,12 @@ fn middle_click_at_xy_inner(
CGEventType::OtherMouseDown,
point,
CGMouseButton::Center,
- ).map_err(|_| anyhow::anyhow!("middle mouse down failed"))?;
+ )
+ .map_err(|_| anyhow::anyhow!("middle mouse down failed"))?;
if flags != CGEventFlags::CGEventFlagNull {
down.set_flags(flags);
}
- post_mouse_event(pid, &down, window_local, None, None, 1, 2);
+ post_mouse_event(pid, &down, window_local, None, None, 1, 2, 3);
std::thread::sleep(std::time::Duration::from_millis(16));
let up = CGEvent::new_mouse_event(
@@ -438,11 +675,12 @@ fn middle_click_at_xy_inner(
CGEventType::OtherMouseUp,
point,
CGMouseButton::Center,
- ).map_err(|_| anyhow::anyhow!("middle mouse up failed"))?;
+ )
+ .map_err(|_| anyhow::anyhow!("middle mouse up failed"))?;
if flags != CGEventFlags::CGEventFlagNull {
up.set_flags(flags);
}
- post_mouse_event(pid, &up, window_local, None, None, 1, 2);
+ post_mouse_event(pid, &up, window_local, None, None, 1, 2, 3);
Ok(())
}
@@ -463,8 +701,10 @@ pub fn right_click_at_xy(pid: i32, x: f64, y: f64, modifiers: &[&str]) -> anyhow
/// already threaded `wid`; right-click did not, which is why it broke.
pub fn right_click_at_xy_with_window_local(
pid: i32,
- x: f64, y: f64,
- wx: f64, wy: f64,
+ x: f64,
+ y: f64,
+ wx: f64,
+ wy: f64,
wid: u32,
modifiers: &[&str],
) -> anyhow::Result<()> {
@@ -473,7 +713,8 @@ pub fn right_click_at_xy_with_window_local(
fn right_click_at_xy_inner(
pid: i32,
- x: f64, y: f64,
+ x: f64,
+ y: f64,
window_local: Option<(f64, f64)>,
wid: Option<u32>,
modifiers: &[&str],
@@ -502,13 +743,14 @@ fn right_click_at_xy_inner(
CGEventType::RightMouseDown,
point,
CGMouseButton::Right,
- ).map_err(|_| anyhow::anyhow!("right mouse down failed"))?;
+ )
+ .map_err(|_| anyhow::anyhow!("right mouse down failed"))?;
if flags != CGEventFlags::CGEventFlagNull {
down.set_flags(flags);
}
// button_number = 1 (right). Stamping 0 here routes the event as a left
// button-number on the receiving side even though the type is rightMouseDown.
- post_mouse_event(pid, &down, window_local, wid, click_group_id, 1, 1);
+ post_mouse_event(pid, &down, window_local, wid, click_group_id, 1, 1, 3);
std::thread::sleep(std::time::Duration::from_millis(28));
let up = CGEvent::new_mouse_event(
@@ -516,11 +758,12 @@ fn right_click_at_xy_inner(
CGEventType::RightMouseUp,
point,
CGMouseButton::Right,
- ).map_err(|_| anyhow::anyhow!("right mouse up failed"))?;
+ )
+ .map_err(|_| anyhow::anyhow!("right mouse up failed"))?;
if flags != CGEventFlags::CGEventFlagNull {
up.set_flags(flags);
}
- post_mouse_event(pid, &up, window_local, wid, click_group_id, 1, 1);
+ post_mouse_event(pid, &up, window_local, wid, click_group_id, 1, 1, 3);
Ok(())
}
@@ -557,6 +800,7 @@ fn post_mouse_event(
click_group_id: Option<i64>,
click_state: i64,
button_number: i64,
+ subtype: i64,
) {
let event_ptr = event.as_ptr() as *mut std::ffi::c_void;
@@ -568,14 +812,16 @@ fn post_mouse_event(
// Chromium / AppKit window-routing fields — stamp when window_id is known.
if let (Some(wid), Some(cgid)) = (wid, click_group_id) {
let window_id = wid as i64;
- let set = |f: u32, v: i64| { crate::input::skylight::set_integer_field(event_ptr, f, v); };
- set(1, click_state); // kCGMouseEventClickState
- set(3, button_number); // kCGMouseEventButtonNumber (0=left, 1=right, 2=middle)
- set(7, 3); // kCGMouseEventSubtype (NSEventSubtypeTouch)
- set(51, window_id); // windowNumber
- set(58, cgid); // click-group ID (gesture coalescing)
- set(91, window_id); // kCGMouseEventWindowUnderMousePointer
- set(92, window_id); // kCGMouseEventWindowUnderMousePointerThatCanHandleThisEvent
+ let set = |f: u32, v: i64| {
+ crate::input::skylight::set_integer_field(event_ptr, f, v);
+ };
+ set(1, click_state); // kCGMouseEventClickState
+ set(3, button_number); // kCGMouseEventButtonNumber (0=left, 1=right, 2=middle)
+ set(7, subtype); // kCGMouseEventSubtype (touch for clicks, normal for drags)
+ set(51, window_id); // windowNumber
+ set(58, cgid); // click-group ID (gesture coalescing)
+ set(91, window_id); // kCGMouseEventWindowUnderMousePointer
+ set(92, window_id); // kCGMouseEventWindowUnderMousePointerThatCanHandleThisEvent
}
// Always stamp f40 = target pid (Chromium synthetic-event filter).
@@ -615,14 +861,14 @@ fn post_mouse_moved_primer(
point,
CGMouseButton::Left,
) {
- post_mouse_event(pid, &mv, window_local, wid, click_group_id, 0, 0);
+ post_mouse_event(pid, &mv, window_local, wid, click_group_id, 0, 0, 3);
}
}
-/// Synthesize a **pixel-unit mouse-wheel** scroll at `(screen_x, screen_y)`,
+/// Synthesize a targeted mouse-wheel scroll at `(screen_x, screen_y)`,
/// posted to `pid`.
///
-/// This is the pixel-wheel scroll path. Unlike the keystroke path (PageDown /
+/// This is the targeted wheel path. Unlike the keystroke path (PageDown /
/// arrow keys), which only ever drives the *focused* / page scroller, a real
/// wheel event is hit-tested by the renderer at the cursor point: whatever
/// element sits under `(screen_x, screen_y)` receives the scroll. That is the
@@ -630,11 +876,11 @@ fn post_mouse_moved_primer(
/// therefore can never take keyboard focus (verified no-op via keystrokes on
/// WKWebView's inner `scroll-tall` and WebView2).
///
-/// `CGEventCreateScrollWheelEvent2` builds the event in `kCGScrollEventUnitPixel`
-/// units; we then anchor it at the target point with `CGEventSetLocation` so the
-/// renderer routes it correctly, and stamp the same background-delivery fields
-/// the click primitives use (window-local point + f40 pid filter + window-routing
-/// fields) so it reaches backgrounded Chromium/Catalyst/WKWebView targets.
+/// `CGEventCreateScrollWheelEvent2` builds a line-unit event; we then anchor it
+/// at the target point with `CGEventSetLocation` so the renderer routes it
+/// correctly, and stamp the same background-delivery fields the click
+/// primitives use (window-local point + f40 pid filter + window-routing fields)
+/// so it reaches backgrounded Chromium/Catalyst/WKWebView targets.
///
/// Sign convention (macOS): a POSITIVE `delta_y_per_tick` scrolls the content
/// toward the top (reveals content ABOVE); NEGATIVE reveals content BELOW.
@@ -651,7 +897,8 @@ fn post_mouse_moved_primer(
/// identical in spirit to `post_mouse_event`.
pub fn scroll_wheel_at_xy(
pid: i32,
- screen_x: f64, screen_y: f64,
+ screen_x: f64,
+ screen_y: f64,
window_local: Option<(f64, f64)>,
wid: Option<u32>,
delta_y_per_tick: i32,
@@ -660,21 +907,45 @@ pub fn scroll_wheel_at_xy(
) -> anyhow::Result<()> {
use core_graphics::event::ScrollEventUnit;
+ // Prime AppKit/WebKit's tracking state at the target before the wheel
+ // gesture. Background windows can retain a stale hit-test location; a
+ // nested overflow scroller then receives the wheel nowhere even though
+ // the event is successfully posted to the target pid.
+ let primer_source = CGEventSource::new(CGEventSourceStateID::HIDSystemState)
+ .map_err(|_| anyhow::anyhow!("CGEventSource::new failed"))?;
+ post_mouse_moved_primer(
+ pid,
+ &primer_source,
+ CGPoint::new(screen_x, screen_y),
+ window_local,
+ wid,
+ Some(
+ std::time::SystemTime::now()
+ .duration_since(std::time::UNIX_EPOCH)
+ .unwrap_or_default()
+ .subsec_nanos() as i64,
+ ),
+ );
+ std::thread::sleep(std::time::Duration::from_millis(12));
+
for _ in 0..ticks.max(1) {
// Fresh source per event, matching the click primitives.
let source = CGEventSource::new(CGEventSourceStateID::HIDSystemState)
.map_err(|_| anyhow::anyhow!("CGEventSource::new failed"))?;
// wheel_count = 2 → both axes carried (vertical = wheel1/axis-1,
- // horizontal = wheel2/axis-2). PIXEL units so the deltas are device
- // pixels, matching a trackpad/precise wheel rather than coarse notches.
+ // horizontal = wheel2/axis-2). Convert the driver's pixel-tuned step
+ // into a bounded line delta for the event API.
+ let wheel_y = (delta_y_per_tick / 120).clamp(-10, 10);
+ let wheel_x = (delta_x_per_tick / 120).clamp(-10, 10);
let event = CGEvent::new_scroll_event(
source,
- ScrollEventUnit::PIXEL,
+ ScrollEventUnit::LINE,
2,
- delta_y_per_tick,
- delta_x_per_tick,
+ wheel_y,
+ wheel_x,
0,
- ).map_err(|_| anyhow::anyhow!("CGEvent::new_scroll_event failed"))?;
+ )
+ .map_err(|_| anyhow::anyhow!("CGEvent::new_scroll_event failed"))?;
let event_ptr = event.as_ptr() as *mut std::ffi::c_void;
@@ -688,7 +959,9 @@ pub fn scroll_wheel_at_xy(
}
if let Some(wid) = wid {
let window_id = wid as i64;
- let set = |f: u32, v: i64| { crate::input::skylight::set_integer_field(event_ptr, f, v); };
+ let set = |f: u32, v: i64| {
+ crate::input::skylight::set_integer_field(event_ptr, f, v);
+ };
set(51, window_id); // windowNumber
set(91, window_id); // kCGMouseEventWindowUnderMousePointer
set(92, window_id); // ...ThatCanHandleThisEvent
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/click.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/click.rs
index 89e9566704..f18b8a36c8 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/click.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/click.rs
@@ -14,15 +14,17 @@
//! to full-window space using the most recent `zoom` context stored per-pid.
use async_trait::async_trait;
-use cua_driver_core::{protocol::ToolResult, tool::{Tool, ToolDef}};
+use cua_driver_core::{
+ protocol::ToolResult,
+ tool::{Tool, ToolDef},
+};
use serde_json::Value;
use std::sync::Arc;
+use crate::apps;
use crate::ax::bindings::{
- copy_action_names, copy_children, copy_string_attr, element_screen_rect,
- AXUIElementRef,
+ copy_action_names, copy_children, copy_string_attr, element_screen_rect, AXUIElementRef,
};
-use crate::apps;
use crate::focus_guard;
use crate::window_change_detector::WindowChangeDetector;
use core_foundation::base::CFRelease;
@@ -34,7 +36,9 @@ pub struct ClickTool {
}
impl ClickTool {
- pub fn new(state: Arc<ToolState>) -> Self { Self { state } }
+ pub fn new(state: Arc<ToolState>) -> Self {
+ Self { state }
+ }
}
static DEF: std::sync::OnceLock<ToolDef> = std::sync::OnceLock::new();
@@ -126,7 +130,9 @@ fn def() -> &'static ToolDef {
#[async_trait]
impl Tool for ClickTool {
- fn def(&self) -> &ToolDef { def() }
+ fn def(&self) -> &ToolDef {
+ def()
+ }
async fn invoke(&self, args: Value) -> ToolResult {
use cua_driver_core::tool_args::ArgsExt;
@@ -160,8 +166,14 @@ impl Tool for ClickTool {
"suggestion": "pass scope=\"desktop\"",
}));
}
- let sx_shot = args.opt_f64("x").or_else(|| args.opt_i64("x").map(|i| i as f64)).unwrap_or(0.0);
- let sy_shot = args.opt_f64("y").or_else(|| args.opt_i64("y").map(|i| i as f64)).unwrap_or(0.0);
+ let sx_shot = args
+ .opt_f64("x")
+ .or_else(|| args.opt_i64("x").map(|i| i as f64))
+ .unwrap_or(0.0);
+ let sy_shot = args
+ .opt_f64("y")
+ .or_else(|| args.opt_i64("y").map(|i| i as f64))
+ .unwrap_or(0.0);
// ── Desktop-screenshot pixels → logical screen points ──────────────
// The vision invariant: the pixel an agent reads off the screenshot it
// was handed is the pixel that gets clicked. `get_desktop_state`
@@ -178,7 +190,8 @@ impl Tool for ClickTool {
// CGDisplayPixelsWide under-reports the backing scale (it returns the
// scaled-mode point width on some Retina configs → a bogus 1.0).
let desktop_ratio = tokio::task::spawn_blocking(|| {
- let logical_w = super::get_screen_size::main_screen_size().map(|(w, _, _)| w as f64);
+ let logical_w =
+ super::get_screen_size::main_screen_size().map(|(w, _, _)| w as f64);
let shot_w = crate::capture::screenshot_display_bytes()
.ok()
.and_then(|png| crate::capture::png_dimensions(&png).ok())
@@ -207,7 +220,9 @@ impl Tool for ClickTool {
// Glide the session's agent cursor to the screen point for visibility.
let cursor_key = super::cursor_tools::resolve_cursor_key(&args);
crate::cursor::overlay::animate_cursor_to(cursor_key.clone(), sx, sy).await;
- self.state.cursor_registry.update_position(&cursor_key, sx, sy);
+ self.state
+ .cursor_registry
+ .update_position(&cursor_key, sx, sy);
// Resolve the frontmost on-screen window under the point (the macOS
// peer of Windows' WindowFromPoint). When found, click THAT pid via
@@ -231,14 +246,16 @@ impl Tool for ClickTool {
// (Ascending picked the BACKMOST occluded window — a real miss when
// windows overlap, e.g. resolving a click to a buried app.)
wins.sort_by(|a, b| b.z_index.cmp(&a.z_index)); // front-to-back
- wins.into_iter().find(|w| {
- w.layer == 0
- && w.pid != own_pid
- && sx >= w.bounds.x
- && sx < w.bounds.x + w.bounds.width
- && sy >= w.bounds.y
- && sy < w.bounds.y + w.bounds.height
- }).map(|w| (w.pid, w.window_id, w.bounds.x, w.bounds.y))
+ wins.into_iter()
+ .find(|w| {
+ w.layer == 0
+ && w.pid != own_pid
+ && sx >= w.bounds.x
+ && sx < w.bounds.x + w.bounds.width
+ && sy >= w.bounds.y
+ && sy < w.bounds.y + w.bounds.height
+ })
+ .map(|w| (w.pid, w.window_id, w.bounds.x, w.bounds.y))
};
let btn = button.clone();
let result = tokio::task::spawn_blocking(move || -> anyhow::Result<Option<i32>> {
@@ -252,13 +269,31 @@ impl Tool for ClickTool {
// as the pixel path); `count` only repeats on the left path.
match btn.as_str() {
"right" => crate::input::mouse::right_click_at_xy_with_window_local(
- pid, sx, sy, wx, wy, wid, &[],
+ pid,
+ sx,
+ sy,
+ wx,
+ wy,
+ wid,
+ &[],
)?,
"middle" => crate::input::mouse::middle_click_at_xy_with_window_local(
- pid, sx, sy, wx, wy, &[],
+ pid,
+ sx,
+ sy,
+ wx,
+ wy,
+ &[],
)?,
_ => crate::input::mouse::click_at_xy_with_window_local(
- pid, sx, sy, wx, wy, wid, count, &[],
+ pid,
+ sx,
+ sy,
+ wx,
+ wy,
+ wid,
+ count,
+ &[],
)?,
}
Ok(Some(pid))
@@ -293,7 +328,10 @@ impl Tool for ClickTool {
};
}
- let pid = match args.require_i32("pid") { Ok(v) => v, Err(e) => return e };
+ let pid = match args.require_i32("pid") {
+ Ok(v) => v,
+ Err(e) => return e,
+ };
// Resolve this action's cursor key so its click-pulse / glide land on
// the calling session's cursor, not the shared "default" one.
let cursor_key = super::cursor_tools::resolve_cursor_key(&args);
@@ -303,7 +341,7 @@ impl Tool for ClickTool {
// stale token returns an explicit error instead of silently
// falling back to the integer (Surface 6 hard constraint).
let element_token_arg = args.opt_str("element_token");
- let window_id_arg = args.opt_u64("window_id").map(|v| v as u32);
+ let window_id_arg = args.opt_u64("window_id").map(|v| v as u32);
let element_index_arg = args.opt_u64("element_index").map(|v| v as usize);
let resolved = match cua_driver_core::element_token::resolve_element_args(
pid,
@@ -318,18 +356,24 @@ impl Tool for ClickTool {
let (element_index, window_id, _via_token) = match resolved {
cua_driver_core::element_token::ResolvedElement::None => (None, window_id_arg, false),
cua_driver_core::element_token::ResolvedElement::Element {
- window_id: wid, element_index: idx, via_token,
+ window_id: wid,
+ element_index: idx,
+ via_token,
} => (Some(idx), wid, via_token),
};
- let x = args.opt_f64("x").or_else(|| args.opt_i64("x").map(|i| i as f64));
- let y = args.opt_f64("y").or_else(|| args.opt_i64("y").map(|i| i as f64));
- let action = args.str_or("action", "press");
+ let x = args
+ .opt_f64("x")
+ .or_else(|| args.opt_i64("x").map(|i| i as f64));
+ let y = args
+ .opt_f64("y")
+ .or_else(|| args.opt_i64("y").map(|i| i as f64));
+ let action = args.str_or("action", "press");
// Surface 5: optional `button` arg, default "left" preserves legacy behaviour.
// Pixel path: routes to left/right/middle CGEvent primitives.
// AX path: "right" delegates to AXShowMenu (same surface as right_click);
// "middle" has no AX equivalent and falls back to a pixel middle-click
// at the element's screen-space center.
- let button_str = args.str_or("button", "left").to_lowercase();
+ let button_str = args.str_or("button", "left").to_lowercase();
// delivery_mode: per-call ladder rung. foreground only applies to the
// pixel path and needs a window_id to front (else it degrades to
// background). A click is never driver-verifiable either way.
@@ -341,9 +385,13 @@ impl Tool for ClickTool {
"click: unknown button \"{button_str}\" — expected one of left, right, middle."
));
}
- let button_str = if button_str.is_empty() { "left".to_string() } else { button_str };
- let count = args.u64_or("count", 1) as usize;
- let from_zoom = args.bool_or("from_zoom", false);
+ let button_str = if button_str.is_empty() {
+ "left".to_string()
+ } else {
+ button_str
+ };
+ let count = args.u64_or("count", 1) as usize;
+ let from_zoom = args.bool_or("from_zoom", false);
let debug_image_out = args.opt_str("debug_image_out");
let modifiers: Vec<String> = args.str_array("modifier");
@@ -355,10 +403,12 @@ impl Tool for ClickTool {
// guard lives to the end of this method, past the AX action below.
let element_guard = match self.state.element_cache.get_element_retained(pid, wid, idx) {
Some(e) => e,
- None => return ToolResult::error(format!(
- "Element index {idx} not found in cache for pid={pid} window_id={wid}. \
+ None => {
+ return ToolResult::error(format!(
+ "Element index {idx} not found in cache for pid={pid} window_id={wid}. \
Call get_window_state first."
- )),
+ ))
+ }
};
let element_ptr = element_guard.as_ptr();
@@ -376,7 +426,10 @@ impl Tool for ClickTool {
let center_ptr = element_ptr;
let center = tokio::task::spawn_blocking(move || unsafe {
crate::ax::bindings::element_screen_center(center_ptr as AXUIElementRef)
- }).await.ok().flatten();
+ })
+ .await
+ .ok()
+ .flatten();
// Surface 5: button=middle on the AX path has no AX equivalent.
// Fall back to a pixel middle-click at the element's screen-space center
@@ -386,23 +439,28 @@ impl Tool for ClickTool {
if button_str == "middle" {
let (cx, cy) = match center {
Some(c) => c,
- None => return ToolResult::error(
- "click(button=middle) on element_index: could not resolve element \
- center for the pixel-middle-click fallback. Pass x, y directly."
- ),
+ None => {
+ return ToolResult::error(
+ "click(button=middle) on element_index: could not resolve element \
+ center for the pixel-middle-click fallback. Pass x, y directly.",
+ )
+ }
};
crate::cursor::overlay::send_command(
cursor_key.clone(),
cursor_overlay::OverlayCommand::PinAbove(wid as u64),
);
crate::cursor::overlay::animate_cursor_to(cursor_key.clone(), cx, cy).await;
- self.state.cursor_registry.update_position(&cursor_key, cx, cy);
+ self.state
+ .cursor_registry
+ .update_position(&cursor_key, cx, cy);
let mods_owned = modifiers.clone();
let result = tokio::task::spawn_blocking(move || {
let m: Vec<&str> = mods_owned.iter().map(String::as_str).collect();
crate::input::mouse::middle_click_at_xy(pid, cx, cy, &m)
- }).await;
+ })
+ .await;
return match result {
Ok(Ok(())) => ToolResult::text(format!(
"✅ Posted middle-click to pid {pid} at element [{idx}] center \
@@ -424,7 +482,9 @@ impl Tool for ClickTool {
// Keep the registry in sync with the overlay so
// get_agent_cursor_state reports a truthful position even when
// the click was dispatched via the AX path (no pixel coords).
- self.state.cursor_registry.update_position(&cursor_key, cx, cy);
+ self.state
+ .cursor_registry
+ .update_position(&cursor_key, cx, cy);
}
// ── Focus-suppression wrap (Swift WindowChangeDetector + FocusGuard) ──
@@ -494,7 +554,7 @@ impl Tool for ClickTool {
ToolResult::text(msg).with_structured(structured)
}
Ok(Err(e)) => ToolResult::error(format!("AX action failed: {e}")),
- Err(e) => ToolResult::error(format!("Task error: {e}")),
+ Err(e) => ToolResult::error(format!("Task error: {e}")),
}
} else if let (Some(mut cx), Some(mut cy)) = (x, y) {
// ── Pixel path ─────────────────────────────────────────────────
@@ -506,35 +566,37 @@ impl Tool for ClickTool {
if from_zoom {
return ToolResult::error(
"debug_image_out is incompatible with from_zoom — \
- received (x, y) would be in zoom-crop space, not window-local."
+ received (x, y) would be in zoom-crop space, not window-local.",
);
}
match window_id {
- None => return ToolResult::error(
- "debug_image_out requires window_id."
- ),
+ None => return ToolResult::error("debug_image_out requires window_id."),
Some(wid) => {
// Session-effective max dimension so debug_image_out
// matches the resize the calling session sees in
// get_window_state (precedence: session override > global).
- let max_dim = self.state.session_config
- .effective_max_image_dimension(
- args.opt_str("_session_id").as_deref(),
- &self.state.config.read().unwrap(),
- );
+ let max_dim = self.state.session_config.effective_max_image_dimension(
+ args.opt_str("_session_id").as_deref(),
+ &self.state.config.read().unwrap(),
+ );
let dbg_path_c = dbg_path.clone();
let dbg_result = tokio::task::spawn_blocking(move || {
let png = crate::capture::screenshot_window_bytes(wid)?;
let png = crate::capture::resize_png_if_needed(&png, max_dim)?;
crate::capture::write_crosshair_png(&png, cx, cy, &dbg_path_c)
- }).await;
+ })
+ .await;
match dbg_result {
- Err(e) => return ToolResult::error(format!(
- "debug_image_out task failed: {e}. Not dispatching click."
- )),
- Ok(Err(e)) => return ToolResult::error(format!(
- "debug_image_out write failed: {e}. Not dispatching click."
- )),
+ Err(e) => {
+ return ToolResult::error(format!(
+ "debug_image_out task failed: {e}. Not dispatching click."
+ ))
+ }
+ Ok(Err(e)) => {
+ return ToolResult::error(format!(
+ "debug_image_out write failed: {e}. Not dispatching click."
+ ))
+ }
Ok(Ok(())) => {}
}
}
@@ -548,9 +610,11 @@ impl Tool for ClickTool {
cx = wx;
cy = wy;
}
- None => return ToolResult::error(format!(
- "from_zoom=true but no zoom context for pid {pid}. Call zoom first."
- )),
+ None => {
+ return ToolResult::error(format!(
+ "from_zoom=true but no zoom context for pid {pid}. Call zoom first."
+ ))
+ }
}
} else if let Some(ratio) = self.state.resize_registry.ratio(pid) {
// Coordinates are in the downscaled image space; scale back to native pixels.
@@ -579,14 +643,27 @@ impl Tool for ClickTool {
// against the logical bounds.
if let Ok(png) = crate::capture::screenshot_window_bytes(wid) {
if png.len() >= 24 {
- let pw = u32::from_be_bytes([png[16], png[17], png[18], png[19]]) as f64;
+ let pw =
+ u32::from_be_bytes([png[16], png[17], png[18], png[19]]) as f64;
let lw = b.width;
- if lw > 0.0 && pw > lw { pw / lw } else { 1.0 }
- } else { 1.0 }
- } else { 1.0 }
- } else { 1.0 };
+ if lw > 0.0 && pw > lw {
+ pw / lw
+ } else {
+ 1.0
+ }
+ } else {
+ 1.0
+ }
+ } else {
+ 1.0
+ }
+ } else {
+ 1.0
+ };
(bounds, scale)
- }).await.unwrap_or((None, 1.0));
+ })
+ .await
+ .unwrap_or((None, 1.0));
if let (Some(b), scale) = result {
let wx = cx / scale;
let wy = cy / scale;
@@ -612,11 +689,16 @@ impl Tool for ClickTool {
// arrive — mirrors Swift's `AgentCursor.shared.animateAndWait(to:)`.
crate::cursor::overlay::animate_cursor_to(cursor_key.clone(), screen_x, screen_y).await;
// Keep the registry in sync with the overlay (see AX path above).
- self.state.cursor_registry.update_position(&cursor_key, screen_x, screen_y);
+ self.state
+ .cursor_registry
+ .update_position(&cursor_key, screen_x, screen_y);
// Show click-pulse on the agent cursor overlay.
crate::cursor::overlay::send_command(
cursor_key.clone(),
- cursor_overlay::OverlayCommand::ClickPulse { x: screen_x, y: screen_y },
+ cursor_overlay::OverlayCommand::ClickPulse {
+ x: screen_x,
+ y: screen_y,
+ },
);
// ── Focus-suppression wrap (Swift WindowChangeDetector + FocusGuard) ──
@@ -636,6 +718,7 @@ impl Tool for ClickTool {
// clicks. Needs window_id to front; without one it degrades to
// background (and is labelled as such).
let fg = delivery_mode.is_foreground() && window_id.is_some();
+
let result = focus_guard::with_focus_suppressed(
Some(pid),
prior_front,
@@ -668,7 +751,14 @@ impl Tool for ClickTool {
// and Chromium-specific fields (f40, f51, f58, f91, f92) onto events
// for better backgrounded-target delivery.
if let Some(wid) = window_id {
- return crate::input::mouse::click_at_xy_with_window_local(
+ if fg {
+ return crate::input::mouse::click_at_xy_with_window_local(
+ pid, screen_x, screen_y,
+ win_local_x, win_local_y,
+ wid, count, &m,
+ );
+ }
+ return crate::input::mouse::click_at_xy_chromium(
pid, screen_x, screen_y,
win_local_x, win_local_y,
wid, count, &m,
@@ -696,9 +786,9 @@ impl Tool for ClickTool {
let changes = snapshot.detect_async().await;
let button_label = match button_str.as_str() {
- "right" => "right-click",
+ "right" => "right-click",
"middle" => "middle-click",
- _ => "click",
+ _ => "click",
};
match result {
Ok(Ok(fronted)) => {
@@ -718,11 +808,11 @@ impl Tool for ClickTool {
.with_structured(serde_json::json!({ "path": path, "verified": false, "effect": "unverifiable" }))
}
Ok(Err(e)) => ToolResult::error(format!("{button_label} failed: {e}")),
- Err(e) => ToolResult::error(format!("Task error: {e}")),
+ Err(e) => ToolResult::error(format!("Task error: {e}")),
}
} else {
ToolResult::error(
- "Provide either (element_index + window_id) or (x + y). pid is always required."
+ "Provide either (element_index + window_id) or (x + y). pid is always required.",
)
}
}
@@ -757,7 +847,7 @@ fn perform_ax_click(
anyhow::bail!("AXUIElementPerformAction({ax_action}) returned {err}");
}
- let role = unsafe { copy_string_attr(element, "AXRole") }.unwrap_or_default();
+ let role = unsafe { copy_string_attr(element, "AXRole") }.unwrap_or_default();
let title = unsafe { copy_string_attr(element, "AXTitle") }.unwrap_or_default();
let mut summary = format!("✅ Performed {ax_action} on [{idx}] {role} \"{title}\".");
@@ -766,11 +856,14 @@ fn perform_ax_click(
if role == "AXPopUpButton" {
let children = unsafe { copy_children(element) };
if !children.is_empty() {
- let options: Vec<String> = children.iter()
+ let options: Vec<String> = children
+ .iter()
.filter_map(|&child| {
let t = unsafe { copy_string_attr(child, "AXTitle") }.unwrap_or_default();
let v = unsafe { copy_string_attr(child, "AXValue") }.unwrap_or_default();
- if t.is_empty() && v.is_empty() { return None; }
+ if t.is_empty() && v.is_empty() {
+ return None;
+ }
Some(if v.is_empty() || v == t {
format!("\"{t}\"")
} else {
@@ -778,7 +871,11 @@ fn perform_ax_click(
})
})
.collect();
- for &child in &children { unsafe { CFRelease(child as _); } }
+ for &child in &children {
+ unsafe {
+ CFRelease(child as _);
+ }
+ }
if !options.is_empty() {
let opt_list = options.join(", ");
@@ -786,7 +883,7 @@ fn perform_ax_click(
"\n\n⚠️ This is a popup/select button. The native macOS menu closes \
immediately when the window is in the background. Do NOT use click \
again — instead, use:\n set_value(pid, window_id, element_index, value)\n\
- Available options: ["
+ Available options: [",
);
summary.push_str(&opt_list);
summary.push(']');
@@ -798,7 +895,11 @@ fn perform_ax_click(
// machine-readable `suspected_noop` signal returned to the caller.
let suspected_noop = !advertised.contains(&ax_action.to_string());
if suspected_noop {
- let adv_list = if advertised.is_empty() { "none".into() } else { advertised.join(", ") };
+ let adv_list = if advertised.is_empty() {
+ "none".into()
+ } else {
+ advertised.join(", ")
+ };
summary.push_str(&format!(
"\n⚠️ Element does not advertise {ax_action} (actions: {adv_list}). \
Action may have been a no-op."
@@ -806,8 +907,8 @@ fn perform_ax_click(
}
// WebKit DOM focus settle: 800 ms for text inputs (returned to async caller).
- let needs_webkit_delay = ax_action == "AXPress"
- && (role == "AXTextField" || role == "AXTextArea");
+ let needs_webkit_delay =
+ ax_action == "AXPress" && (role == "AXTextField" || role == "AXTextArea");
// Show focus-rect highlight around the element (matches Swift showFocusRect).
// Also move the cursor to the element center so the glide animation plays.
@@ -827,20 +928,21 @@ fn perform_ax_click(
cursor_overlay::OverlayCommand::ClickPulse { x: cx, y: cy },
);
}
- let _ = pid; let _ = window_id; // used by caller context
+ let _ = pid;
+ let _ = window_id; // used by caller context
Ok((summary, needs_webkit_delay, suspected_noop))
}
fn map_action(action: &str) -> &'static str {
match action.to_lowercase().as_str() {
- "press" | "click" => "AXPress",
+ "press" | "click" => "AXPress",
"show_menu" | "right_click" => "AXShowMenu",
- "pick" => "AXPick",
- "confirm" => "AXConfirm",
- "cancel" => "AXCancel",
- "open" => "AXOpen",
- _ => "AXPress",
+ "pick" => "AXPick",
+ "confirm" => "AXConfirm",
+ "cancel" => "AXCancel",
+ "open" => "AXOpen",
+ _ => "AXPress",
}
}
@@ -879,9 +981,18 @@ mod tests {
fn description_mentions_button_default() {
let d = def();
let desc = d.description.to_ascii_lowercase();
- assert!(desc.contains("button"), "description should mention button arg");
- assert!(desc.contains("left"), "description should mention left default");
- assert!(desc.contains("middle"), "description should mention middle button");
+ assert!(
+ desc.contains("button"),
+ "description should mention button arg"
+ );
+ assert!(
+ desc.contains("left"),
+ "description should mention left default"
+ );
+ assert!(
+ desc.contains("middle"),
+ "description should mention middle button"
+ );
}
/// Existing default behaviour preserved: no `button` field on the call →
@@ -893,7 +1004,11 @@ mod tests {
use cua_driver_core::tool_args::ArgsExt;
let args = serde_json::json!({ "pid": 1234 });
let button_str_raw = args.str_or("button", "left").to_lowercase();
- let resolved = if button_str_raw.is_empty() { "left".to_string() } else { button_str_raw };
+ let resolved = if button_str_raw.is_empty() {
+ "left".to_string()
+ } else {
+ button_str_raw
+ };
assert_eq!(resolved, "left");
}
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/drag.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/drag.rs
index 09c157ff51..8deff2249d 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/drag.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/drag.rs
@@ -8,22 +8,27 @@
//! recent zoom crop context, same as click/double_click.
use async_trait::async_trait;
-use cua_driver_core::{protocol::ToolResult, tool::{Tool, ToolDef}};
+use cua_driver_core::{
+ protocol::ToolResult,
+ tool::{Tool, ToolDef},
+};
use serde_json::Value;
use std::sync::Arc;
+use super::ToolState;
use crate::apps;
use crate::focus_guard;
use crate::input::mouse::DragButton;
use crate::window_change_detector::WindowChangeDetector;
-use super::ToolState;
pub struct DragTool {
pub state: Arc<ToolState>,
}
impl DragTool {
- pub fn new(state: Arc<ToolState>) -> Self { Self { state } }
+ pub fn new(state: Arc<ToolState>) -> Self {
+ Self { state }
+ }
}
static DEF: std::sync::OnceLock<ToolDef> = std::sync::OnceLock::new();
@@ -98,11 +103,16 @@ fn def() -> &'static ToolDef {
#[async_trait]
impl Tool for DragTool {
- fn def(&self) -> &ToolDef { def() }
+ fn def(&self) -> &ToolDef {
+ def()
+ }
async fn invoke(&self, args: Value) -> ToolResult {
use cua_driver_core::tool_args::ArgsExt;
- let pid = match args.require_i32("pid") { Ok(v) => v, Err(e) => return e };
+ let pid = match args.require_i32("pid") {
+ Ok(v) => v,
+ Err(e) => return e,
+ };
// delivery_mode: foreground briefly fronts the window before the
// press-drag-release gesture (the explicit last resort for surfaces
// that drop background CGEvents), via the same skylight assist click
@@ -112,7 +122,8 @@ impl Tool for DragTool {
// Coerce integer or float from JSON for coordinate fields.
let coerce = |key: &str| -> Option<f64> {
- args.opt_f64(key).or_else(|| args.opt_i64(key).map(|i| i as f64))
+ args.opt_f64(key)
+ .or_else(|| args.opt_i64(key).map(|i| i as f64))
};
let mut from_x = match coerce("from_x") {
@@ -132,20 +143,22 @@ impl Tool for DragTool {
None => return ToolResult::error("Missing required parameter: to_y"),
};
- let window_id = args.opt_u64("window_id").map(|v| v as u32);
+ let window_id = args.opt_u64("window_id").map(|v| v as u32);
let duration_ms = args.u64_or("duration_ms", 500);
- let steps = args.u64_or("steps", 20) as usize;
- let from_zoom = args.bool_or("from_zoom", false);
- let button_str = args.str_or("button", "left");
+ let steps = args.u64_or("steps", 20) as usize;
+ let from_zoom = args.bool_or("from_zoom", false);
+ let button_str = args.str_or("button", "left");
let modifiers: Vec<String> = args.str_array("modifier");
let button = match button_str.to_lowercase().as_str() {
- "left" => DragButton::Left,
- "right" => DragButton::Right,
+ "left" => DragButton::Left,
+ "right" => DragButton::Right,
"middle" => DragButton::Middle,
- other => return ToolResult::error(format!(
- "Unknown button \"{other}\" — expected left, right, or middle."
- )),
+ other => {
+ return ToolResult::error(format!(
+ "Unknown button \"{other}\" — expected left, right, or middle."
+ ))
+ }
};
// from_zoom: translate from last zoom crop context.
@@ -154,47 +167,76 @@ impl Tool for DragTool {
Some(ctx) => {
let (wx, wy) = ctx.zoom_to_window(from_x, from_y);
let (wx2, wy2) = ctx.zoom_to_window(to_x, to_y);
- from_x = wx; from_y = wy;
- to_x = wx2; to_y = wy2;
+ from_x = wx;
+ from_y = wy;
+ to_x = wx2;
+ to_y = wy2;
+ }
+ None => {
+ return ToolResult::error(format!(
+ "from_zoom=true but no zoom context for pid {pid}. Call zoom first."
+ ))
}
- None => return ToolResult::error(format!(
- "from_zoom=true but no zoom context for pid {pid}. Call zoom first."
- )),
}
} else if let Some(ratio) = self.state.resize_registry.ratio(pid) {
- from_x *= ratio; from_y *= ratio;
- to_x *= ratio; to_y *= ratio;
+ from_x *= ratio;
+ from_y *= ratio;
+ to_x *= ratio;
+ to_y *= ratio;
}
// Translate window-local screenshot pixels → screen coordinates.
// Also compute window-local logical coords for CGEventSetWindowLocation.
- let (from_sx, from_sy, from_lx, from_ly,
- to_sx, to_sy, to_lx, to_ly) = if let Some(wid) = window_id {
- let result = tokio::task::spawn_blocking(move || {
- let bounds = crate::windows::window_bounds_by_id(wid);
- let scale: f64 = if let Some(ref b) = bounds {
- if let Ok(png) = crate::capture::screenshot_window_bytes(wid) {
- if png.len() >= 24 {
- let pw = u32::from_be_bytes([png[16], png[17], png[18], png[19]]) as f64;
- let lw = b.width;
- if lw > 0.0 && pw > lw { pw / lw } else { 1.0 }
- } else { 1.0 }
- } else { 1.0 }
- } else { 1.0 };
- (bounds, scale)
- }).await.unwrap_or((None, 1.0));
+ let (from_sx, from_sy, from_lx, from_ly, to_sx, to_sy, to_lx, to_ly) =
+ if let Some(wid) = window_id {
+ let result = tokio::task::spawn_blocking(move || {
+ let bounds = crate::windows::window_bounds_by_id(wid);
+ let scale: f64 = if let Some(ref b) = bounds {
+ if let Ok(png) = crate::capture::screenshot_window_bytes(wid) {
+ if png.len() >= 24 {
+ let pw =
+ u32::from_be_bytes([png[16], png[17], png[18], png[19]]) as f64;
+ let lw = b.width;
+ if lw > 0.0 && pw > lw {
+ pw / lw
+ } else {
+ 1.0
+ }
+ } else {
+ 1.0
+ }
+ } else {
+ 1.0
+ }
+ } else {
+ 1.0
+ };
+ (bounds, scale)
+ })
+ .await
+ .unwrap_or((None, 1.0));
- if let (Some(b), scale) = result {
- let flx = from_x / scale; let fly = from_y / scale;
- let tlx = to_x / scale; let tly = to_y / scale;
- (b.x + flx, b.y + fly, flx, fly,
- b.x + tlx, b.y + tly, tlx, tly)
+ if let (Some(b), scale) = result {
+ let flx = from_x / scale;
+ let fly = from_y / scale;
+ let tlx = to_x / scale;
+ let tly = to_y / scale;
+ (
+ b.x + flx,
+ b.y + fly,
+ flx,
+ fly,
+ b.x + tlx,
+ b.y + tly,
+ tlx,
+ tly,
+ )
+ } else {
+ (from_x, from_y, from_x, from_y, to_x, to_y, to_x, to_y)
+ }
} else {
(from_x, from_y, from_x, from_y, to_x, to_y, to_x, to_y)
- }
- } else {
- (from_x, from_y, from_x, from_y, to_x, to_y, to_x, to_y)
- };
+ };
// Animate agent cursor along drag path (start → end).
if let Some(wid) = window_id {
@@ -217,30 +259,64 @@ impl Tool for DragTool {
let mods_owned = modifiers.clone();
let fg = delivery_mode.is_foreground() && window_id.is_some();
let result = focus_guard::with_focus_suppressed(
- Some(pid),
+ // Foreground drag deliberately activates the target so the global
+ // HID stream carries the pressed-button state. A suppression lease
+ // here would race that activation and restore the prior app before
+ // Chromium receives the gesture.
+ if fg { None } else { Some(pid) },
prior_front,
"drag.CGEvent",
|| async move {
tokio::task::spawn_blocking(move || -> anyhow::Result<()> {
let do_it = move || -> anyhow::Result<()> {
let m: Vec<&str> = mods_owned.iter().map(String::as_str).collect();
+ if fg {
+ // HID delivery is global, so foreground mode must
+ // establish a real active application before the
+ // gesture begins. The SkyLight flash can be
+ // unavailable for Electron child windows; the
+ // documented Cocoa activation is the fallback.
+ apps::activate_pid(pid);
+ std::thread::sleep(std::time::Duration::from_millis(40));
+ return crate::input::mouse::drag_at_xy_foreground(
+ from_sx,
+ from_sy,
+ to_sx,
+ to_sy,
+ duration_ms,
+ steps,
+ &m,
+ button,
+ );
+ }
crate::input::mouse::drag_at_xy(
pid,
- from_sx, from_sy,
- to_sx, to_sy,
+ from_sx,
+ from_sy,
+ to_sx,
+ to_sy,
Some((from_lx, from_ly)),
- Some((to_lx, to_ly)),
+ Some((to_lx, to_ly)),
window_id,
duration_ms,
steps,
&m,
button,
+ fg,
)
};
- // Foreground rung: brief front → drag → restore prior frontmost.
+ // Foreground rung: activate for the complete HID gesture,
+ // then restore the prior app after pointer capture settles.
match (fg, window_id) {
- (true, Some(wid)) => {
- crate::input::skylight::with_foreground_assist(pid as libc::pid_t, wid, do_it)?;
+ (true, Some(_wid)) => {
+ let result = do_it();
+ std::thread::sleep(std::time::Duration::from_millis(100));
+ if let Some(previous_pid) = prior_front {
+ if previous_pid != pid {
+ apps::activate_pid(previous_pid);
+ }
+ }
+ result?;
Ok(())
}
_ => do_it(),
@@ -267,11 +343,17 @@ impl Tool for DragTool {
} else {
format!(" with {}", modifiers.join("+"))
};
- let btn_suffix = if button_str == "left" { String::new() } else {
+ let btn_suffix = if button_str == "left" {
+ String::new()
+ } else {
format!(" ({button_str} button)")
};
- let mode_label = if fg { " (delivery_mode:foreground)" } else { "" };
+ let mode_label = if fg {
+ " (delivery_mode:foreground)"
+ } else {
+ ""
+ };
match result {
Ok(Ok(())) => ToolResult::text(format!(
"✅ Posted drag{btn_suffix}{mod_suffix} to pid {pid} \
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/scroll.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/scroll.rs
index 070196f4ff..38d8ae59f7 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/scroll.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/scroll.rs
@@ -39,10 +39,10 @@ fn def() -> &'static ToolDef {
DEF.get_or_init(|| ToolDef {
name: "scroll".into(),
description: "Scroll the target pid. Two paths, picked by how you address the scroll:\n\n\
- • **Pixel-wheel path (targeted)** — when you pass a target, either \
+ • **Targeted wheel path** — when you pass a target, either \
`element_index`/`element_token` (preferred) or window-local `x, y` pixels: \
the driver synthesizes a real mouse-wheel event (CGEventCreateScrollWheelEvent, \
- pixel units) AT that screen point. The renderer hit-tests the wheel at the \
+ at that screen point. The renderer hit-tests the wheel at the \
cursor, so the scroll lands on whatever element is under the point — exactly \
like physically rolling the wheel over it. This is the ONLY way to scroll a \
nested `overflow:auto` region (e.g. a scrollable <div> with no tabindex): such \
@@ -53,7 +53,7 @@ fn def() -> &'static ToolDef {
(by='line'); horizontal uses Left/Right arrows. Drives the focused / page \
scroller only.\n\n\
Mapping: by='page' → larger step; by='line' → smaller step; amount = number of \
- wheel notches (pixel path) or keystroke repetitions (keystroke path).".into(),
+ wheel notches (targeted path) or keystroke repetitions (keystroke path).".into(),
input_schema: serde_json::json!({
"type": "object",
// `pid` conditionally required (validated in code), not pinned in the
@@ -155,7 +155,7 @@ impl Tool for ScrollTool {
}
}
- // ── Pixel-wheel path (targeted scroll) ───────────────────────────────
+ // ── Targeted wheel path ─────────────────────────────────────────────
// A target — element (preferred) OR window-local x,y — routes the scroll
// through a synthesized mouse-wheel event at that screen point, so the
// renderer's hit-test delivers it to whatever element is under the
@@ -165,7 +165,7 @@ impl Tool for ScrollTool {
let x_arg = args.opt_f64("x").or_else(|| args.opt_i64("x").map(|v| v as f64));
let y_arg = args.opt_f64("y").or_else(|| args.opt_i64("y").map(|v| v as f64));
- // Per-notch pixel step + direction→delta mapping (sign convention lives
+ // Per-notch step + direction→delta mapping (sign convention lives
// here; the mouse primitive stays sign-agnostic). macOS: +y reveals
// content ABOVE, -y reveals BELOW; +x reveals LEFT, -x reveals RIGHT.
let step = if by == "page" { WHEEL_STEP_PAGE_PX } else { WHEEL_STEP_LINE_PX };
@@ -184,6 +184,18 @@ impl Tool for ScrollTool {
// Retina scaling is needed here.
let wid = window_id;
tokio::task::spawn_blocking(move || {
+ // Web content can be present in AX while its frame is below
+ // the outer page viewport. Ask the accessibility hierarchy to
+ // reveal the target before taking the screen-space center;
+ // otherwise the wheel is posted outside the rendered window
+ // and nested overflow regions never receive it.
+ unsafe {
+ let _ = crate::ax::bindings::perform_action(
+ element_ptr as AXUIElementRef,
+ "AXScrollToVisible",
+ );
+ }
+ std::thread::sleep(std::time::Duration::from_millis(40));
let center = unsafe { element_screen_center(element_ptr as AXUIElementRef) };
center.map(|(cx, cy)| {
let win_local = wid