Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

feat(cua-driver/macos): implement page click_element + fix Chromium window targeting (#2082)

Gabriel Handford73fe8227982c
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/browser/browser_js.rs b/libs/cua-driver/rust/crates/platform-macos/src/browser/browser_js.rs
index 5dc67f1521..2ed7f6cf2a 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/browser/browser_js.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/browser/browser_js.rs
@@ -3,14 +3,26 @@
use std::time::Duration;
use anyhow::Context;
+use crate::windows::WindowBounds;
+
pub struct BrowserJs;
+const CHROME_APP_BUNDLE_PREFIX: &str = "com.google.Chrome.app.";
+
+#[derive(Clone, Debug)]
+struct NativeWindowTarget {
+ title: String,
+ bounds: WindowBounds,
+ same_bounds_ordinal: usize,
+}
+
fn app_name_for_bundle(bundle_id: &str) -> Option<&'static str> {
match bundle_id {
"com.google.Chrome" => Some("Google Chrome"),
"com.brave.Browser" => Some("Brave Browser"),
"com.microsoft.edgemac" => Some("Microsoft Edge"),
"com.apple.Safari" => Some("Safari"),
+ _ if bundle_id.starts_with(CHROME_APP_BUNDLE_PREFIX) => Some("Google Chrome"),
_ => None,
}
}
@@ -26,22 +38,17 @@ impl BrowserJs {
let app_name = app_name_for_bundle(bundle_id)
.ok_or_else(|| anyhow::anyhow!("Unsupported browser bundle: {bundle_id}"))?;
- // Get window title from CGWindowID.
- let win_title = {
+ // Resolve the WindowServer CGWindowID to stable properties that can be
+ // matched against the browser's AppleScript window model.
+ let target = {
let wid = window_id;
- tokio::task::spawn_blocking(move || {
- crate::windows::all_windows()
- .into_iter()
- .find(|w| w.window_id == wid)
- .map(|w| w.title)
- .unwrap_or_default()
- }).await?
- };
+ tokio::task::spawn_blocking(move || native_window_target(wid)).await?
+ }?;
- let escaped_title = escape_for_applescript_string(&win_title);
let escaped_js = escape_js_for_applescript(javascript);
let script = if bundle_id == "com.apple.Safari" {
+ let escaped_title = escape_for_applescript_string(&target.title);
format!(
r#"tell application "Safari"
set matchedDoc to missing value
@@ -58,23 +65,7 @@ impl BrowserJs {
end tell"#
)
} else {
- format!(
- r#"tell application "{app_name}"
- set matchedWindow to missing value
- repeat with w in windows
- if name of w contains "{escaped_title}" then
- set matchedWindow to w
- exit repeat
- end if
- end repeat
- if matchedWindow is missing value then
- set matchedWindow to front window
- end if
- tell active tab of matchedWindow
- execute javascript {escaped_js}
- end tell
-end tell"#
- )
+ chromium_window_script(app_name, &target, &escaped_js, window_id)
};
run_osascript(&script).await
@@ -100,6 +91,7 @@ end tell"#
let home = std::env::var("HOME").unwrap_or_default();
let profiles_dir = match bundle_id {
"com.google.Chrome" => format!("{home}/Library/Application Support/Google/Chrome"),
+ _ if bundle_id.starts_with(CHROME_APP_BUNDLE_PREFIX) => format!("{home}/Library/Application Support/Google/Chrome"),
"com.brave.Browser" => format!("{home}/Library/Application Support/BraveSoftware/Brave-Browser"),
"com.microsoft.edgemac" => format!("{home}/Library/Application Support/Microsoft Edge"),
_ => anyhow::bail!("No profiles directory for {bundle_id}"),
@@ -124,6 +116,92 @@ end tell"#
}
}
+fn native_window_target(window_id: u32) -> anyhow::Result<NativeWindowTarget> {
+ let windows = crate::windows::all_windows();
+ let target = windows
+ .iter()
+ .find(|w| w.window_id == window_id)
+ .ok_or_else(|| anyhow::anyhow!("No macOS window found for window_id {window_id}"))?;
+ let mut same_bounds = windows
+ .iter()
+ .filter(|window| window.pid == target.pid && same_bounds(&window.bounds, &target.bounds))
+ .collect::<Vec<_>>();
+ same_bounds.sort_by(|a, b| b.z_index.cmp(&a.z_index));
+ let same_bounds_ordinal = same_bounds
+ .iter()
+ .position(|window| window.window_id == target.window_id)
+ .unwrap_or(0);
+
+ Ok(NativeWindowTarget {
+ title: target.title.clone(),
+ bounds: target.bounds.clone(),
+ same_bounds_ordinal,
+ })
+}
+
+fn chromium_window_script(
+ app_name: &str,
+ target: &NativeWindowTarget,
+ escaped_js: &str,
+ window_id: u32,
+) -> String {
+ let left = rounded_i64(target.bounds.x);
+ let top = rounded_i64(target.bounds.y);
+ let right = rounded_i64(target.bounds.x + target.bounds.width);
+ let bottom = rounded_i64(target.bounds.y + target.bounds.height);
+ let ordinal = target.same_bounds_ordinal;
+ let title_fallback = if target.title.trim().is_empty() {
+ String::new()
+ } else {
+ let escaped_title = escape_for_applescript_string(&target.title);
+ format!(
+ r#" if matchedWindow is missing value then
+ repeat with w in windows
+ if name of w contains "{escaped_title}" then
+ set matchedWindow to w
+ exit repeat
+ end if
+ end repeat
+ end if
+"#
+ )
+ };
+
+ format!(
+ r#"tell application "{app_name}"
+ set matchedWindow to missing value
+ set matchingBoundsSeen to 0
+ repeat with w in windows
+ set candidateBounds to bounds of w
+ if (item 1 of candidateBounds is {left}) and (item 2 of candidateBounds is {top}) and (item 3 of candidateBounds is {right}) and (item 4 of candidateBounds is {bottom}) then
+ if matchingBoundsSeen is {ordinal} then
+ set matchedWindow to w
+ exit repeat
+ end if
+ set matchingBoundsSeen to matchingBoundsSeen + 1
+ end if
+ end repeat
+{title_fallback} if matchedWindow is missing value then
+ error "No {app_name} AppleScript window matched native window_id {window_id}"
+ end if
+ tell active tab of matchedWindow
+ execute javascript {escaped_js}
+ end tell
+end tell"#
+ )
+}
+
+fn same_bounds(left: &WindowBounds, right: &WindowBounds) -> bool {
+ rounded_i64(left.x) == rounded_i64(right.x)
+ && rounded_i64(left.y) == rounded_i64(right.y)
+ && rounded_i64(left.width) == rounded_i64(right.width)
+ && rounded_i64(left.height) == rounded_i64(right.height)
+}
+
+fn rounded_i64(value: f64) -> i64 {
+ value.round() as i64
+}
+
fn find_preferences_files(profiles_dir: &str) -> Vec<String> {
let Ok(entries) = std::fs::read_dir(profiles_dir) else { return vec![] };
let mut result = Vec::new();
@@ -236,3 +314,82 @@ fn rand_u64() -> u64 {
let tid = std::thread::current().id();
t.subsec_nanos() as u64 ^ t.as_secs().wrapping_mul(6364136223846793005) ^ format!("{tid:?}").len() as u64
}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ fn target(title: &str, same_bounds_ordinal: usize) -> NativeWindowTarget {
+ NativeWindowTarget {
+ title: title.to_owned(),
+ bounds: WindowBounds {
+ x: 22.0,
+ y: 55.0,
+ width: 1200.0,
+ height: 829.0,
+ },
+ same_bounds_ordinal,
+ }
+ }
+
+ #[test]
+ fn chromium_script_matches_window_bounds_and_same_bounds_ordinal() {
+ let script = chromium_window_script(
+ "Google Chrome",
+ &target("", 2),
+ "\"window.location.href\"",
+ 27655,
+ );
+
+ assert!(script.contains("item 1 of candidateBounds is 22"));
+ assert!(script.contains("item 2 of candidateBounds is 55"));
+ assert!(script.contains("item 3 of candidateBounds is 1222"));
+ assert!(script.contains("item 4 of candidateBounds is 884"));
+ assert!(script.contains("if matchingBoundsSeen is 2"));
+ assert!(
+ script.contains("No Google Chrome AppleScript window matched native window_id 27655")
+ );
+ }
+
+ #[test]
+ fn chromium_script_does_not_fall_back_to_front_window_or_empty_title_match() {
+ let script =
+ chromium_window_script("Google Chrome", &target("", 0), "\"document.title\"", 27655);
+
+ assert!(!script.contains("front window"));
+ assert!(!script.contains("contains \"\""));
+ assert!(script.contains(
+ "error \"No Google Chrome AppleScript window matched native window_id 27655\""
+ ));
+ }
+
+ #[test]
+ fn chromium_script_uses_non_empty_title_as_secondary_fallback() {
+ let script = chromium_window_script(
+ "Google Chrome",
+ &target("Example \"quoted\" Domain", 0),
+ "\"document.title\"",
+ 27655,
+ );
+
+ assert!(script.contains("if name of w contains \"Example \\\"quoted\\\" Domain\""));
+ }
+
+ #[test]
+ fn same_bounds_compares_rounded_window_geometry() {
+ assert!(same_bounds(
+ &WindowBounds {
+ x: 22.2,
+ y: 55.4,
+ width: 1199.8,
+ height: 829.1,
+ },
+ &WindowBounds {
+ x: 22.0,
+ y: 55.0,
+ width: 1200.0,
+ height: 829.0,
+ }
+ ));
+ }
+}
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/page.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/page.rs
index 99e6b312f5..e2d7703abe 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/page.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/page.rs
@@ -10,7 +10,7 @@
//! `cua_driver_core::page` — this file only implements the backend.
use async_trait::async_trait;
-use cua_driver_core::page::PageBackend;
+use cua_driver_core::page::{ClickElementResult, PageBackend};
use std::sync::Arc;
use super::ToolState;
@@ -99,12 +99,85 @@ impl PageBackend for MacOsPageBackend {
execute_js(javascript, &bundle_id, pid, window_id).await
}
+ async fn click_element(
+ &self,
+ pid: i32,
+ window_id: u32,
+ selector: &str,
+ ) -> anyhow::Result<ClickElementResult> {
+ let selector_js = json_string(selector);
+ let probe_js = format!(
+ r#"(function() {{
+ var selector = {selector_js};
+ var el = document.querySelector(selector);
+ if (!el) throw new Error("element_not_found:" + selector);
+ var r = el.getBoundingClientRect();
+ return JSON.stringify({{
+ vx: r.left + r.width / 2,
+ vy: r.top + r.height / 2,
+ sx: window.screenX + (window.outerWidth - window.innerWidth) / 2,
+ sy: window.screenY + (window.outerHeight - window.innerHeight),
+ dpr: window.devicePixelRatio || 1
+ }});
+}})();"#
+ );
+ let probe_raw = self.execute_javascript(pid, window_id, &probe_js).await?;
+ let parsed = parse_click_probe(&probe_raw)?;
+
+ let vx = required_finite(&parsed, "vx", &probe_raw)?;
+ let vy = required_finite(&parsed, "vy", &probe_raw)?;
+ let sx = required_finite(&parsed, "sx", &probe_raw)?;
+ let sy = required_finite(&parsed, "sy", &probe_raw)?;
+ let dpr = parsed
+ .get("dpr")
+ .and_then(serde_json::Value::as_f64)
+ .filter(|value| value.is_finite() && *value > 0.0)
+ .unwrap_or(1.0);
+
+ let screen_x = sx + vx * dpr;
+ let screen_y = sy + vy * dpr;
+ let cursor_key = "default".to_owned();
+ crate::cursor::overlay::send_command(
+ cursor_key.clone(),
+ cursor_overlay::OverlayCommand::PinAbove(window_id as u64),
+ );
+ crate::cursor::overlay::animate_cursor_to(cursor_key.clone(), screen_x, screen_y).await;
+ self.state
+ .cursor_registry
+ .update_position(&cursor_key, screen_x, screen_y);
+ crate::cursor::overlay::send_command(
+ cursor_key,
+ cursor_overlay::OverlayCommand::ClickPulse {
+ x: screen_x,
+ y: screen_y,
+ },
+ );
+
+ let click_js = format!(
+ r#"(function() {{
+ var selector = {selector_js};
+ var el = document.querySelector(selector);
+ if (!el) throw new Error("element_not_found_on_click:" + selector);
+ el.click();
+ return "clicked:" + selector;
+}})();"#
+ );
+ let _ = self.execute_javascript(pid, window_id, &click_js).await?;
+
+ Ok(ClickElementResult {
+ screen_x,
+ screen_y,
+ viewport_x: vx,
+ viewport_y: vy,
+ message: format!(
+ "Clicked {selector} at screen ({screen_x:.0},{screen_y:.0}) on pid {pid}."
+ ),
+ })
+ }
+
async fn enable_javascript_apple_events(&self, bundle_id: &str) -> anyhow::Result<String> {
BrowserJs::enable_javascript_apple_events(bundle_id).await?;
- Ok(
- "JavaScript from Apple Events has been enabled. The browser is restarting."
- .to_owned(),
- )
+ Ok("JavaScript from Apple Events has been enabled. The browser is restarting.".to_owned())
}
async fn insert_text(
@@ -160,12 +233,7 @@ async fn resolve_cdp_port(pid: i32, cdp_port: Option<u16>, action: &str) -> anyh
}
/// Route JavaScript execution to the appropriate backend.
-async fn execute_js(
- js: &str,
- bundle_id: &str,
- pid: i32,
- window_id: u32,
-) -> anyhow::Result<String> {
+async fn execute_js(js: &str, bundle_id: &str, pid: i32, window_id: u32) -> anyhow::Result<String> {
if BrowserJs::supports(bundle_id) {
return BrowserJs::execute(js, bundle_id, window_id).await;
}
@@ -185,11 +253,10 @@ async fn execute_js(
/// Extract page text via the AX tree.
async fn ax_text_fallback(pid: i32, window_id: u32) -> anyhow::Result<String> {
- let result = tokio::task::spawn_blocking(move || {
- crate::ax::tree::walk_tree(pid, Some(window_id), None)
- })
- .await
- .map_err(|e| anyhow::anyhow!("AX walk task failed: {e}"))?;
+ let result =
+ tokio::task::spawn_blocking(move || crate::ax::tree::walk_tree(pid, Some(window_id), None))
+ .await
+ .map_err(|e| anyhow::anyhow!("AX walk task failed: {e}"))?;
Ok(AXPageReader::extract_text(&result.tree_markdown))
}
@@ -200,11 +267,10 @@ async fn ax_query_fallback(
selector: &str,
) -> anyhow::Result<Vec<crate::browser::ax_page_reader::AXElement>> {
let sel = selector.to_owned();
- let result = tokio::task::spawn_blocking(move || {
- crate::ax::tree::walk_tree(pid, Some(window_id), None)
- })
- .await
- .map_err(|e| anyhow::anyhow!("AX walk task failed: {e}"))?;
+ let result =
+ tokio::task::spawn_blocking(move || crate::ax::tree::walk_tree(pid, Some(window_id), None))
+ .await
+ .map_err(|e| anyhow::anyhow!("AX walk task failed: {e}"))?;
Ok(AXPageReader::query(&sel, &result.tree_markdown))
}
@@ -269,3 +335,29 @@ fn json_string(value: &str) -> String {
.replace('\t', "\\t");
format!("\"{escaped}\"")
}
+
+fn parse_click_probe(raw: &str) -> anyhow::Result<serde_json::Value> {
+ let first = serde_json::from_str::<serde_json::Value>(raw.trim()).map_err(|error| {
+ anyhow::anyhow!("click_element: could not parse coord JSON from probe {raw:?}: {error}")
+ })?;
+ match first {
+ serde_json::Value::String(inner) => serde_json::from_str(&inner).map_err(|error| {
+ anyhow::anyhow!(
+ "click_element: could not parse inner coord JSON from probe {raw:?}: {error}"
+ )
+ }),
+ other => Ok(other),
+ }
+}
+
+fn required_finite(value: &serde_json::Value, key: &str, raw: &str) -> anyhow::Result<f64> {
+ value
+ .get(key)
+ .and_then(serde_json::Value::as_f64)
+ .filter(|number| number.is_finite())
+ .ok_or_else(|| {
+ anyhow::anyhow!(
+ "click_element: probe JSON missing/invalid required field '{key}' (raw: {raw:?})"
+ )
+ })
+}