Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
feat(cua-driver/macos): implement page click_element + fix Chromium window targeting (#2082)
Gabriel Handford73fe8227982c
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/browser/browser_js.rs b/libs/cua-driver/rust/crates/platform-macos/src/browser/browser_js.rsindex 5dc67f1521..2ed7f6cf2a 100644--- a/libs/cua-driver/rust/crates/platform-macos/src/browser/browser_js.rs+++ b/libs/cua-driver/rust/crates/platform-macos/src/browser/browser_js.rs@@ -3,14 +3,26 @@use std::time::Duration;use anyhow::Context;+use crate::windows::WindowBounds;+pub struct BrowserJs;+const CHROME_APP_BUNDLE_PREFIX: &str = "com.google.Chrome.app.";++#[derive(Clone, Debug)]+struct NativeWindowTarget {+ title: String,+ bounds: WindowBounds,+ same_bounds_ordinal: usize,+}+fn app_name_for_bundle(bundle_id: &str) -> Option<&'static str> {match bundle_id {"com.google.Chrome" => Some("Google Chrome"),"com.brave.Browser" => Some("Brave Browser"),"com.microsoft.edgemac" => Some("Microsoft Edge"),"com.apple.Safari" => Some("Safari"),+ _ if bundle_id.starts_with(CHROME_APP_BUNDLE_PREFIX) => Some("Google Chrome"),_ => None,}}@@ -26,22 +38,17 @@ impl BrowserJs {let app_name = app_name_for_bundle(bundle_id).ok_or_else(|| anyhow::anyhow!("Unsupported browser bundle: {bundle_id}"))?;- // Get window title from CGWindowID.- let win_title = {+ // Resolve the WindowServer CGWindowID to stable properties that can be+ // matched against the browser's AppleScript window model.+ let target = {let wid = window_id;- tokio::task::spawn_blocking(move || {- crate::windows::all_windows()- .into_iter()- .find(|w| w.window_id == wid)- .map(|w| w.title)- .unwrap_or_default()- }).await?- };+ tokio::task::spawn_blocking(move || native_window_target(wid)).await?+ }?;- let escaped_title = escape_for_applescript_string(&win_title);let escaped_js = escape_js_for_applescript(javascript);let script = if bundle_id == "com.apple.Safari" {+ let escaped_title = escape_for_applescript_string(&target.title);format!(r#"tell application "Safari"set matchedDoc to missing value@@ -58,23 +65,7 @@ impl BrowserJs {end tell"#)} else {- format!(- r#"tell application "{app_name}"- set matchedWindow to missing value- repeat with w in windows- if name of w contains "{escaped_title}" then- set matchedWindow to w- exit repeat- end if- end repeat- if matchedWindow is missing value then- set matchedWindow to front window- end if- tell active tab of matchedWindow- execute javascript {escaped_js}- end tell-end tell"#- )+ chromium_window_script(app_name, &target, &escaped_js, window_id)};run_osascript(&script).await@@ -100,6 +91,7 @@ end tell"#let home = std::env::var("HOME").unwrap_or_default();let profiles_dir = match bundle_id {"com.google.Chrome" => format!("{home}/Library/Application Support/Google/Chrome"),+ _ if bundle_id.starts_with(CHROME_APP_BUNDLE_PREFIX) => format!("{home}/Library/Application Support/Google/Chrome"),"com.brave.Browser" => format!("{home}/Library/Application Support/BraveSoftware/Brave-Browser"),"com.microsoft.edgemac" => format!("{home}/Library/Application Support/Microsoft Edge"),_ => anyhow::bail!("No profiles directory for {bundle_id}"),@@ -124,6 +116,92 @@ end tell"#}}+fn native_window_target(window_id: u32) -> anyhow::Result<NativeWindowTarget> {+ let windows = crate::windows::all_windows();+ let target = windows+ .iter()+ .find(|w| w.window_id == window_id)+ .ok_or_else(|| anyhow::anyhow!("No macOS window found for window_id {window_id}"))?;+ let mut same_bounds = windows+ .iter()+ .filter(|window| window.pid == target.pid && same_bounds(&window.bounds, &target.bounds))+ .collect::<Vec<_>>();+ same_bounds.sort_by(|a, b| b.z_index.cmp(&a.z_index));+ let same_bounds_ordinal = same_bounds+ .iter()+ .position(|window| window.window_id == target.window_id)+ .unwrap_or(0);++ Ok(NativeWindowTarget {+ title: target.title.clone(),+ bounds: target.bounds.clone(),+ same_bounds_ordinal,+ })+}++fn chromium_window_script(+ app_name: &str,+ target: &NativeWindowTarget,+ escaped_js: &str,+ window_id: u32,+) -> String {+ let left = rounded_i64(target.bounds.x);+ let top = rounded_i64(target.bounds.y);+ let right = rounded_i64(target.bounds.x + target.bounds.width);+ let bottom = rounded_i64(target.bounds.y + target.bounds.height);+ let ordinal = target.same_bounds_ordinal;+ let title_fallback = if target.title.trim().is_empty() {+ String::new()+ } else {+ let escaped_title = escape_for_applescript_string(&target.title);+ format!(+ r#" if matchedWindow is missing value then+ repeat with w in windows+ if name of w contains "{escaped_title}" then+ set matchedWindow to w+ exit repeat+ end if+ end repeat+ end if+"#+ )+ };++ format!(+ r#"tell application "{app_name}"+ set matchedWindow to missing value+ set matchingBoundsSeen to 0+ repeat with w in windows+ set candidateBounds to bounds of w+ if (item 1 of candidateBounds is {left}) and (item 2 of candidateBounds is {top}) and (item 3 of candidateBounds is {right}) and (item 4 of candidateBounds is {bottom}) then+ if matchingBoundsSeen is {ordinal} then+ set matchedWindow to w+ exit repeat+ end if+ set matchingBoundsSeen to matchingBoundsSeen + 1+ end if+ end repeat+{title_fallback} if matchedWindow is missing value then+ error "No {app_name} AppleScript window matched native window_id {window_id}"+ end if+ tell active tab of matchedWindow+ execute javascript {escaped_js}+ end tell+end tell"#+ )+}++fn same_bounds(left: &WindowBounds, right: &WindowBounds) -> bool {+ rounded_i64(left.x) == rounded_i64(right.x)+ && rounded_i64(left.y) == rounded_i64(right.y)+ && rounded_i64(left.width) == rounded_i64(right.width)+ && rounded_i64(left.height) == rounded_i64(right.height)+}++fn rounded_i64(value: f64) -> i64 {+ value.round() as i64+}+fn find_preferences_files(profiles_dir: &str) -> Vec<String> {let Ok(entries) = std::fs::read_dir(profiles_dir) else { return vec![] };let mut result = Vec::new();@@ -236,3 +314,82 @@ fn rand_u64() -> u64 {let tid = std::thread::current().id();t.subsec_nanos() as u64 ^ t.as_secs().wrapping_mul(6364136223846793005) ^ format!("{tid:?}").len() as u64}++#[cfg(test)]+mod tests {+ use super::*;++ fn target(title: &str, same_bounds_ordinal: usize) -> NativeWindowTarget {+ NativeWindowTarget {+ title: title.to_owned(),+ bounds: WindowBounds {+ x: 22.0,+ y: 55.0,+ width: 1200.0,+ height: 829.0,+ },+ same_bounds_ordinal,+ }+ }++ #[test]+ fn chromium_script_matches_window_bounds_and_same_bounds_ordinal() {+ let script = chromium_window_script(+ "Google Chrome",+ &target("", 2),+ "\"window.location.href\"",+ 27655,+ );++ assert!(script.contains("item 1 of candidateBounds is 22"));+ assert!(script.contains("item 2 of candidateBounds is 55"));+ assert!(script.contains("item 3 of candidateBounds is 1222"));+ assert!(script.contains("item 4 of candidateBounds is 884"));+ assert!(script.contains("if matchingBoundsSeen is 2"));+ assert!(+ script.contains("No Google Chrome AppleScript window matched native window_id 27655")+ );+ }++ #[test]+ fn chromium_script_does_not_fall_back_to_front_window_or_empty_title_match() {+ let script =+ chromium_window_script("Google Chrome", &target("", 0), "\"document.title\"", 27655);++ assert!(!script.contains("front window"));+ assert!(!script.contains("contains \"\""));+ assert!(script.contains(+ "error \"No Google Chrome AppleScript window matched native window_id 27655\""+ ));+ }++ #[test]+ fn chromium_script_uses_non_empty_title_as_secondary_fallback() {+ let script = chromium_window_script(+ "Google Chrome",+ &target("Example \"quoted\" Domain", 0),+ "\"document.title\"",+ 27655,+ );++ assert!(script.contains("if name of w contains \"Example \\\"quoted\\\" Domain\""));+ }++ #[test]+ fn same_bounds_compares_rounded_window_geometry() {+ assert!(same_bounds(+ &WindowBounds {+ x: 22.2,+ y: 55.4,+ width: 1199.8,+ height: 829.1,+ },+ &WindowBounds {+ x: 22.0,+ y: 55.0,+ width: 1200.0,+ height: 829.0,+ }+ ));+ }+}diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/page.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/page.rsindex 99e6b312f5..e2d7703abe 100644--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/page.rs+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/page.rs@@ -10,7 +10,7 @@//! `cua_driver_core::page` — this file only implements the backend.use async_trait::async_trait;-use cua_driver_core::page::PageBackend;+use cua_driver_core::page::{ClickElementResult, PageBackend};use std::sync::Arc;use super::ToolState;@@ -99,12 +99,85 @@ impl PageBackend for MacOsPageBackend {execute_js(javascript, &bundle_id, pid, window_id).await}+ async fn click_element(+ &self,+ pid: i32,+ window_id: u32,+ selector: &str,+ ) -> anyhow::Result<ClickElementResult> {+ let selector_js = json_string(selector);+ let probe_js = format!(+ r#"(function() {{+ var selector = {selector_js};+ var el = document.querySelector(selector);+ if (!el) throw new Error("element_not_found:" + selector);+ var r = el.getBoundingClientRect();+ return JSON.stringify({{+ vx: r.left + r.width / 2,+ vy: r.top + r.height / 2,+ sx: window.screenX + (window.outerWidth - window.innerWidth) / 2,+ sy: window.screenY + (window.outerHeight - window.innerHeight),+ dpr: window.devicePixelRatio || 1+ }});+}})();"#+ );+ let probe_raw = self.execute_javascript(pid, window_id, &probe_js).await?;+ let parsed = parse_click_probe(&probe_raw)?;++ let vx = required_finite(&parsed, "vx", &probe_raw)?;+ let vy = required_finite(&parsed, "vy", &probe_raw)?;+ let sx = required_finite(&parsed, "sx", &probe_raw)?;+ let sy = required_finite(&parsed, "sy", &probe_raw)?;+ let dpr = parsed+ .get("dpr")+ .and_then(serde_json::Value::as_f64)+ .filter(|value| value.is_finite() && *value > 0.0)+ .unwrap_or(1.0);++ let screen_x = sx + vx * dpr;+ let screen_y = sy + vy * dpr;+ let cursor_key = "default".to_owned();+ crate::cursor::overlay::send_command(+ cursor_key.clone(),+ cursor_overlay::OverlayCommand::PinAbove(window_id as u64),+ );+ crate::cursor::overlay::animate_cursor_to(cursor_key.clone(), screen_x, screen_y).await;+ self.state+ .cursor_registry+ .update_position(&cursor_key, screen_x, screen_y);+ crate::cursor::overlay::send_command(+ cursor_key,+ cursor_overlay::OverlayCommand::ClickPulse {+ x: screen_x,+ y: screen_y,+ },+ );++ let click_js = format!(+ r#"(function() {{+ var selector = {selector_js};+ var el = document.querySelector(selector);+ if (!el) throw new Error("element_not_found_on_click:" + selector);+ el.click();+ return "clicked:" + selector;+}})();"#+ );+ let _ = self.execute_javascript(pid, window_id, &click_js).await?;++ Ok(ClickElementResult {+ screen_x,+ screen_y,+ viewport_x: vx,+ viewport_y: vy,+ message: format!(+ "Clicked {selector} at screen ({screen_x:.0},{screen_y:.0}) on pid {pid}."+ ),+ })+ }+async fn enable_javascript_apple_events(&self, bundle_id: &str) -> anyhow::Result<String> {BrowserJs::enable_javascript_apple_events(bundle_id).await?;- Ok(- "JavaScript from Apple Events has been enabled. The browser is restarting."- .to_owned(),- )+ Ok("JavaScript from Apple Events has been enabled. The browser is restarting.".to_owned())}async fn insert_text(@@ -160,12 +233,7 @@ async fn resolve_cdp_port(pid: i32, cdp_port: Option<u16>, action: &str) -> anyh}/// Route JavaScript execution to the appropriate backend.-async fn execute_js(- js: &str,- bundle_id: &str,- pid: i32,- window_id: u32,-) -> anyhow::Result<String> {+async fn execute_js(js: &str, bundle_id: &str, pid: i32, window_id: u32) -> anyhow::Result<String> {if BrowserJs::supports(bundle_id) {return BrowserJs::execute(js, bundle_id, window_id).await;}@@ -185,11 +253,10 @@ async fn execute_js(/// Extract page text via the AX tree.async fn ax_text_fallback(pid: i32, window_id: u32) -> anyhow::Result<String> {- let result = tokio::task::spawn_blocking(move || {- crate::ax::tree::walk_tree(pid, Some(window_id), None)- })- .await- .map_err(|e| anyhow::anyhow!("AX walk task failed: {e}"))?;+ let result =+ tokio::task::spawn_blocking(move || crate::ax::tree::walk_tree(pid, Some(window_id), None))+ .await+ .map_err(|e| anyhow::anyhow!("AX walk task failed: {e}"))?;Ok(AXPageReader::extract_text(&result.tree_markdown))}@@ -200,11 +267,10 @@ async fn ax_query_fallback(selector: &str,) -> anyhow::Result<Vec<crate::browser::ax_page_reader::AXElement>> {let sel = selector.to_owned();- let result = tokio::task::spawn_blocking(move || {- crate::ax::tree::walk_tree(pid, Some(window_id), None)- })- .await- .map_err(|e| anyhow::anyhow!("AX walk task failed: {e}"))?;+ let result =+ tokio::task::spawn_blocking(move || crate::ax::tree::walk_tree(pid, Some(window_id), None))+ .await+ .map_err(|e| anyhow::anyhow!("AX walk task failed: {e}"))?;Ok(AXPageReader::query(&sel, &result.tree_markdown))}@@ -269,3 +335,29 @@ fn json_string(value: &str) -> String {.replace('\t', "\\t");format!("\"{escaped}\"")}++fn parse_click_probe(raw: &str) -> anyhow::Result<serde_json::Value> {+ let first = serde_json::from_str::<serde_json::Value>(raw.trim()).map_err(|error| {+ anyhow::anyhow!("click_element: could not parse coord JSON from probe {raw:?}: {error}")+ })?;+ match first {+ serde_json::Value::String(inner) => serde_json::from_str(&inner).map_err(|error| {+ anyhow::anyhow!(+ "click_element: could not parse inner coord JSON from probe {raw:?}: {error}"+ )+ }),+ other => Ok(other),+ }+}++fn required_finite(value: &serde_json::Value, key: &str, raw: &str) -> anyhow::Result<f64> {+ value+ .get(key)+ .and_then(serde_json::Value::as_f64)+ .filter(|number| number.is_finite())+ .ok_or_else(|| {+ anyhow::anyhow!(+ "click_element: probe JSON missing/invalid required field '{key}' (raw: {raw:?})"+ )+ })+}