Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
fix(cua-driver): count a bundle-identifier capture approval as consent (fixes hosted macOS preflight) (#5402)
Francesco Bonacci8347d6cd849b
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/capture_consent.rs b/libs/cua-driver/rust/crates/platform-macos/src/capture_consent.rsindex 8d00831949..62752a2f05 100644--- a/libs/cua-driver/rust/crates/platform-macos/src/capture_consent.rs+++ b/libs/cua-driver/rust/crates/platform-macos/src/capture_consent.rs@@ -15,7 +15,9 @@//! `~/Library/Group Containers/group.com.apple.replayd/ScreenCaptureApprovals.plist`,//! keyed by the app bundle's file URL, as soon as the alert is shown. With//! an entry present, later captures (also after the entry's hint date)-//! raise nothing.+//! raise nothing. An entry keyed by the bundle identifier also counts:+//! that is how a managed or pre-provisioned machine (and the hosted CI+//! runner) records the approval ahead of time.//!//! So the driver surfaces the consent in `cua-driver permissions grant`,//! where the user expects system prompts (see `check_permissions`), and@@ -67,7 +69,12 @@ pub fn capture_blocked_reason() -> Option<String> {}}let home = std::env::var_os("HOME")?;- match consent_state(&Path::new(&home).join(APPROVALS_RELATIVE_PATH), &bundle) {+ let bundle_id = crate::tools::main_bundle_identifier();+ match consent_state(+ &Path::new(&home).join(APPROVALS_RELATIVE_PATH),+ &bundle,+ bundle_id.as_deref(),+ ) {ConsentState::Recorded | ConsentState::Unknown => {RECORDED.store(true, Ordering::Relaxed);None@@ -105,14 +112,14 @@ enum ConsentState {Unknown,}-fn consent_state(approvals: &Path, bundle: &Path) -> ConsentState {+fn consent_state(approvals: &Path, bundle: &Path, bundle_id: Option<&str>) -> ConsentState {let bytes = match std::fs::read(approvals) {Ok(bytes) => bytes,Err(error) if error.kind() == std::io::ErrorKind::NotFound => return ConsentState::Missing,Err(_) => return ConsentState::Unknown,};match approval_keys(&bytes) {- Some(keys) if keys.iter().any(|key| key_names_bundle(key, bundle)) => {+ Some(keys) if keys.iter().any(|key| key_names_app(key, bundle, bundle_id)) => {ConsentState::Recorded}Some(_) => ConsentState::Missing,@@ -147,6 +154,12 @@ fn approval_keys(bytes: &[u8]) -> Option<Vec<String>> {)}+/// Whether an approvals key names this app: its bundle file URL, as replayd+/// writes it, or its bundle identifier, as a pre-provisioned approval does.+fn key_names_app(key: &str, bundle: &Path, bundle_id: Option<&str>) -> bool {+ bundle_id.is_some_and(|id| !id.is_empty() && key == id) || key_names_bundle(key, bundle)+}+/// Whether an approvals key (`file:///…/CuaDriver.app/`) names `bundle`.fn key_names_bundle(key: &str, bundle: &Path) -> bool {let Some(path) = key.strip_prefix("file://") else {@@ -265,12 +278,21 @@ mod tests {std::fs::create_dir_all(&dir).unwrap();let file = dir.join("ScreenCaptureApprovals.plist");let bundle = Path::new("/Applications/CuaDriver.app");+ let id = Some("com.trycua.driver");let _ = std::fs::remove_file(&file);- assert_eq!(consent_state(&file, bundle), ConsentState::Missing);+ assert_eq!(consent_state(&file, bundle, id), ConsentState::Missing);write_plist(&file, &["file:///Applications/Other.app/"]);- assert_eq!(consent_state(&file, bundle), ConsentState::Missing);+ assert_eq!(consent_state(&file, bundle, id), ConsentState::Missing);++ write_plist(&file, &["com.trycua.driver"]);+ assert_eq!(consent_state(&file, bundle, id), ConsentState::Recorded);+ assert_eq!(consent_state(&file, bundle, None), ConsentState::Missing);+ assert_eq!(+ consent_state(&file, bundle, Some("com.trycua.driver.local")),+ ConsentState::Missing+ );write_plist(&file,@@ -279,10 +301,10 @@ mod tests {"file:///Applications/CuaDriver.app/",],);- assert_eq!(consent_state(&file, bundle), ConsentState::Recorded);+ assert_eq!(consent_state(&file, bundle, id), ConsentState::Recorded);std::fs::write(&file, b"not a plist").unwrap();- assert_eq!(consent_state(&file, bundle), ConsentState::Unknown);+ assert_eq!(consent_state(&file, bundle, id), ConsentState::Unknown);let _ = std::fs::remove_dir_all(&dir);}}diff --git a/libs/cua-driver/rust/crates/platform-macos/src/input/ax_actions.rs b/libs/cua-driver/rust/crates/platform-macos/src/input/ax_actions.rsindex 15eb1d2d13..0adce0e332 100644--- a/libs/cua-driver/rust/crates/platform-macos/src/input/ax_actions.rs+++ b/libs/cua-driver/rust/crates/platform-macos/src/input/ax_actions.rs@@ -283,6 +283,19 @@ pub fn is_element_focused(pid: i32, element_ptr: usize) -> bool {}}+/// Report whether `element_ptr` holds keyboard focus, by either readback.+///+/// The application's `AXFocusedUIElement` is the stronger signal, but for+/// web content (the Tauri fixture) in a background window it does not name+/// the DOM control even after the control took focus, so a targeted+/// background `press_key` was refused although the key would have landed.+/// Accept the control's own `AXFocused = true` too. A write that AppKit+/// clobbered reads back `false` either way.+pub fn element_has_keyboard_focus(pid: i32, element_ptr: usize) -> bool {+ is_element_focused(pid, element_ptr)+ || unsafe { copy_bool_attr(element_ptr as AXUIElementRef, "AXFocused") } == Some(true)+}+/// Set the AXValue of an element (for dropdowns, text fields, etc.).pub fn set_ax_value(element_ptr: usize, value: &str) -> anyhow::Result<()> {let err = unsafe { set_string_attr(element_ptr as AXUIElementRef, "AXValue", value) };diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/health_report.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/health_report.rsindex 69bf12ab02..fbca96e439 100644--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/health_report.rs+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/health_report.rs@@ -348,7 +348,7 @@ fn bundle_identifier(bundle: &core_foundation::bundle::CFBundle) -> Option<Strin/// Read the running process's `CFBundleIdentifier` via CoreFoundation./// Returns `None` when the process has no associated bundle (e.g./// running the bare binary outside `CuaDriver.app`).-fn current_bundle_identifier() -> Option<String> {+pub(crate) fn current_bundle_identifier() -> Option<String> {use core_foundation::base::TCFType;use core_foundation::bundle::{CFBundle, CFBundleGetMainBundle};diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/mod.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/mod.rsindex 4cde8b43bb..771e6717e3 100644--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/mod.rs+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/mod.rs@@ -264,6 +264,7 @@ pub use check_permissions::{};pub use cua_driver_core::snapshot_store::ZoomContext;+pub(crate) use health_report::current_bundle_identifier as main_bundle_identifier;pub(crate) use health_report::sw_vers_product_version as macos_product_version;/// The shared per-call delivery mode; see [`cua_driver_core::delivery`].diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/press_key.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/press_key.rsindex aec7eb7262..e4d007f0bc 100644--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/press_key.rs+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/press_key.rs@@ -467,7 +467,7 @@ impl Tool for PressKeyTool {pre_focus_ptr,crate::input::ax_actions::focus_element,|target| {- crate::input::ax_actions::is_element_focused(+ crate::input::ax_actions::element_has_keyboard_focus(pid, target,)},@@ -482,7 +482,9 @@ impl Tool for PressKeyTool {with_confirmed_key_focus(pre_focus_ptr,crate::input::ax_actions::focus_element,- |target| crate::input::ax_actions::is_element_focused(pid, target),+ |target| {+ crate::input::ax_actions::element_has_keyboard_focus(pid, target)+ },|| crate::input::keyboard::press_key(pid, &key, &m),)})diff --git a/scripts/ci/macos/run-hosted-rust-e2e.sh b/scripts/ci/macos/run-hosted-rust-e2e.shindex 01215d8453..0103449a82 100755--- a/scripts/ci/macos/run-hosted-rust-e2e.sh+++ b/scripts/ci/macos/run-hosted-rust-e2e.sh@@ -375,6 +375,8 @@ codesign --verify --deep --strict "${LOCAL_APP}"# macOS 15+ separately reminds each responsible app before direct capture,# even when Screen Recording is granted. The hosted image pre-approves its# runner agent, but this signed app is the responsible ScreenCaptureKit client.+# The driver reads this bundle-identifier entry as recorded consent; without+# it, implicit captures refuse with direct_capture_consent_required (CUA-1792).echo "[CAPTURE] Suppressing the app-specific private-window-picker reminder"mark_phase "screen-capture-approval"mkdir -p "$(dirname "${SCREEN_CAPTURE_APPROVALS}")"