Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

fix(cua-driver): count a bundle-identifier capture approval as consent (fixes hosted macOS preflight) (#5402)

Francesco Bonacci8347d6cd849b
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/capture_consent.rs b/libs/cua-driver/rust/crates/platform-macos/src/capture_consent.rs
index 8d00831949..62752a2f05 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/capture_consent.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/capture_consent.rs
@@ -15,7 +15,9 @@
//! `~/Library/Group Containers/group.com.apple.replayd/ScreenCaptureApprovals.plist`,
//! keyed by the app bundle's file URL, as soon as the alert is shown. With
//! an entry present, later captures (also after the entry's hint date)
-//! raise nothing.
+//! raise nothing. An entry keyed by the bundle identifier also counts:
+//! that is how a managed or pre-provisioned machine (and the hosted CI
+//! runner) records the approval ahead of time.
//!
//! So the driver surfaces the consent in `cua-driver permissions grant`,
//! where the user expects system prompts (see `check_permissions`), and
@@ -67,7 +69,12 @@ pub fn capture_blocked_reason() -> Option<String> {
}
}
let home = std::env::var_os("HOME")?;
- match consent_state(&Path::new(&home).join(APPROVALS_RELATIVE_PATH), &bundle) {
+ let bundle_id = crate::tools::main_bundle_identifier();
+ match consent_state(
+ &Path::new(&home).join(APPROVALS_RELATIVE_PATH),
+ &bundle,
+ bundle_id.as_deref(),
+ ) {
ConsentState::Recorded | ConsentState::Unknown => {
RECORDED.store(true, Ordering::Relaxed);
None
@@ -105,14 +112,14 @@ enum ConsentState {
Unknown,
}
-fn consent_state(approvals: &Path, bundle: &Path) -> ConsentState {
+fn consent_state(approvals: &Path, bundle: &Path, bundle_id: Option<&str>) -> ConsentState {
let bytes = match std::fs::read(approvals) {
Ok(bytes) => bytes,
Err(error) if error.kind() == std::io::ErrorKind::NotFound => return ConsentState::Missing,
Err(_) => return ConsentState::Unknown,
};
match approval_keys(&bytes) {
- Some(keys) if keys.iter().any(|key| key_names_bundle(key, bundle)) => {
+ Some(keys) if keys.iter().any(|key| key_names_app(key, bundle, bundle_id)) => {
ConsentState::Recorded
}
Some(_) => ConsentState::Missing,
@@ -147,6 +154,12 @@ fn approval_keys(bytes: &[u8]) -> Option<Vec<String>> {
)
}
+/// Whether an approvals key names this app: its bundle file URL, as replayd
+/// writes it, or its bundle identifier, as a pre-provisioned approval does.
+fn key_names_app(key: &str, bundle: &Path, bundle_id: Option<&str>) -> bool {
+ bundle_id.is_some_and(|id| !id.is_empty() && key == id) || key_names_bundle(key, bundle)
+}
+
/// Whether an approvals key (`file:///…/CuaDriver.app/`) names `bundle`.
fn key_names_bundle(key: &str, bundle: &Path) -> bool {
let Some(path) = key.strip_prefix("file://") else {
@@ -265,12 +278,21 @@ mod tests {
std::fs::create_dir_all(&dir).unwrap();
let file = dir.join("ScreenCaptureApprovals.plist");
let bundle = Path::new("/Applications/CuaDriver.app");
+ let id = Some("com.trycua.driver");
let _ = std::fs::remove_file(&file);
- assert_eq!(consent_state(&file, bundle), ConsentState::Missing);
+ assert_eq!(consent_state(&file, bundle, id), ConsentState::Missing);
write_plist(&file, &["file:///Applications/Other.app/"]);
- assert_eq!(consent_state(&file, bundle), ConsentState::Missing);
+ assert_eq!(consent_state(&file, bundle, id), ConsentState::Missing);
+
+ write_plist(&file, &["com.trycua.driver"]);
+ assert_eq!(consent_state(&file, bundle, id), ConsentState::Recorded);
+ assert_eq!(consent_state(&file, bundle, None), ConsentState::Missing);
+ assert_eq!(
+ consent_state(&file, bundle, Some("com.trycua.driver.local")),
+ ConsentState::Missing
+ );
write_plist(
&file,
@@ -279,10 +301,10 @@ mod tests {
"file:///Applications/CuaDriver.app/",
],
);
- assert_eq!(consent_state(&file, bundle), ConsentState::Recorded);
+ assert_eq!(consent_state(&file, bundle, id), ConsentState::Recorded);
std::fs::write(&file, b"not a plist").unwrap();
- assert_eq!(consent_state(&file, bundle), ConsentState::Unknown);
+ assert_eq!(consent_state(&file, bundle, id), ConsentState::Unknown);
let _ = std::fs::remove_dir_all(&dir);
}
}
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/input/ax_actions.rs b/libs/cua-driver/rust/crates/platform-macos/src/input/ax_actions.rs
index 15eb1d2d13..0adce0e332 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/input/ax_actions.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/input/ax_actions.rs
@@ -283,6 +283,19 @@ pub fn is_element_focused(pid: i32, element_ptr: usize) -> bool {
}
}
+/// Report whether `element_ptr` holds keyboard focus, by either readback.
+///
+/// The application's `AXFocusedUIElement` is the stronger signal, but for
+/// web content (the Tauri fixture) in a background window it does not name
+/// the DOM control even after the control took focus, so a targeted
+/// background `press_key` was refused although the key would have landed.
+/// Accept the control's own `AXFocused = true` too. A write that AppKit
+/// clobbered reads back `false` either way.
+pub fn element_has_keyboard_focus(pid: i32, element_ptr: usize) -> bool {
+ is_element_focused(pid, element_ptr)
+ || unsafe { copy_bool_attr(element_ptr as AXUIElementRef, "AXFocused") } == Some(true)
+}
+
/// Set the AXValue of an element (for dropdowns, text fields, etc.).
pub fn set_ax_value(element_ptr: usize, value: &str) -> anyhow::Result<()> {
let err = unsafe { set_string_attr(element_ptr as AXUIElementRef, "AXValue", value) };
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/health_report.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/health_report.rs
index 69bf12ab02..fbca96e439 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/health_report.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/health_report.rs
@@ -348,7 +348,7 @@ fn bundle_identifier(bundle: &core_foundation::bundle::CFBundle) -> Option<Strin
/// Read the running process's `CFBundleIdentifier` via CoreFoundation.
/// Returns `None` when the process has no associated bundle (e.g.
/// running the bare binary outside `CuaDriver.app`).
-fn current_bundle_identifier() -> Option<String> {
+pub(crate) fn current_bundle_identifier() -> Option<String> {
use core_foundation::base::TCFType;
use core_foundation::bundle::{CFBundle, CFBundleGetMainBundle};
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/mod.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/mod.rs
index 4cde8b43bb..771e6717e3 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/mod.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/mod.rs
@@ -264,6 +264,7 @@ pub use check_permissions::{
};
pub use cua_driver_core::snapshot_store::ZoomContext;
+pub(crate) use health_report::current_bundle_identifier as main_bundle_identifier;
pub(crate) use health_report::sw_vers_product_version as macos_product_version;
/// The shared per-call delivery mode; see [`cua_driver_core::delivery`].
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/press_key.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/press_key.rs
index aec7eb7262..e4d007f0bc 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/press_key.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/press_key.rs
@@ -467,7 +467,7 @@ impl Tool for PressKeyTool {
pre_focus_ptr,
crate::input::ax_actions::focus_element,
|target| {
- crate::input::ax_actions::is_element_focused(
+ crate::input::ax_actions::element_has_keyboard_focus(
pid, target,
)
},
@@ -482,7 +482,9 @@ impl Tool for PressKeyTool {
with_confirmed_key_focus(
pre_focus_ptr,
crate::input::ax_actions::focus_element,
- |target| crate::input::ax_actions::is_element_focused(pid, target),
+ |target| {
+ crate::input::ax_actions::element_has_keyboard_focus(pid, target)
+ },
|| crate::input::keyboard::press_key(pid, &key, &m),
)
})
diff --git a/scripts/ci/macos/run-hosted-rust-e2e.sh b/scripts/ci/macos/run-hosted-rust-e2e.sh
index 01215d8453..0103449a82 100755
--- a/scripts/ci/macos/run-hosted-rust-e2e.sh
+++ b/scripts/ci/macos/run-hosted-rust-e2e.sh
@@ -375,6 +375,8 @@ codesign --verify --deep --strict "${LOCAL_APP}"
# macOS 15+ separately reminds each responsible app before direct capture,
# even when Screen Recording is granted. The hosted image pre-approves its
# runner agent, but this signed app is the responsible ScreenCaptureKit client.
+# The driver reads this bundle-identifier entry as recorded consent; without
+# it, implicit captures refuse with direct_capture_consent_required (CUA-1792).
echo "[CAPTURE] Suppressing the app-specific private-window-picker reminder"
mark_phase "screen-capture-approval"
mkdir -p "$(dirname "${SCREEN_CAPTURE_APPROVALS}")"