Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
fix(cua-driver): ask for macOS's direct-capture consent during setup, not over the user's window (CUA-1792) (#5352)
Francesco Bonaccia75d1c2e9ba2
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/capture.rs b/libs/cua-driver/rust/crates/platform-macos/src/capture.rsindex 0db5c6f10a..fd519a81c9 100644--- a/libs/cua-driver/rust/crates/platform-macos/src/capture.rs+++ b/libs/cua-driver/rust/crates/platform-macos/src/capture.rs@@ -749,6 +749,7 @@ fn screenshot_window_bytes_sck(window_id: u32) -> anyhow::Result<Vec<u8>> {/// Tries ScreenCaptureKit first; falls back to the `screencapture` CLI on/// native error or empty output.pub fn screenshot_window_bytes(window_id: u32) -> anyhow::Result<Vec<u8>> {+ refuse_without_direct_capture_consent()?;let off_space = crate::windows::window_on_current_space_by_id(window_id) == Some(false);capture_window_with_space_route(window_id,@@ -887,6 +888,7 @@ pub fn screenshot_window(window_id: u32) -> anyhow::Result<(String, u32, u32)> {/// Capture the full main display./// Returns raw PNG bytes or an error.pub fn screenshot_display_bytes() -> anyhow::Result<Vec<u8>> {+ refuse_without_direct_capture_consent()?;let capture = SecureCapturePath::new("display.png")?;let tmp_path = capture.file.to_string_lossy().into_owned();@@ -966,6 +968,7 @@ pub fn screenshot_display_bytes_excluding_overlay() -> anyhow::Result<(cursor_overlay::capture_exclusion::AgentOverlayCapture,)> {use cursor_overlay::capture_exclusion::{capture_excluding_overlays, ResidualCheck};+ refuse_without_direct_capture_consent()?;capture_excluding_overlays(&OverlayExcluder, |overlay| {let Some(&overlay_window_id) = overlay else {return Ok((screenshot_display_bytes()?, ResidualCheck::Clean));@@ -1051,6 +1054,42 @@ fn screenshot_main_display_sck_excluding(excluded_window_id: u32) -> anyhow::Rescgimage_to_png(&image, "main display")}+/// Skip an implicit capture that would make macOS show its one-time+/// direct-capture consent over the user's window (CUA-1792).+fn refuse_without_direct_capture_consent() -> anyhow::Result<()> {+ match crate::capture_consent::capture_blocked_reason() {+ Some(reason) => Err(anyhow::anyhow!(reason)),+ None => Ok(()),+ }+}++/// Take one tiny frame of the main display through ScreenCaptureKit.+///+/// Listing shareable content does not make macOS ask for its direct-capture+/// consent; the first real capture does. The explicit grant flow calls this so+/// the consent appears there, while the user expects system prompts, and not+/// later over their window. Returns whether the frame came back.+pub fn capture_probe_frame() -> bool {+ use screencapturekit::prelude::{SCContentFilter, SCShareableContent, SCStreamConfiguration};+ use screencapturekit::screenshot_manager::SCScreenshotManager;++ let Ok(content) = SCShareableContent::get() else {+ return false;+ };+ let Some(display) = content.displays().into_iter().next() else {+ return false;+ };+ let filter = SCContentFilter::create()+ .with_display(&display)+ .with_excluding_windows(&[])+ .build();+ let config = SCStreamConfiguration::new()+ .with_width(16)+ .with_height(16)+ .with_shows_cursor(false);+ SCScreenshotManager::capture_image(&filter, &config).is_ok()+}+/// Capture the main display and return (base64-encoded PNG, width, height).pub fn screenshot_display() -> anyhow::Result<(String, u32, u32)> {let bytes = screenshot_display_bytes()?;diff --git a/libs/cua-driver/rust/crates/platform-macos/src/capture_consent.rs b/libs/cua-driver/rust/crates/platform-macos/src/capture_consent.rsnew file mode 100644index 0000000000..8d00831949--- /dev/null+++ b/libs/cua-driver/rust/crates/platform-macos/src/capture_consent.rs@@ -0,0 +1,288 @@+//! macOS's one-time direct screen-capture consent (CUA-1792).+//!+//! From macOS 15, the first time an app captures the screen without the+//! system picker, `replayd` shows "<App> is requesting to bypass the system+//! private window picker and directly access your screen and audio". The+//! alert is a `UserNotificationCenter` window: it takes the front and key+//! focus from whatever the user is typing in.+//!+//! Measured on macOS 26.5.2 in a fresh VM with Screen Recording granted:+//! - Every capture path raises it: `SCScreenshotManager`,+//! `SLSHWCaptureWindowList`, `CGWindowListCreateImage` and the+//! `screencapture` CLI. Switching APIs does not avoid it.+//! - Listing `SCShareableContent` does not raise it.+//! - `replayd` records the app in+//! `~/Library/Group Containers/group.com.apple.replayd/ScreenCaptureApprovals.plist`,+//! keyed by the app bundle's file URL, as soon as the alert is shown. With+//! an entry present, later captures (also after the entry's hint date)+//! raise nothing.+//!+//! So the driver surfaces the consent in `cua-driver permissions grant`,+//! where the user expects system prompts (see `check_permissions`), and+//! skips implicit captures while no entry exists, rather than letting the+//! alert pop over the user's window in the middle of background work.++use std::path::{Path, PathBuf};+use std::sync::atomic::{AtomicBool, Ordering};+use std::sync::Mutex;+use std::time::{Duration, Instant};++/// Machine-readable code carried in the capture error.+pub const CONSENT_REQUIRED_CODE: &str = "direct_capture_consent_required";++/// Set to `1` to capture anyway and let macOS show its alert (for unattended+/// machines where nobody is at the screen).+pub const ALLOW_ALERT_ENV: &str = "CUA_DRIVER_ALLOW_CAPTURE_CONSENT_ALERT";++const APPROVALS_RELATIVE_PATH: &str =+ "Library/Group Containers/group.com.apple.replayd/ScreenCaptureApprovals.plist";++/// How long a "no consent yet" answer is reused before the file is read again.+const NEGATIVE_RECHECK: Duration = Duration::from_secs(2);++/// Error when an implicit capture would raise the consent alert, `None` when+/// the capture may run.+pub fn capture_blocked_reason() -> Option<String> {+ static RECORDED: AtomicBool = AtomicBool::new(false);+ static LAST_MISS: Mutex<Option<Instant>> = Mutex::new(None);++ if RECORDED.load(Ordering::Relaxed) {+ return None;+ }+ if std::env::var(ALLOW_ALERT_ENV).is_ok_and(|value| value == "1") {+ return None;+ }+ let Some(bundle) = own_app_bundle() else {+ // A bare binary is not the responsible app; its alert names the+ // terminal or host, which this check cannot look up.+ return None;+ };+ if !macos_has_direct_capture_consent() {+ RECORDED.store(true, Ordering::Relaxed);+ return None;+ }+ if let Ok(last) = LAST_MISS.lock() {+ if last.is_some_and(|at| at.elapsed() < NEGATIVE_RECHECK) {+ return Some(blocked_message(&bundle));+ }+ }+ let home = std::env::var_os("HOME")?;+ match consent_state(&Path::new(&home).join(APPROVALS_RELATIVE_PATH), &bundle) {+ ConsentState::Recorded | ConsentState::Unknown => {+ RECORDED.store(true, Ordering::Relaxed);+ None+ }+ ConsentState::Missing => {+ if let Ok(mut last) = LAST_MISS.lock() {+ *last = Some(Instant::now());+ }+ Some(blocked_message(&bundle))+ }+ }+}++fn blocked_message(bundle: &Path) -> String {+ format!(+ "{CONSENT_REQUIRED_CODE}: screen capture skipped. macOS has not yet asked whether {} \+ may capture the screen directly, and the first capture would show that system alert \+ over the user's window and take their keyboard focus. Ask the user to run \+ `cua-driver permissions grant` and choose Allow once; captures then work. The \+ accessibility tree is unaffected. Set {ALLOW_ALERT_ENV}=1 on the daemon to capture \+ anyway on a machine where nobody is at the screen.",+ bundle+ .file_name()+ .and_then(|name| name.to_str())+ .unwrap_or("Cua Driver")+ )+}++#[derive(Debug, Clone, Copy, PartialEq, Eq)]+enum ConsentState {+ Recorded,+ Missing,+ /// The file exists but could not be read or parsed: do not block on a+ /// format this code does not understand.+ Unknown,+}++fn consent_state(approvals: &Path, bundle: &Path) -> ConsentState {+ let bytes = match std::fs::read(approvals) {+ Ok(bytes) => bytes,+ Err(error) if error.kind() == std::io::ErrorKind::NotFound => return ConsentState::Missing,+ Err(_) => return ConsentState::Unknown,+ };+ match approval_keys(&bytes) {+ Some(keys) if keys.iter().any(|key| key_names_bundle(key, bundle)) => {+ ConsentState::Recorded+ }+ Some(_) => ConsentState::Missing,+ None => ConsentState::Unknown,+ }+}++/// Top-level keys of the approvals property list (binary or XML).+fn approval_keys(bytes: &[u8]) -> Option<Vec<String>> {+ use core_foundation::base::TCFType;+ use core_foundation::data::CFData;+ use core_foundation::dictionary::CFDictionary;+ use core_foundation::propertylist::{+ create_with_data, kCFPropertyListImmutable, CFPropertyList,+ };+ use core_foundation::string::CFString;++ let (plist, _) = create_with_data(CFData::from_buffer(bytes), kCFPropertyListImmutable).ok()?;+ // SAFETY: `create_with_data` returns a +1 property list on success.+ let plist = unsafe { CFPropertyList::wrap_under_create_rule(plist.cast()) };+ let dictionary = plist.downcast_into::<CFDictionary>()?;+ let (keys, _) = dictionary.get_keys_and_values();+ Some(+ keys.into_iter()+ .filter_map(|key| {+ // SAFETY: `key` is a borrowed CF object owned by `dictionary`;+ // `wrap_under_get_rule` retains it for the CFString wrapper.+ let key = unsafe { core_foundation::base::CFType::wrap_under_get_rule(key) };+ key.downcast::<CFString>().map(|key| key.to_string())+ })+ .collect(),+ )+}++/// Whether an approvals key (`file:///…/CuaDriver.app/`) names `bundle`.+fn key_names_bundle(key: &str, bundle: &Path) -> bool {+ let Some(path) = key.strip_prefix("file://") else {+ return false;+ };+ let Some(decoded) = percent_decode(path) else {+ return false;+ };+ let decoded = decoded.trim_end_matches('/');+ let wanted = bundle.to_string_lossy();+ if decoded == wanted.trim_end_matches('/') {+ return true;+ }+ std::fs::canonicalize(decoded).is_ok_and(|resolved| resolved == bundle)+}++fn percent_decode(text: &str) -> Option<String> {+ let bytes = text.as_bytes();+ let mut out = Vec::with_capacity(bytes.len());+ let mut index = 0;+ while index < bytes.len() {+ if bytes[index] == b'%' {+ let hex = text.get(index + 1..index + 3)?;+ out.push(u8::from_str_radix(hex, 16).ok()?);+ index += 3;+ } else {+ out.push(bytes[index]);+ index += 1;+ }+ }+ String::from_utf8(out).ok()+}++/// The `.app` bundle this process runs from, canonicalized.+fn own_app_bundle() -> Option<PathBuf> {+ let executable = std::fs::canonicalize(std::env::current_exe().ok()?).ok()?;+ app_bundle_of(&executable)+}++fn app_bundle_of(executable: &Path) -> Option<PathBuf> {+ let macos = executable.parent()?;+ let contents = macos.parent()?;+ let bundle = contents.parent()?;+ let is_bundle = macos.file_name()? == "MacOS"+ && contents.file_name()? == "Contents"+ && bundle.extension()? == "app";+ is_bundle.then(|| bundle.to_path_buf())+}++/// macOS 15 introduced the direct-capture consent.+fn macos_has_direct_capture_consent() -> bool {+ crate::tools::macos_product_version()+ .and_then(|version| version.split('.').next()?.parse::<u32>().ok())+ .is_some_and(|major| major >= 15)+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn app_bundle_is_found_only_for_bundled_executables() {+ assert_eq!(+ app_bundle_of(Path::new(+ "/Applications/CuaDriver.app/Contents/MacOS/cua-driver"+ )),+ Some(PathBuf::from("/Applications/CuaDriver.app"))+ );+ assert_eq!(app_bundle_of(Path::new("/usr/local/bin/cua-driver")), None);+ assert_eq!(+ app_bundle_of(Path::new("/tmp/CuaDriver/Contents/MacOS/cua-driver")),+ None+ );+ }++ #[test]+ fn approval_key_matches_the_bundle_url() {+ let bundle = Path::new("/Users/lume/My Apps/CuaDriver.app");+ assert!(key_names_bundle(+ "file:///Users/lume/My%20Apps/CuaDriver.app/",+ bundle+ ));+ assert!(!key_names_bundle(+ "file:///Users/lume/Other/CuaDriver.app/",+ bundle+ ));+ assert!(!key_names_bundle(+ "/Users/lume/My Apps/CuaDriver.app",+ bundle+ ));+ }++ fn write_plist(path: &Path, xml_keys: &[&str]) {+ let entries: String = xml_keys+ .iter()+ .map(|key| {+ format!(+ "<key>{key}</key><dict><key>kScreenCapturePrivacyHintPolicy</key>\+ <integer>2592000</integer></dict>"+ )+ })+ .collect();+ std::fs::write(+ path,+ format!(+ "<?xml version=\"1.0\" encoding=\"UTF-8\"?><plist version=\"1.0\">\+ <dict>{entries}</dict></plist>"+ ),+ )+ .unwrap();+ }++ #[test]+ fn consent_state_reads_the_approvals_file() {+ let dir = std::env::temp_dir().join(format!("cua-consent-{}", std::process::id()));+ std::fs::create_dir_all(&dir).unwrap();+ let file = dir.join("ScreenCaptureApprovals.plist");+ let bundle = Path::new("/Applications/CuaDriver.app");++ let _ = std::fs::remove_file(&file);+ assert_eq!(consent_state(&file, bundle), ConsentState::Missing);++ write_plist(&file, &["file:///Applications/Other.app/"]);+ assert_eq!(consent_state(&file, bundle), ConsentState::Missing);++ write_plist(+ &file,+ &[+ "file:///Applications/Other.app/",+ "file:///Applications/CuaDriver.app/",+ ],+ );+ assert_eq!(consent_state(&file, bundle), ConsentState::Recorded);++ std::fs::write(&file, b"not a plist").unwrap();+ assert_eq!(consent_state(&file, bundle), ConsentState::Unknown);+ let _ = std::fs::remove_dir_all(&dir);+ }+}diff --git a/libs/cua-driver/rust/crates/platform-macos/src/lib.rs b/libs/cua-driver/rust/crates/platform-macos/src/lib.rsindex 376e3e90fb..329c629853 100644--- a/libs/cua-driver/rust/crates/platform-macos/src/lib.rs+++ b/libs/cua-driver/rust/crates/platform-macos/src/lib.rs@@ -24,6 +24,8 @@ pub mod browser;#[cfg(target_os = "macos")]pub mod capture;#[cfg(target_os = "macos")]+pub mod capture_consent;+#[cfg(target_os = "macos")]pub mod cursor;#[cfg(target_os = "macos")]pub mod document_edit;diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/check_permissions.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/check_permissions.rsindex 280f81c164..b7617427f1 100644--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/check_permissions.rs+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/check_permissions.rs@@ -94,11 +94,17 @@ fn current_direct_capture_evidence_store(source: &Value) -> Option<DirectCapture/// documents. `SCShareableContent::get()` does a live query: it only/// returns displays when the answering process can genuinely capture. When/// it disagrees with the preflight boolean, the preflight one is lying.+///+/// Listing content alone does not make macOS ask for its direct-capture+/// consent (measured on macOS 26.5), so the probe also takes one tiny frame:+/// the consent then appears during this explicit grant, not over the user's+/// window during later background work (CUA-1792).fn screen_recording_capturable() -> bool {use screencapturekit::prelude::SCShareableContent;- SCShareableContent::get()+ let listed = SCShareableContent::get().map(|c| !c.displays().is_empty())- .unwrap_or(false)+ .unwrap_or(false);+ listed && crate::capture::capture_probe_frame()}const DIRECT_CAPTURE_PROBE_TIMEOUT: Duration = Duration::from_secs(10);@@ -389,9 +395,9 @@ impl Tool for CheckPermissionsTool {let accessibility = accessibility_granted();let screen_recording = screen_recording_granted();// (A) Authoritative live probe — see `screen_recording_capturable`.- // SCShareableContent::get() can itself raise Tahoe's separate- // private-window-picker bypass consent. A status/read-only call must- // therefore never execute it. The explicit grant path opts in with+ // The probe's capture raises Tahoe's separate private-window-picker+ // bypass consent. A status/read-only call must therefore never+ // execute it. The explicit grant path opts in with// `prompt:true`, explains the dialog first, and verifies the result.let direct_capture_probe =if should_probe_direct_capture(should_prompt, screen_recording, probe_direct_capture) {diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/health_report.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/health_report.rsindex 15e27369a7..69bf12ab02 100644--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/health_report.rs+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/health_report.rs@@ -371,7 +371,7 @@ fn current_bundle_identifier() -> Option<String> {/// Read in-process from the `kern.osproductversion` sysctl. The previous/// implementation spawned `/usr/bin/sw_vers` — a full fork/exec (several/// ms) on every health_report call — to read the same value.-fn sw_vers_product_version() -> Option<String> {+pub(crate) fn sw_vers_product_version() -> Option<String> {let name = std::ffi::CString::new("kern.osproductversion").ok()?;let mut buf = [0u8; 64];let mut len = buf.len();diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/mod.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/mod.rsindex cd39cdd64f..b45e8b603b 100644--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/mod.rs+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/mod.rs@@ -264,6 +264,7 @@ pub use check_permissions::{};pub use cua_driver_core::snapshot_store::ZoomContext;+pub(crate) use health_report::sw_vers_product_version as macos_product_version;/// The shared per-call delivery mode; see [`cua_driver_core::delivery`].///