Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

fix(cua-driver): ask for macOS's direct-capture consent during setup, not over the user's window (CUA-1792) (#5352)

Francesco Bonaccia75d1c2e9ba2
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/capture.rs b/libs/cua-driver/rust/crates/platform-macos/src/capture.rs
index 0db5c6f10a..fd519a81c9 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/capture.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/capture.rs
@@ -749,6 +749,7 @@ fn screenshot_window_bytes_sck(window_id: u32) -> anyhow::Result<Vec<u8>> {
/// Tries ScreenCaptureKit first; falls back to the `screencapture` CLI on
/// native error or empty output.
pub fn screenshot_window_bytes(window_id: u32) -> anyhow::Result<Vec<u8>> {
+ refuse_without_direct_capture_consent()?;
let off_space = crate::windows::window_on_current_space_by_id(window_id) == Some(false);
capture_window_with_space_route(
window_id,
@@ -887,6 +888,7 @@ pub fn screenshot_window(window_id: u32) -> anyhow::Result<(String, u32, u32)> {
/// Capture the full main display.
/// Returns raw PNG bytes or an error.
pub fn screenshot_display_bytes() -> anyhow::Result<Vec<u8>> {
+ refuse_without_direct_capture_consent()?;
let capture = SecureCapturePath::new("display.png")?;
let tmp_path = capture.file.to_string_lossy().into_owned();
@@ -966,6 +968,7 @@ pub fn screenshot_display_bytes_excluding_overlay() -> anyhow::Result<(
cursor_overlay::capture_exclusion::AgentOverlayCapture,
)> {
use cursor_overlay::capture_exclusion::{capture_excluding_overlays, ResidualCheck};
+ refuse_without_direct_capture_consent()?;
capture_excluding_overlays(&OverlayExcluder, |overlay| {
let Some(&overlay_window_id) = overlay else {
return Ok((screenshot_display_bytes()?, ResidualCheck::Clean));
@@ -1051,6 +1054,42 @@ fn screenshot_main_display_sck_excluding(excluded_window_id: u32) -> anyhow::Res
cgimage_to_png(&image, "main display")
}
+/// Skip an implicit capture that would make macOS show its one-time
+/// direct-capture consent over the user's window (CUA-1792).
+fn refuse_without_direct_capture_consent() -> anyhow::Result<()> {
+ match crate::capture_consent::capture_blocked_reason() {
+ Some(reason) => Err(anyhow::anyhow!(reason)),
+ None => Ok(()),
+ }
+}
+
+/// Take one tiny frame of the main display through ScreenCaptureKit.
+///
+/// Listing shareable content does not make macOS ask for its direct-capture
+/// consent; the first real capture does. The explicit grant flow calls this so
+/// the consent appears there, while the user expects system prompts, and not
+/// later over their window. Returns whether the frame came back.
+pub fn capture_probe_frame() -> bool {
+ use screencapturekit::prelude::{SCContentFilter, SCShareableContent, SCStreamConfiguration};
+ use screencapturekit::screenshot_manager::SCScreenshotManager;
+
+ let Ok(content) = SCShareableContent::get() else {
+ return false;
+ };
+ let Some(display) = content.displays().into_iter().next() else {
+ return false;
+ };
+ let filter = SCContentFilter::create()
+ .with_display(&display)
+ .with_excluding_windows(&[])
+ .build();
+ let config = SCStreamConfiguration::new()
+ .with_width(16)
+ .with_height(16)
+ .with_shows_cursor(false);
+ SCScreenshotManager::capture_image(&filter, &config).is_ok()
+}
+
/// Capture the main display and return (base64-encoded PNG, width, height).
pub fn screenshot_display() -> anyhow::Result<(String, u32, u32)> {
let bytes = screenshot_display_bytes()?;
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/capture_consent.rs b/libs/cua-driver/rust/crates/platform-macos/src/capture_consent.rs
new file mode 100644
index 0000000000..8d00831949
--- /dev/null
+++ b/libs/cua-driver/rust/crates/platform-macos/src/capture_consent.rs
@@ -0,0 +1,288 @@
+//! macOS's one-time direct screen-capture consent (CUA-1792).
+//!
+//! From macOS 15, the first time an app captures the screen without the
+//! system picker, `replayd` shows "<App> is requesting to bypass the system
+//! private window picker and directly access your screen and audio". The
+//! alert is a `UserNotificationCenter` window: it takes the front and key
+//! focus from whatever the user is typing in.
+//!
+//! Measured on macOS 26.5.2 in a fresh VM with Screen Recording granted:
+//! - Every capture path raises it: `SCScreenshotManager`,
+//! `SLSHWCaptureWindowList`, `CGWindowListCreateImage` and the
+//! `screencapture` CLI. Switching APIs does not avoid it.
+//! - Listing `SCShareableContent` does not raise it.
+//! - `replayd` records the app in
+//! `~/Library/Group Containers/group.com.apple.replayd/ScreenCaptureApprovals.plist`,
+//! keyed by the app bundle's file URL, as soon as the alert is shown. With
+//! an entry present, later captures (also after the entry's hint date)
+//! raise nothing.
+//!
+//! So the driver surfaces the consent in `cua-driver permissions grant`,
+//! where the user expects system prompts (see `check_permissions`), and
+//! skips implicit captures while no entry exists, rather than letting the
+//! alert pop over the user's window in the middle of background work.
+
+use std::path::{Path, PathBuf};
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::sync::Mutex;
+use std::time::{Duration, Instant};
+
+/// Machine-readable code carried in the capture error.
+pub const CONSENT_REQUIRED_CODE: &str = "direct_capture_consent_required";
+
+/// Set to `1` to capture anyway and let macOS show its alert (for unattended
+/// machines where nobody is at the screen).
+pub const ALLOW_ALERT_ENV: &str = "CUA_DRIVER_ALLOW_CAPTURE_CONSENT_ALERT";
+
+const APPROVALS_RELATIVE_PATH: &str =
+ "Library/Group Containers/group.com.apple.replayd/ScreenCaptureApprovals.plist";
+
+/// How long a "no consent yet" answer is reused before the file is read again.
+const NEGATIVE_RECHECK: Duration = Duration::from_secs(2);
+
+/// Error when an implicit capture would raise the consent alert, `None` when
+/// the capture may run.
+pub fn capture_blocked_reason() -> Option<String> {
+ static RECORDED: AtomicBool = AtomicBool::new(false);
+ static LAST_MISS: Mutex<Option<Instant>> = Mutex::new(None);
+
+ if RECORDED.load(Ordering::Relaxed) {
+ return None;
+ }
+ if std::env::var(ALLOW_ALERT_ENV).is_ok_and(|value| value == "1") {
+ return None;
+ }
+ let Some(bundle) = own_app_bundle() else {
+ // A bare binary is not the responsible app; its alert names the
+ // terminal or host, which this check cannot look up.
+ return None;
+ };
+ if !macos_has_direct_capture_consent() {
+ RECORDED.store(true, Ordering::Relaxed);
+ return None;
+ }
+ if let Ok(last) = LAST_MISS.lock() {
+ if last.is_some_and(|at| at.elapsed() < NEGATIVE_RECHECK) {
+ return Some(blocked_message(&bundle));
+ }
+ }
+ let home = std::env::var_os("HOME")?;
+ match consent_state(&Path::new(&home).join(APPROVALS_RELATIVE_PATH), &bundle) {
+ ConsentState::Recorded | ConsentState::Unknown => {
+ RECORDED.store(true, Ordering::Relaxed);
+ None
+ }
+ ConsentState::Missing => {
+ if let Ok(mut last) = LAST_MISS.lock() {
+ *last = Some(Instant::now());
+ }
+ Some(blocked_message(&bundle))
+ }
+ }
+}
+
+fn blocked_message(bundle: &Path) -> String {
+ format!(
+ "{CONSENT_REQUIRED_CODE}: screen capture skipped. macOS has not yet asked whether {} \
+ may capture the screen directly, and the first capture would show that system alert \
+ over the user's window and take their keyboard focus. Ask the user to run \
+ `cua-driver permissions grant` and choose Allow once; captures then work. The \
+ accessibility tree is unaffected. Set {ALLOW_ALERT_ENV}=1 on the daemon to capture \
+ anyway on a machine where nobody is at the screen.",
+ bundle
+ .file_name()
+ .and_then(|name| name.to_str())
+ .unwrap_or("Cua Driver")
+ )
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+enum ConsentState {
+ Recorded,
+ Missing,
+ /// The file exists but could not be read or parsed: do not block on a
+ /// format this code does not understand.
+ Unknown,
+}
+
+fn consent_state(approvals: &Path, bundle: &Path) -> ConsentState {
+ let bytes = match std::fs::read(approvals) {
+ Ok(bytes) => bytes,
+ Err(error) if error.kind() == std::io::ErrorKind::NotFound => return ConsentState::Missing,
+ Err(_) => return ConsentState::Unknown,
+ };
+ match approval_keys(&bytes) {
+ Some(keys) if keys.iter().any(|key| key_names_bundle(key, bundle)) => {
+ ConsentState::Recorded
+ }
+ Some(_) => ConsentState::Missing,
+ None => ConsentState::Unknown,
+ }
+}
+
+/// Top-level keys of the approvals property list (binary or XML).
+fn approval_keys(bytes: &[u8]) -> Option<Vec<String>> {
+ use core_foundation::base::TCFType;
+ use core_foundation::data::CFData;
+ use core_foundation::dictionary::CFDictionary;
+ use core_foundation::propertylist::{
+ create_with_data, kCFPropertyListImmutable, CFPropertyList,
+ };
+ use core_foundation::string::CFString;
+
+ let (plist, _) = create_with_data(CFData::from_buffer(bytes), kCFPropertyListImmutable).ok()?;
+ // SAFETY: `create_with_data` returns a +1 property list on success.
+ let plist = unsafe { CFPropertyList::wrap_under_create_rule(plist.cast()) };
+ let dictionary = plist.downcast_into::<CFDictionary>()?;
+ let (keys, _) = dictionary.get_keys_and_values();
+ Some(
+ keys.into_iter()
+ .filter_map(|key| {
+ // SAFETY: `key` is a borrowed CF object owned by `dictionary`;
+ // `wrap_under_get_rule` retains it for the CFString wrapper.
+ let key = unsafe { core_foundation::base::CFType::wrap_under_get_rule(key) };
+ key.downcast::<CFString>().map(|key| key.to_string())
+ })
+ .collect(),
+ )
+}
+
+/// Whether an approvals key (`file:///…/CuaDriver.app/`) names `bundle`.
+fn key_names_bundle(key: &str, bundle: &Path) -> bool {
+ let Some(path) = key.strip_prefix("file://") else {
+ return false;
+ };
+ let Some(decoded) = percent_decode(path) else {
+ return false;
+ };
+ let decoded = decoded.trim_end_matches('/');
+ let wanted = bundle.to_string_lossy();
+ if decoded == wanted.trim_end_matches('/') {
+ return true;
+ }
+ std::fs::canonicalize(decoded).is_ok_and(|resolved| resolved == bundle)
+}
+
+fn percent_decode(text: &str) -> Option<String> {
+ let bytes = text.as_bytes();
+ let mut out = Vec::with_capacity(bytes.len());
+ let mut index = 0;
+ while index < bytes.len() {
+ if bytes[index] == b'%' {
+ let hex = text.get(index + 1..index + 3)?;
+ out.push(u8::from_str_radix(hex, 16).ok()?);
+ index += 3;
+ } else {
+ out.push(bytes[index]);
+ index += 1;
+ }
+ }
+ String::from_utf8(out).ok()
+}
+
+/// The `.app` bundle this process runs from, canonicalized.
+fn own_app_bundle() -> Option<PathBuf> {
+ let executable = std::fs::canonicalize(std::env::current_exe().ok()?).ok()?;
+ app_bundle_of(&executable)
+}
+
+fn app_bundle_of(executable: &Path) -> Option<PathBuf> {
+ let macos = executable.parent()?;
+ let contents = macos.parent()?;
+ let bundle = contents.parent()?;
+ let is_bundle = macos.file_name()? == "MacOS"
+ && contents.file_name()? == "Contents"
+ && bundle.extension()? == "app";
+ is_bundle.then(|| bundle.to_path_buf())
+}
+
+/// macOS 15 introduced the direct-capture consent.
+fn macos_has_direct_capture_consent() -> bool {
+ crate::tools::macos_product_version()
+ .and_then(|version| version.split('.').next()?.parse::<u32>().ok())
+ .is_some_and(|major| major >= 15)
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn app_bundle_is_found_only_for_bundled_executables() {
+ assert_eq!(
+ app_bundle_of(Path::new(
+ "/Applications/CuaDriver.app/Contents/MacOS/cua-driver"
+ )),
+ Some(PathBuf::from("/Applications/CuaDriver.app"))
+ );
+ assert_eq!(app_bundle_of(Path::new("/usr/local/bin/cua-driver")), None);
+ assert_eq!(
+ app_bundle_of(Path::new("/tmp/CuaDriver/Contents/MacOS/cua-driver")),
+ None
+ );
+ }
+
+ #[test]
+ fn approval_key_matches_the_bundle_url() {
+ let bundle = Path::new("/Users/lume/My Apps/CuaDriver.app");
+ assert!(key_names_bundle(
+ "file:///Users/lume/My%20Apps/CuaDriver.app/",
+ bundle
+ ));
+ assert!(!key_names_bundle(
+ "file:///Users/lume/Other/CuaDriver.app/",
+ bundle
+ ));
+ assert!(!key_names_bundle(
+ "/Users/lume/My Apps/CuaDriver.app",
+ bundle
+ ));
+ }
+
+ fn write_plist(path: &Path, xml_keys: &[&str]) {
+ let entries: String = xml_keys
+ .iter()
+ .map(|key| {
+ format!(
+ "<key>{key}</key><dict><key>kScreenCapturePrivacyHintPolicy</key>\
+ <integer>2592000</integer></dict>"
+ )
+ })
+ .collect();
+ std::fs::write(
+ path,
+ format!(
+ "<?xml version=\"1.0\" encoding=\"UTF-8\"?><plist version=\"1.0\">\
+ <dict>{entries}</dict></plist>"
+ ),
+ )
+ .unwrap();
+ }
+
+ #[test]
+ fn consent_state_reads_the_approvals_file() {
+ let dir = std::env::temp_dir().join(format!("cua-consent-{}", std::process::id()));
+ std::fs::create_dir_all(&dir).unwrap();
+ let file = dir.join("ScreenCaptureApprovals.plist");
+ let bundle = Path::new("/Applications/CuaDriver.app");
+
+ let _ = std::fs::remove_file(&file);
+ assert_eq!(consent_state(&file, bundle), ConsentState::Missing);
+
+ write_plist(&file, &["file:///Applications/Other.app/"]);
+ assert_eq!(consent_state(&file, bundle), ConsentState::Missing);
+
+ write_plist(
+ &file,
+ &[
+ "file:///Applications/Other.app/",
+ "file:///Applications/CuaDriver.app/",
+ ],
+ );
+ assert_eq!(consent_state(&file, bundle), ConsentState::Recorded);
+
+ std::fs::write(&file, b"not a plist").unwrap();
+ assert_eq!(consent_state(&file, bundle), ConsentState::Unknown);
+ let _ = std::fs::remove_dir_all(&dir);
+ }
+}
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/lib.rs b/libs/cua-driver/rust/crates/platform-macos/src/lib.rs
index 376e3e90fb..329c629853 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/lib.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/lib.rs
@@ -24,6 +24,8 @@ pub mod browser;
#[cfg(target_os = "macos")]
pub mod capture;
#[cfg(target_os = "macos")]
+pub mod capture_consent;
+#[cfg(target_os = "macos")]
pub mod cursor;
#[cfg(target_os = "macos")]
pub mod document_edit;
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/check_permissions.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/check_permissions.rs
index 280f81c164..b7617427f1 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/check_permissions.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/check_permissions.rs
@@ -94,11 +94,17 @@ fn current_direct_capture_evidence_store(source: &Value) -> Option<DirectCapture
/// documents. `SCShareableContent::get()` does a live query: it only
/// returns displays when the answering process can genuinely capture. When
/// it disagrees with the preflight boolean, the preflight one is lying.
+///
+/// Listing content alone does not make macOS ask for its direct-capture
+/// consent (measured on macOS 26.5), so the probe also takes one tiny frame:
+/// the consent then appears during this explicit grant, not over the user's
+/// window during later background work (CUA-1792).
fn screen_recording_capturable() -> bool {
use screencapturekit::prelude::SCShareableContent;
- SCShareableContent::get()
+ let listed = SCShareableContent::get()
.map(|c| !c.displays().is_empty())
- .unwrap_or(false)
+ .unwrap_or(false);
+ listed && crate::capture::capture_probe_frame()
}
const DIRECT_CAPTURE_PROBE_TIMEOUT: Duration = Duration::from_secs(10);
@@ -389,9 +395,9 @@ impl Tool for CheckPermissionsTool {
let accessibility = accessibility_granted();
let screen_recording = screen_recording_granted();
// (A) Authoritative live probe — see `screen_recording_capturable`.
- // SCShareableContent::get() can itself raise Tahoe's separate
- // private-window-picker bypass consent. A status/read-only call must
- // therefore never execute it. The explicit grant path opts in with
+ // The probe's capture raises Tahoe's separate private-window-picker
+ // bypass consent. A status/read-only call must therefore never
+ // execute it. The explicit grant path opts in with
// `prompt:true`, explains the dialog first, and verifies the result.
let direct_capture_probe =
if should_probe_direct_capture(should_prompt, screen_recording, probe_direct_capture) {
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/health_report.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/health_report.rs
index 15e27369a7..69bf12ab02 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/health_report.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/health_report.rs
@@ -371,7 +371,7 @@ fn current_bundle_identifier() -> Option<String> {
/// Read in-process from the `kern.osproductversion` sysctl. The previous
/// implementation spawned `/usr/bin/sw_vers` — a full fork/exec (several
/// ms) on every health_report call — to read the same value.
-fn sw_vers_product_version() -> Option<String> {
+pub(crate) fn sw_vers_product_version() -> Option<String> {
let name = std::ffi::CString::new("kern.osproductversion").ok()?;
let mut buf = [0u8; 64];
let mut len = buf.len();
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/mod.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/mod.rs
index cd39cdd64f..b45e8b603b 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/mod.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/mod.rs
@@ -264,6 +264,7 @@ pub use check_permissions::{
};
pub use cua_driver_core::snapshot_store::ZoomContext;
+pub(crate) use health_report::sw_vers_product_version as macos_product_version;
/// The shared per-call delivery mode; see [`cua_driver_core::delivery`].
///