Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

fix(cua-driver): capture macOS desktops without relying on PATH (#3755)

injaneityf67be123e198
diff --git a/libs/cua-driver/rust/crates/cua-driver/tests/macos_capture_environment_test.rs b/libs/cua-driver/rust/crates/cua-driver/tests/macos_capture_environment_test.rs
new file mode 100644
index 0000000000..f3078b4b77
--- /dev/null
+++ b/libs/cua-driver/rust/crates/cua-driver/tests/macos_capture_environment_test.rs
@@ -0,0 +1,89 @@
+#![cfg(target_os = "macos")]
+
+use std::path::Path;
+use std::process::{Command, Stdio};
+use std::time::{Duration, Instant};
+
+use cua_driver_testkit::{driver_binary, Driver, McpDriver};
+use serde_json::json;
+
+struct StopDaemon<'a> {
+ binary: &'a Path,
+ socket: &'a Path,
+}
+
+impl Drop for StopDaemon<'_> {
+ fn drop(&mut self) {
+ let _ = Command::new(self.binary)
+ .args(["stop", "--socket"])
+ .arg(self.socket)
+ .stdout(Stdio::null())
+ .stderr(Stdio::null())
+ .status();
+ }
+}
+
+#[test]
+#[ignore = "requires an unlocked macOS desktop and a TCC-authorized installed driver"]
+fn desktop_capture_does_not_require_sbin_on_path() {
+ let binary = driver_binary().canonicalize().expect("installed driver");
+ let app = binary.ancestors().nth(3).expect("installed app bundle");
+ assert_eq!(app.extension().and_then(|s| s.to_str()), Some("app"));
+ for path in ["/usr/bin:/bin:/usr/sbin:/sbin", "/usr/bin:/bin"] {
+ eprintln!("desktop capture with daemon PATH={path}");
+ let output = tempfile::tempdir_in("/tmp").unwrap();
+ let socket = output.path().join("d.sock");
+ let _stop = StopDaemon {
+ binary: &binary,
+ socket: &socket,
+ };
+ assert!(Command::new("/usr/bin/open")
+ .args([
+ "-n",
+ "-g",
+ "--env",
+ &format!("PATH={path}"),
+ "--env",
+ "CUA_DRIVER_RS_TELEMETRY_ENABLED=false"
+ ])
+ .arg(app)
+ .args(["--args", "serve", "--socket"])
+ .arg(&socket)
+ .args([
+ "--permission-mode",
+ "unrestricted",
+ "--dangerously-bypass-approvals",
+ "--no-overlay"
+ ])
+ .status()
+ .expect("launch the installed app")
+ .success());
+ let deadline = Instant::now() + Duration::from_secs(10);
+ while !socket.exists() {
+ assert!(
+ Instant::now() < deadline,
+ "owned daemon did not create its socket"
+ );
+ std::thread::sleep(Duration::from_millis(50));
+ }
+ let mut driver = McpDriver::spawn_daemon_proxy_unrecorded(socket.to_str().unwrap())
+ .expect("connect to the owned daemon");
+ let session = driver.call(
+ "start_session",
+ json!({"session": "capture-path", "capture_scope": "desktop"}),
+ );
+ assert!(!session.is_error(), "{}", session.text());
+ let png = output.path().join("desktop.png");
+ let result = driver.call(
+ "get_desktop_state",
+ json!({"session": "capture-path", "screenshot_out_file": png}),
+ );
+ assert!(!result.is_error(), "{}", result.text());
+ let image = image::open(&png).expect("decode the captured desktop PNG");
+ let metadata = result.structured();
+ assert_eq!(metadata["screenshot_mime_type"], "image/png");
+ assert_eq!(metadata["screenshot_width"], image.width());
+ assert_eq!(metadata["screenshot_height"], image.height());
+ eprintln!("decoded desktop PNG: {}x{}", image.width(), image.height());
+ }
+}
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/capture.rs b/libs/cua-driver/rust/crates/platform-macos/src/capture.rs
index 88ea72f1ae..b0eea64447 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/capture.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/capture.rs
@@ -748,7 +748,7 @@ pub fn screenshot_display_bytes() -> anyhow::Result<Vec<u8>> {
let capture = SecureCapturePath::new("display.png")?;
let tmp_path = capture.file.to_string_lossy().into_owned();
- let output = Command::new("screencapture")
+ let output = Command::new("/usr/sbin/screencapture")
.args(["-x", &tmp_path])
.output()?;
diff --git a/scripts/ci/macos/run-rust-e2e.sh b/scripts/ci/macos/run-rust-e2e.sh
index 8be9ad4da2..ce540a36ff 100755
--- a/scripts/ci/macos/run-rust-e2e.sh
+++ b/scripts/ci/macos/run-rust-e2e.sh
@@ -399,6 +399,9 @@ fi
if [[ "${SUITE}" == capture || "${SUITE}" == all ]]; then
run_test capture-contract cargo test -p cua-driver --test capture_contract_test -- \
--ignored --nocapture --test-threads=1
+ run_test capture-environment env CUA_TEST_DRIVER_BIN="${MACOS_DAEMON_BIN}" \
+ cargo test -p cua-driver --test macos_capture_environment_test -- \
+ --ignored --nocapture --test-threads=1
run_test desktop-scope cargo test -p cua-driver --test desktop_scope_macos_test -- \
--ignored --nocapture --test-threads=1
fi