Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
fix(cua-driver): capture macOS desktops without relying on PATH (#3755)
injaneityf67be123e198
diff --git a/libs/cua-driver/rust/crates/cua-driver/tests/macos_capture_environment_test.rs b/libs/cua-driver/rust/crates/cua-driver/tests/macos_capture_environment_test.rsnew file mode 100644index 0000000000..f3078b4b77--- /dev/null+++ b/libs/cua-driver/rust/crates/cua-driver/tests/macos_capture_environment_test.rs@@ -0,0 +1,89 @@+#![cfg(target_os = "macos")]++use std::path::Path;+use std::process::{Command, Stdio};+use std::time::{Duration, Instant};++use cua_driver_testkit::{driver_binary, Driver, McpDriver};+use serde_json::json;++struct StopDaemon<'a> {+ binary: &'a Path,+ socket: &'a Path,+}++impl Drop for StopDaemon<'_> {+ fn drop(&mut self) {+ let _ = Command::new(self.binary)+ .args(["stop", "--socket"])+ .arg(self.socket)+ .stdout(Stdio::null())+ .stderr(Stdio::null())+ .status();+ }+}++#[test]+#[ignore = "requires an unlocked macOS desktop and a TCC-authorized installed driver"]+fn desktop_capture_does_not_require_sbin_on_path() {+ let binary = driver_binary().canonicalize().expect("installed driver");+ let app = binary.ancestors().nth(3).expect("installed app bundle");+ assert_eq!(app.extension().and_then(|s| s.to_str()), Some("app"));+ for path in ["/usr/bin:/bin:/usr/sbin:/sbin", "/usr/bin:/bin"] {+ eprintln!("desktop capture with daemon PATH={path}");+ let output = tempfile::tempdir_in("/tmp").unwrap();+ let socket = output.path().join("d.sock");+ let _stop = StopDaemon {+ binary: &binary,+ socket: &socket,+ };+ assert!(Command::new("/usr/bin/open")+ .args([+ "-n",+ "-g",+ "--env",+ &format!("PATH={path}"),+ "--env",+ "CUA_DRIVER_RS_TELEMETRY_ENABLED=false"+ ])+ .arg(app)+ .args(["--args", "serve", "--socket"])+ .arg(&socket)+ .args([+ "--permission-mode",+ "unrestricted",+ "--dangerously-bypass-approvals",+ "--no-overlay"+ ])+ .status()+ .expect("launch the installed app")+ .success());+ let deadline = Instant::now() + Duration::from_secs(10);+ while !socket.exists() {+ assert!(+ Instant::now() < deadline,+ "owned daemon did not create its socket"+ );+ std::thread::sleep(Duration::from_millis(50));+ }+ let mut driver = McpDriver::spawn_daemon_proxy_unrecorded(socket.to_str().unwrap())+ .expect("connect to the owned daemon");+ let session = driver.call(+ "start_session",+ json!({"session": "capture-path", "capture_scope": "desktop"}),+ );+ assert!(!session.is_error(), "{}", session.text());+ let png = output.path().join("desktop.png");+ let result = driver.call(+ "get_desktop_state",+ json!({"session": "capture-path", "screenshot_out_file": png}),+ );+ assert!(!result.is_error(), "{}", result.text());+ let image = image::open(&png).expect("decode the captured desktop PNG");+ let metadata = result.structured();+ assert_eq!(metadata["screenshot_mime_type"], "image/png");+ assert_eq!(metadata["screenshot_width"], image.width());+ assert_eq!(metadata["screenshot_height"], image.height());+ eprintln!("decoded desktop PNG: {}x{}", image.width(), image.height());+ }+}diff --git a/libs/cua-driver/rust/crates/platform-macos/src/capture.rs b/libs/cua-driver/rust/crates/platform-macos/src/capture.rsindex 88ea72f1ae..b0eea64447 100644--- a/libs/cua-driver/rust/crates/platform-macos/src/capture.rs+++ b/libs/cua-driver/rust/crates/platform-macos/src/capture.rs@@ -748,7 +748,7 @@ pub fn screenshot_display_bytes() -> anyhow::Result<Vec<u8>> {let capture = SecureCapturePath::new("display.png")?;let tmp_path = capture.file.to_string_lossy().into_owned();- let output = Command::new("screencapture")+ let output = Command::new("/usr/sbin/screencapture").args(["-x", &tmp_path]).output()?;diff --git a/scripts/ci/macos/run-rust-e2e.sh b/scripts/ci/macos/run-rust-e2e.shindex 8be9ad4da2..ce540a36ff 100755--- a/scripts/ci/macos/run-rust-e2e.sh+++ b/scripts/ci/macos/run-rust-e2e.sh@@ -399,6 +399,9 @@ fiif [[ "${SUITE}" == capture || "${SUITE}" == all ]]; thenrun_test capture-contract cargo test -p cua-driver --test capture_contract_test -- \--ignored --nocapture --test-threads=1+ run_test capture-environment env CUA_TEST_DRIVER_BIN="${MACOS_DAEMON_BIN}" \+ cargo test -p cua-driver --test macos_capture_environment_test -- \+ --ignored --nocapture --test-threads=1run_test desktop-scope cargo test -p cua-driver --test desktop_scope_macos_test -- \--ignored --nocapture --test-threads=1fi