Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
build(cua-driver): make Rust harness support portable
Francesco Bonacci150586d48eaf
diff --git a/libs/cua-driver/rust/.cargo/config.toml b/libs/cua-driver/rust/.cargo/config.tomlindex ac2b23f814..1cd4eddc90 100644--- a/libs/cua-driver/rust/.cargo/config.toml+++ b/libs/cua-driver/rust/.cargo/config.toml@@ -1,2 +1,5 @@+[env]+MACOSX_DEPLOYMENT_TARGET = "13.0"+[target.x86_64-pc-windows-msvc]rustflags = ["-C", "target-feature=+crt-static"]diff --git a/libs/cua-driver/rust/crates/cua-driver-testkit/src/mcp.rs b/libs/cua-driver/rust/crates/cua-driver-testkit/src/mcp.rsindex f8af6d0537..78cdfadafb 100644--- a/libs/cua-driver/rust/crates/cua-driver-testkit/src/mcp.rs+++ b/libs/cua-driver/rust/crates/cua-driver-testkit/src/mcp.rs@@ -31,6 +31,11 @@ impl McpDriver {/// Returns `None` (with a skip message) if the binary isn't built — the/// caller's test should early-return so an un-built binary skips, not fails.pub fn spawn() -> Option<Self> {+ Self::spawn_with_env(&[])+ }++ /// Spawn the driver with extra environment variables set on the child.+ pub fn spawn_with_env(env: &[(&str, &str)]) -> Option<Self> {let bin = driver_binary();if !bin.exists() {eprintln!("[testkit] driver binary not built at {bin:?} — skipping");@@ -38,14 +43,16 @@ impl McpDriver {}let mut reaper = ChildReaper::new();- let mut driver = spawn_in_job(- Command::new(&bin)- .stdin(Stdio::piped())- .stdout(Stdio::piped())- .stderr(Stdio::null()),- )- .inspect_err(|e| eprintln!("[testkit] driver spawn failed: {e}"))- .ok()?;+ let mut cmd = Command::new(&bin);+ cmd.stdin(Stdio::piped())+ .stdout(Stdio::piped())+ .stderr(Stdio::null());+ for (key, value) in env {+ cmd.env(key, value);+ }+ let mut driver = spawn_in_job(&mut cmd)+ .inspect_err(|e| eprintln!("[testkit] driver spawn failed: {e}"))+ .ok()?;let stdin = driver.stdin.take().unwrap();let stdout = driver.stdout.take().unwrap();reaper.push(driver);@@ -67,11 +74,37 @@ impl McpDriver {}});- let mut d = McpDriver { reaper, stdin, rx, next_id: 2 };+ let mut d = McpDriver {+ reaper,+ stdin,+ rx,+ next_id: 2,+ };d.initialize();Some(d)}+ /// macOS GUI harness tests need the installed, TCC-authorized daemon path.+ ///+ /// A raw `target/debug/cua-driver` MCP process initializes AppKit for the+ /// cursor overlay and can lose Screen Recording-attributed window titles even+ /// when the shell's one-shot `cua-driver call` path can see them. The installed+ /// daemon is the product path agents use, so ignored GUI tests proxy through an+ /// already-running daemon and skip with a clear note when it is absent.+ #[cfg(target_os = "macos")]+ pub fn spawn_macos_daemon_proxy() -> Option<Self> {+ let home = std::env::var("HOME").unwrap_or_else(|_| "/tmp".into());+ let socket = format!("{home}/Library/Caches/cua-driver/cua-driver.sock");+ if std::os::unix::net::UnixStream::connect(&socket).is_err() {+ eprintln!(+ "[testkit] CuaDriver daemon not listening at {socket} — \+ run `./scripts/install-local.sh` and `open -n -g -a CuaDriver --args serve`; skipping"+ );+ return None;+ }+ Self::spawn_with_env(&[("CUA_DRIVER_RS_MCP_FORCE_PROXY", "1")])+ }+fn initialize(&mut self) {self.send(serde_json::json!({"jsonrpc": "2.0", "id": 1, "method": "initialize", "params": {}diff --git a/libs/cua-driver/rust/crates/cua-driver-testkit/src/response.rs b/libs/cua-driver/rust/crates/cua-driver-testkit/src/response.rsindex f373efdd02..e828766fa6 100644--- a/libs/cua-driver/rust/crates/cua-driver-testkit/src/response.rs+++ b/libs/cua-driver/rust/crates/cua-driver-testkit/src/response.rs@@ -21,7 +21,12 @@ pub struct ToolResponse {impl ToolResponse {pub(crate) fn new(text: String, structured: Value, is_error: bool, raw: Value) -> Self {- Self { text, structured, is_error, raw }+ Self {+ text,+ structured,+ is_error,+ raw,+ }}/// Build from an MCP JSON-RPC response envelope.@@ -31,8 +36,8 @@ impl ToolResponse {.unwrap_or("").to_string();let structured = raw["result"]["structuredContent"].clone();- let is_error = raw["result"]["isError"].as_bool().unwrap_or(false)- || raw.get("error").is_some();+ let is_error =+ raw["result"]["isError"].as_bool().unwrap_or(false) || raw.get("error").is_some();Self::new(text, structured, is_error, raw)}@@ -47,6 +52,19 @@ impl ToolResponse {&self.structured}+ /// The full accessibility tree text from `get_window_state`.+ ///+ /// Older/direct MCP responses often put the rendered tree in the text content,+ /// while daemon-proxied responses carry a short summary in text and the tree+ /// in `structuredContent.tree_markdown`.+ pub fn tree_text(&self) -> &str {+ self.structured+ .get("tree_markdown")+ .and_then(Value::as_str)+ .filter(|s| !s.is_empty())+ .unwrap_or(&self.text)+ }+/// Whether the call errored (MCP `isError`/`error`, or CLI nonzero exit).pub fn is_error(&self) -> bool {self.is_error@@ -75,6 +93,9 @@ impl ToolResponse {/// `get_window_state` degraded flag: an AX walk that ran but returned zero/// actionable elements (non-AX surface / tree not ready). `false` when absent.pub fn degraded(&self) -> bool {- self.structured.get("degraded").and_then(Value::as_bool).unwrap_or(false)+ self.structured+ .get("degraded")+ .and_then(Value::as_bool)+ .unwrap_or(false)}}diff --git a/libs/cua-driver/rust/crates/cua-driver/build.rs b/libs/cua-driver/rust/crates/cua-driver/build.rsindex 0c11c96116..a47b84cc64 100644--- a/libs/cua-driver/rust/crates/cua-driver/build.rs+++ b/libs/cua-driver/rust/crates/cua-driver/build.rs@@ -20,17 +20,74 @@ fn main() {if std::env::var("CARGO_CFG_TARGET_OS").as_deref() != Ok("macos") {return;}+ emit_sdk_framework_search_path();+ emit_swift_runtime_link_args();+}++fn emit_sdk_framework_search_path() {+ let sdk_root = std::env::var("SDKROOT").unwrap_or_else(|_| {+ std::process::Command::new("xcrun")+ .args(["--sdk", "macosx", "--show-sdk-path"])+ .output()+ .ok()+ .and_then(|out| String::from_utf8(out.stdout).ok())+ .map(|out| out.trim().to_owned())+ .unwrap_or_default()+ });++ if !sdk_root.is_empty() {+ println!("cargo:rustc-link-search=framework={sdk_root}/System/Library/Frameworks");+ }+}++fn emit_swift_runtime_link_args() {+ use std::collections::BTreeSet;+ use std::path::{Path, PathBuf};+ use std::process::Command;+println!("cargo:rustc-link-arg=-Wl,-rpath,/usr/lib/swift");- if let Ok(out) = std::process::Command::new("xcode-select").arg("-p").output() {+ let mut dirs = BTreeSet::new();++ if let Ok(out) = std::process::Command::new("xcode-select")+ .arg("-p")+ .output()+ {if out.status.success() {let xcode_path = String::from_utf8_lossy(&out.stdout).trim().to_string();+ let developer_dir = PathBuf::from(xcode_path);for sub in ["Toolchains/XcodeDefault.xctoolchain/usr/lib/swift/macosx","Toolchains/XcodeDefault.xctoolchain/usr/lib/swift-5.5/macosx",+ "Toolchains/XcodeDefault.xctoolchain/usr/lib/swift-6.2/macosx",+ "usr/lib/swift/macosx",+ "usr/lib/swift-5.5/macosx",+ "usr/lib/swift-6.2/macosx",] {- println!("cargo:rustc-link-arg=-Wl,-rpath,{xcode_path}/{sub}");+ dirs.insert(developer_dir.join(sub));}}}++ if let Ok(out) = Command::new("xcrun").args(["--find", "swiftc"]).output() {+ if out.status.success() {+ let swiftc = PathBuf::from(String::from_utf8_lossy(&out.stdout).trim().to_string());+ if let Some(usr_dir) = swiftc.parent().and_then(Path::parent) {+ for sub in [+ "lib/swift/macosx",+ "lib/swift-5.5/macosx",+ "lib/swift-6.2/macosx",+ ] {+ dirs.insert(usr_dir.join(sub));+ }+ }+ }+ }++ for dir in dirs {+ if dir.is_dir() {+ println!("cargo:rustc-link-search=native={}", dir.display());+ println!("cargo:rustc-link-arg=-Wl,-rpath,{}", dir.display());+ }+ }}diff --git a/libs/cua-driver/rust/crates/cua-driver/src/bundle.rs b/libs/cua-driver/rust/crates/cua-driver/src/bundle.rsindex cf642fe786..1dd1d79fe6 100644--- a/libs/cua-driver/rust/crates/cua-driver/src/bundle.rs+++ b/libs/cua-driver/rust/crates/cua-driver/src/bundle.rs@@ -66,6 +66,7 @@ pub fn is_executable_inside_cuadriver_app() -> bool {/// Mirrors Swift's `if getppid() == 1 { return false }` gate in/// `MCPCommand.shouldUseDaemonProxy()`.#[cfg(unix)]+#[cfg_attr(not(target_os = "macos"), allow(dead_code))]pub fn parent_is_not_launchd() -> bool {// SAFETY: `libc::getppid` is a thread-safe POSIX getter that// takes no args and returns the parent pid. No invariants todiff --git a/libs/cua-driver/rust/crates/platform-macos/build.rs b/libs/cua-driver/rust/crates/platform-macos/build.rsindex 720b93b5f8..3e1a1e3e31 100644--- a/libs/cua-driver/rust/crates/platform-macos/build.rs+++ b/libs/cua-driver/rust/crates/platform-macos/build.rs@@ -30,22 +30,61 @@ fn main() {if !sdk_root.is_empty() {println!("cargo:rustc-link-search={sdk_root}/usr/lib/system");println!("cargo:rustc-link-search={sdk_root}/usr/lib");+ println!("cargo:rustc-link-search=framework={sdk_root}/System/Library/Frameworks");}// The ScreenCaptureKit bindings pull in Swift runtime libraries such as// libswift_Concurrency.dylib. The cua-driver binary emits these rpaths in// its own build.rs, but platform-macos' unit-test binary is linked from// this package, so it needs the same runtime search paths here.+ emit_swift_runtime_link_args();+}++fn emit_swift_runtime_link_args() {+ use std::collections::BTreeSet;+ use std::path::{Path, PathBuf};+ use std::process::Command;+println!("cargo:rustc-link-arg=-Wl,-rpath,/usr/lib/swift");- if let Ok(out) = std::process::Command::new("xcode-select").arg("-p").output() {++ let mut dirs = BTreeSet::new();++ if let Ok(out) = Command::new("xcode-select").arg("-p").output() {if out.status.success() {let xcode_path = String::from_utf8_lossy(&out.stdout).trim().to_string();+ let developer_dir = PathBuf::from(xcode_path);for sub in ["Toolchains/XcodeDefault.xctoolchain/usr/lib/swift/macosx","Toolchains/XcodeDefault.xctoolchain/usr/lib/swift-5.5/macosx",+ "Toolchains/XcodeDefault.xctoolchain/usr/lib/swift-6.2/macosx",+ "usr/lib/swift/macosx",+ "usr/lib/swift-5.5/macosx",+ "usr/lib/swift-6.2/macosx",] {- println!("cargo:rustc-link-arg=-Wl,-rpath,{xcode_path}/{sub}");+ dirs.insert(developer_dir.join(sub));}}}++ if let Ok(out) = Command::new("xcrun").args(["--find", "swiftc"]).output() {+ if out.status.success() {+ let swiftc = PathBuf::from(String::from_utf8_lossy(&out.stdout).trim().to_string());+ if let Some(usr_dir) = swiftc.parent().and_then(Path::parent) {+ for sub in [+ "lib/swift/macosx",+ "lib/swift-5.5/macosx",+ "lib/swift-6.2/macosx",+ ] {+ dirs.insert(usr_dir.join(sub));+ }+ }+ }+ }++ for dir in dirs {+ if dir.is_dir() {+ println!("cargo:rustc-link-search=native={}", dir.display());+ println!("cargo:rustc-link-arg=-Wl,-rpath,{}", dir.display());+ }+ }}