Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
fix(cua-driver): reconcile macOS launch readiness (#2697)
Francesco Bonacci771e0165e7ec
diff --git a/docs/content/docs/reference/cua-driver/mcp-tools.mdx b/docs/content/docs/reference/cua-driver/mcp-tools.mdxindex 4b4e41faad..bdc1a8e830 100644--- a/docs/content/docs/reference/cua-driver/mcp-tools.mdx+++ b/docs/content/docs/reference/cua-driver/mcp-tools.mdx@@ -164,7 +164,7 @@ Optional `creates_new_application_instance`: when true, forces a new app instancOptional `additional_arguments`: extra argv strings appended after --args.-Returns the launched app's pid, bundle_id, name, and a `windows` array (same shape as `list_windows`) so callers can skip an extra round-trip before `get_window_state(pid, window_id)`. When the focus-steal belt-and-braces demotion check ran (target pid ≠ prior frontmost), the response also includes `self_activation_suppressed: bool` — true if focus stayed with the prior frontmost, false if the launched app held focus despite the re-demote attempt.+Returns the launched app's pid, bundle_id, name, and a `windows` array (same shape as `list_windows`) so callers can skip an extra round-trip before `get_window_state(pid, window_id)`. `launch_state` distinguishes whether the request was sent, the process is running, and a window is ready. When the focus-steal belt-and-braces demotion check ran (target pid ≠ prior frontmost), the response also includes `self_activation_suppressed: bool` — true if focus stayed with the prior frontmost, false if the launched app held focus despite the re-demote attempt.**Arguments:**diff --git a/libs/cua-driver/rust/crates/platform-macos/src/apps/mod.rs b/libs/cua-driver/rust/crates/platform-macos/src/apps/mod.rsindex 110295617f..09324dbbdd 100644--- a/libs/cua-driver/rust/crates/platform-macos/src/apps/mod.rs+++ b/libs/cua-driver/rust/crates/platform-macos/src/apps/mod.rs@@ -2,6 +2,7 @@pub mod nsworkspace;+use anyhow::Context;use serde::{Deserialize, Serialize};use std::process::Command;@@ -98,7 +99,7 @@ pub fn launch_app(bundle_id: &str) -> anyhow::Result<i32> {..Default::default()};let running = nsworkspace::open_application(bundle_id, &cfg)- .map_err(|e| anyhow::anyhow!("Failed to launch {bundle_id}: {e}"))?;+ .with_context(|| format!("Failed to launch {bundle_id}"))?;let pid: i32 = unsafe { running.processIdentifier() };Ok(pid)}@@ -118,7 +119,7 @@ pub fn launch_app_by_name(name: &str) -> anyhow::Result<i32> {..Default::default()};let running = nsworkspace::open_application(&app_ref, &cfg)- .map_err(|e| anyhow::anyhow!("Failed to launch '{name}': {e}"))?;+ .with_context(|| format!("Failed to launch '{name}'"))?;let pid: i32 = unsafe { running.processIdentifier() };Ok(pid)}@@ -159,7 +160,7 @@ pub fn launch_with_urls_by_bundle(} else {nsworkspace::open_urls_with_application(urls, bundle_id, &cfg)}- .map_err(|e| anyhow::anyhow!("Failed to launch {bundle_id}: {e}"))?;+ .with_context(|| format!("Failed to launch {bundle_id}"))?;let pid: i32 = unsafe { running.processIdentifier() };Ok(pid)}@@ -190,7 +191,7 @@ pub fn launch_with_urls_by_name(} else {nsworkspace::open_urls_with_application(urls, &app_ref, &cfg)}- .map_err(|e| anyhow::anyhow!("Failed to launch '{name}': {e}"))?;+ .with_context(|| format!("Failed to launch '{name}'"))?;let pid: i32 = unsafe { running.processIdentifier() };Ok(pid)}diff --git a/libs/cua-driver/rust/crates/platform-macos/src/apps/nsworkspace.rs b/libs/cua-driver/rust/crates/platform-macos/src/apps/nsworkspace.rsindex 5dfeab60f7..7893a01322 100644--- a/libs/cua-driver/rust/crates/platform-macos/src/apps/nsworkspace.rs+++ b/libs/cua-driver/rust/crates/platform-macos/src/apps/nsworkspace.rs@@ -27,9 +27,10 @@//! selector which `objc2-foundation 0.2.2` does not bind natively — we//! hand-roll the binding in [`apple_event`] via `extern_methods!`.+use std::collections::HashSet;use std::ptr::NonNull;use std::sync::Arc;-use std::time::Duration;+use std::time::{Duration, Instant};use block2::RcBlock;use objc2::rc::Retained;@@ -72,6 +73,7 @@ pub struct OpenConfig {/// `open(...)` call returns `Err(LaunchError::Timeout)` instead of hanging/// the calling thread forever.const COMPLETION_TIMEOUT: Duration = Duration::from_secs(30);+const COMPLETION_POLL_INTERVAL: Duration = Duration::from_millis(50);/// Errors returned by the NSWorkspace launch helpers.#[derive(Debug, thiserror::Error)]@@ -80,7 +82,10 @@ pub enum LaunchError {Cocoa(String),#[error("NSWorkspace launch returned no NSRunningApplication and no NSError")]NoApp,- #[error("NSWorkspace launch did not complete within {:?}", COMPLETION_TIMEOUT)]+ #[error(+ "NSWorkspace launch callback did not complete within {:?}, and no matching running application was observed",+ COMPLETION_TIMEOUT+ )]Timeout,#[error("invalid url: {0}")]BadUrl(String),@@ -112,6 +117,7 @@ pub fn open_application(let ws = unsafe { NSWorkspace::sharedWorkspace() };let url = resolve_application_url(&ws, app_url)?;let config = build_configuration(cfg);+ let reconciliation = Reconciliation::new(cfg);let (tx, rx) = std::sync::mpsc::sync_channel::<CompletionResult>(1);// NSWorkspace may invoke its completion on another thread; retain atomic@@ -128,7 +134,7 @@ pub fn open_application(ws.openApplicationAtURL_configuration_completionHandler(&url, &config, Some(&block));}- wait_for_completion(rx)+ wait_for_completion(rx, reconciliation)}/// Launch the application bundle at `app_url` and hand it `urls` via@@ -147,6 +153,7 @@ pub fn open_urls_with_application(let ws = unsafe { NSWorkspace::sharedWorkspace() };let url = resolve_application_url(&ws, app_url)?;let config = build_configuration(cfg);+ let reconciliation = Reconciliation::new(cfg);let ns_urls: Vec<Retained<NSURL>> = urls.iter()@@ -174,7 +181,7 @@ pub fn open_urls_with_application();}- wait_for_completion(rx)+ wait_for_completion(rx, reconciliation)}// ── Internal helpers ─────────────────────────────────────────────────────────@@ -287,6 +294,75 @@ fn file_or_app_url(s: &str) -> Result<Retained<NSURL>, LaunchError> {type CompletionResult = Result<Retained<NSRunningApplication>, LaunchError>;+/// Actual-process fallback for daemons where LaunchServices accepts the+/// request but never delivers the completion block. A bundle id is required+/// because it is the only stable identity shared by the request and+/// `NSRunningApplication`.+struct Reconciliation {+ bundle_id: Option<String>,+ pids_before_request: HashSet<i32>,+ require_new_pid: bool,+}++impl Reconciliation {+ fn new(cfg: &OpenConfig) -> Self {+ let bundle_id = cfg.apple_event_bundle_id.clone();+ let pids_before_request = bundle_id+ .as_deref()+ .map(running_pids_for_bundle)+ .unwrap_or_default();+ Self {+ bundle_id,+ pids_before_request,+ require_new_pid: cfg.creates_new_instance,+ }+ }++ fn running_application(&self) -> Option<Retained<NSRunningApplication>> {+ let bundle_id = self.bundle_id.as_deref()?;+ running_application_for_bundle(bundle_id, &self.pids_before_request, self.require_new_pid)+ }+}++fn running_pids_for_bundle(bundle_id: &str) -> HashSet<i32> {+ let bundle_id = NSString::from_str(bundle_id);+ let running =+ unsafe { NSRunningApplication::runningApplicationsWithBundleIdentifier(&bundle_id) };+ let mut pids = HashSet::new();+ unsafe {+ for index in 0..running.count() {+ let app = running.objectAtIndex(index);+ let pid = app.processIdentifier();+ if pid > 0 && !app.isTerminated() {+ pids.insert(pid);+ }+ }+ }+ pids+}++fn running_application_for_bundle(+ bundle_id: &str,+ pids_before_request: &HashSet<i32>,+ require_new_pid: bool,+) -> Option<Retained<NSRunningApplication>> {+ let current_pids = running_pids_for_bundle(bundle_id);+ reconciliation_pid(¤t_pids, pids_before_request, require_new_pid).and_then(|pid| unsafe {+ NSRunningApplication::runningApplicationWithProcessIdentifier(pid)+ })+}++fn reconciliation_pid(+ current_pids: &HashSet<i32>,+ pids_before_request: &HashSet<i32>,+ require_new_pid: bool,+) -> Option<i32> {+ current_pids+ .iter()+ .copied()+ .find(|pid| !require_new_pid || !pids_before_request.contains(pid))+}+/// Build the `(NSRunningApplication?, NSError?) -> Void` completion block.////// Sends exactly one result through `tx`. Late completions (after timeout)@@ -321,15 +397,135 @@ fn make_completion_block()}-/// Block on `rx` for up to `COMPLETION_TIMEOUT`. Returns `Err(Timeout)` if-/// the completion handler never fires (wedged LaunchServices).+/// Wait for either the completion callback or actual process registration.+///+/// LaunchServices can accept an `openApplication...` request and register an+/// `NSRunningApplication` without delivering the callback to a background+/// daemon. Treating the callback as the sole success signal turns that state+/// into a false 30-second timeout. Poll the authoritative running-app state+/// while retaining the callback as the preferred source of Cocoa errors.fn wait_for_completion(rx: std::sync::mpsc::Receiver<CompletionResult>,+ reconciliation: Reconciliation,) -> Result<Retained<NSRunningApplication>, LaunchError> {- match rx.recv_timeout(COMPLETION_TIMEOUT) {- Ok(r) => r,- Err(std::sync::mpsc::RecvTimeoutError::Timeout) => Err(LaunchError::Timeout),- Err(std::sync::mpsc::RecvTimeoutError::Disconnected) => Err(LaunchError::NoApp),+ wait_for_launch_signal(rx, COMPLETION_TIMEOUT, COMPLETION_POLL_INTERVAL, || {+ reconciliation.running_application()+ })+}++fn wait_for_launch_signal<T>(+ rx: std::sync::mpsc::Receiver<Result<T, LaunchError>>,+ timeout: Duration,+ poll_interval: Duration,+ mut reconcile: impl FnMut() -> Option<T>,+) -> Result<T, LaunchError> {+ let deadline = Instant::now() + timeout;+ let mut callback_disconnected = false;++ loop {+ if !callback_disconnected {+ match rx.try_recv() {+ Ok(result) => return result,+ Err(std::sync::mpsc::TryRecvError::Empty) => {}+ Err(std::sync::mpsc::TryRecvError::Disconnected) => {+ callback_disconnected = true;+ }+ }+ }++ if let Some(running) = reconcile() {+ return Ok(running);+ }++ let now = Instant::now();+ if now >= deadline {+ return Err(LaunchError::Timeout);+ }+ let wait = poll_interval.min(deadline.saturating_duration_since(now));++ if callback_disconnected {+ std::thread::sleep(wait);+ continue;+ }++ match rx.recv_timeout(wait) {+ Ok(result) => return result,+ Err(std::sync::mpsc::RecvTimeoutError::Timeout) => {}+ Err(std::sync::mpsc::RecvTimeoutError::Disconnected) => {+ callback_disconnected = true;+ }+ }+ }+}++#[cfg(test)]+mod tests {+ use super::{reconciliation_pid, wait_for_launch_signal, LaunchError};+ use std::collections::HashSet;+ use std::sync::mpsc;+ use std::time::Duration;++ #[test]+ fn missing_callback_reconciles_registered_process() {+ let (_tx, rx) = mpsc::sync_channel(1);+ let mut polls = 0;+ let result = wait_for_launch_signal(+ rx,+ Duration::from_millis(100),+ Duration::from_millis(1),+ || {+ polls += 1;+ (polls == 3).then_some(4242)+ },+ );++ assert_eq!(result.unwrap(), 4242);+ }++ #[test]+ fn queued_callback_error_wins_over_process_reconciliation() {+ let (tx, rx) = mpsc::sync_channel(1);+ tx.send(Err(LaunchError::Cocoa("denied".to_owned())))+ .unwrap();++ let result = wait_for_launch_signal(+ rx,+ Duration::from_millis(100),+ Duration::from_millis(1),+ || Some(4242),+ );++ assert!(matches!(result, Err(LaunchError::Cocoa(message)) if message == "denied"));+ }++ #[test]+ fn no_callback_or_process_returns_precise_bounded_timeout() {+ let (_tx, rx) = mpsc::sync_channel::<Result<i32, LaunchError>>(1);+ let result = wait_for_launch_signal(+ rx,+ Duration::from_millis(5),+ Duration::from_millis(1),+ || None,+ );++ assert!(matches!(result, Err(LaunchError::Timeout)));+ }++ #[test]+ fn new_instance_reconciliation_rejects_preexisting_pid() {+ let before = HashSet::from([100]);+ assert_eq!(+ reconciliation_pid(&HashSet::from([100]), &before, true),+ None+ );+ assert_eq!(+ reconciliation_pid(&HashSet::from([100, 200]), &before, true),+ Some(200)+ );+ assert_eq!(+ reconciliation_pid(&HashSet::from([100]), &before, false),+ Some(100)+ );}}diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/launch_app.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/launch_app.rsindex 5eefcebde1..0d50c35a86 100644--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/launch_app.rs+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/launch_app.rs@@ -33,7 +33,9 @@ fn def() -> &'static ToolDef {Optional `additional_arguments`: extra argv strings appended after --args.\n\n\Returns the launched app's pid, bundle_id, name, and a `windows` array \(same shape as `list_windows`) so callers can skip an extra round-trip before \- `get_window_state(pid, window_id)`. When the focus-steal belt-and-braces \+ `get_window_state(pid, window_id)`. `launch_state` distinguishes whether the \+ request was sent, the process is running, and a window is ready. When the \+ focus-steal belt-and-braces \demotion check ran (target pid ≠ prior frontmost), the response also includes \`self_activation_suppressed: bool` — true if focus stayed with the prior \frontmost, false if the launched app held focus despite the re-demote attempt."@@ -87,6 +89,8 @@ impl Tool for LaunchAppTool {use cua_driver_core::tool_args::ArgsExt;let bundle_id = args.opt_str("bundle_id");let name = args.opt_str("name");+ let mut response_bundle_id = bundle_id.clone();+ let response_requested_name = name.clone();let urls: Vec<String> = args.str_array("urls");if args.get("cdp_debugging_port").is_some() {return ToolResult::error(@@ -136,6 +140,7 @@ impl Tool for LaunchAppTool {);};let (_, resolved_bundle_id) = locator.app_ref_and_bundle_id();+ response_bundle_id = resolved_bundle_id.clone();if resolved_bundle_id.as_deref().is_some_and(is_cua_driver_bundle_id)@@ -426,11 +431,11 @@ impl Tool for LaunchAppTool {match launch_result {Ok(Ok((pid, app_info, windows))) => {- let app_name = app_info.as_ref().map(|a| a.name.as_str()).unwrap_or("?");- let bid = app_info- .as_ref()- .and_then(|a| a.bundle_id.as_deref())- .unwrap_or("?");+ let (app_name, bid) = response_identity(+ app_info.as_ref(),+ response_bundle_id.as_deref(),+ response_requested_name.as_deref(),+ );let mut summary =format!("Launched {app_name} (pid {pid}) in background.{port_summary}");@@ -472,6 +477,7 @@ impl Tool for LaunchAppTool {"bundle_id": bid,"name": app_name,"windows": windows_json,+ "launch_state": launch_state(true, true, !windows.is_empty()),});// Only emit `self_activation_suppressed` when the// belt-and-braces demotion check actually ran. `None`@@ -483,7 +489,7 @@ impl Tool for LaunchAppTool {}ToolResult::text(summary).with_structured(structured)}- Ok(Err(e)) => ToolResult::error(format!("Launch failed: {e}")),+ Ok(Err(e)) => structured_launch_failure(&e),Err(e) => ToolResult::error(format!("Task error: {e}")),}}@@ -548,6 +554,77 @@ fn structured_launch_error(code: &str, message: String, details: serde_json::ValToolResult::error(message).with_structured(payload)}+fn launch_state(requested: bool, process_running: bool, window_ready: bool) -> serde_json::Value {+ serde_json::json!({+ "requested": requested,+ "process_running": process_running,+ "window_ready": window_ready,+ })+}++fn response_identity(+ app_info: Option<&crate::apps::AppInfo>,+ requested_bundle_id: Option<&str>,+ requested_name: Option<&str>,+) -> (String, String) {+ let bundle_id = app_info+ .and_then(|app| app.bundle_id.as_deref())+ .filter(|value| !value.is_empty())+ .or(requested_bundle_id)+ .unwrap_or("?")+ .to_owned();++ let name = app_info+ .map(|app| app.name.as_str())+ .filter(|value| !value.is_empty())+ .map(str::to_owned)+ .unwrap_or_else(|| requested_app_name(requested_name, requested_bundle_id));++ (name, bundle_id)+}++fn requested_app_name(requested_name: Option<&str>, requested_bundle_id: Option<&str>) -> String {+ if let Some(name) = requested_name.filter(|name| Some(*name) != requested_bundle_id) {+ let file_name = std::path::Path::new(name)+ .file_name()+ .and_then(|value| value.to_str())+ .unwrap_or(name);+ return file_name+ .strip_suffix(".app")+ .unwrap_or(file_name)+ .to_owned();+ }++ requested_bundle_id+ .and_then(|bundle_id| bundle_id.rsplit('.').next())+ .filter(|name| !name.is_empty())+ .unwrap_or("?")+ .to_owned()+}++fn structured_launch_failure(error: &anyhow::Error) -> ToolResult {+ use crate::apps::nsworkspace::LaunchError;++ let (code, requested) = if let Some(launch_error) = error.downcast_ref::<LaunchError>() {+ match launch_error {+ LaunchError::Cocoa(_) => ("NSWORKSPACE_LAUNCH_FAILED", true),+ LaunchError::NoApp => ("LAUNCH_RESULT_MISSING", true),+ LaunchError::Timeout => ("LAUNCH_CALLBACK_TIMEOUT", true),+ LaunchError::BadUrl(_) => ("APP_URL_INVALID", false),+ }+ } else {+ ("LAUNCH_FAILED", false)+ };++ structured_launch_error(+ code,+ format!("Launch failed: {error:#}"),+ serde_json::json!({+ "launch_state": launch_state(requested, false, false),+ }),+ )+}+fn preflight_file_urls(urls: &[String]) -> Option<ToolResult> {for raw in urls {let Some(path) = local_file_target(raw) else {@@ -633,7 +710,7 @@ fn hex_value(byte: u8) -> Option<u8> {mod tests {use super::{contains_remote_debugging_flag, is_cua_driver_bundle_id, local_file_target,- preflight_file_urls, LaunchAppTool,+ preflight_file_urls, response_identity, structured_launch_failure, LaunchAppTool,};use cua_driver_core::tool::Tool;use serde_json::json;@@ -700,6 +777,51 @@ mod tests {assert!(!is_cua_driver_bundle_id("com.trycua.harness.tauri"));}+ #[test]+ fn launch_timeout_reports_requested_without_process_or_window() {+ let error = anyhow::Error::new(crate::apps::nsworkspace::LaunchError::Timeout)+ .context("Failed to launch com.example.App");+ let result = structured_launch_failure(&error);+ let structured = result.structured_content.expect("structured error");++ assert_eq!(result.is_error, Some(true));+ assert_eq!(structured["error"], "LAUNCH_CALLBACK_TIMEOUT");+ assert_eq!(structured["launch_state"]["requested"], true);+ assert_eq!(structured["launch_state"]["process_running"], false);+ assert_eq!(structured["launch_state"]["window_ready"], false);+ }++ #[test]+ fn invalid_url_reports_request_was_not_sent() {+ let error = anyhow::Error::new(crate::apps::nsworkspace::LaunchError::BadUrl(+ "bad url".to_owned(),+ ))+ .context("Failed to launch com.example.App");+ let result = structured_launch_failure(&error);+ let structured = result.structured_content.expect("structured error");++ assert_eq!(structured["error"], "APP_URL_INVALID");+ assert_eq!(structured["launch_state"]["requested"], false);+ assert_eq!(structured["launch_state"]["process_running"], false);+ assert_eq!(structured["launch_state"]["window_ready"], false);+ }++ #[test]+ fn process_only_response_falls_back_to_requested_identity() {+ assert_eq!(+ response_identity(None, Some("com.apple.Safari"), None),+ ("Safari".to_owned(), "com.apple.Safari".to_owned())+ );+ assert_eq!(+ response_identity(+ None,+ Some("com.example.Editor"),+ Some("/Applications/Example Editor.app"),+ ),+ ("Example Editor".to_owned(), "com.example.Editor".to_owned())+ );+ }+#[tokio::test]async fn launch_app_cannot_reach_private_permission_host_entrypoint() {let result = LaunchAppTool