Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

fix(cua-driver): reconcile macOS launch readiness (#2697)

Francesco Bonacci771e0165e7ec
diff --git a/docs/content/docs/reference/cua-driver/mcp-tools.mdx b/docs/content/docs/reference/cua-driver/mcp-tools.mdx
index 4b4e41faad..bdc1a8e830 100644
--- a/docs/content/docs/reference/cua-driver/mcp-tools.mdx
+++ b/docs/content/docs/reference/cua-driver/mcp-tools.mdx
@@ -164,7 +164,7 @@ Optional `creates_new_application_instance`: when true, forces a new app instanc
Optional `additional_arguments`: extra argv strings appended after --args.
-Returns the launched app's pid, bundle_id, name, and a `windows` array (same shape as `list_windows`) so callers can skip an extra round-trip before `get_window_state(pid, window_id)`. When the focus-steal belt-and-braces demotion check ran (target pid ≠ prior frontmost), the response also includes `self_activation_suppressed: bool` — true if focus stayed with the prior frontmost, false if the launched app held focus despite the re-demote attempt.
+Returns the launched app's pid, bundle_id, name, and a `windows` array (same shape as `list_windows`) so callers can skip an extra round-trip before `get_window_state(pid, window_id)`. `launch_state` distinguishes whether the request was sent, the process is running, and a window is ready. When the focus-steal belt-and-braces demotion check ran (target pid ≠ prior frontmost), the response also includes `self_activation_suppressed: bool` — true if focus stayed with the prior frontmost, false if the launched app held focus despite the re-demote attempt.
**Arguments:**
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/apps/mod.rs b/libs/cua-driver/rust/crates/platform-macos/src/apps/mod.rs
index 110295617f..09324dbbdd 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/apps/mod.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/apps/mod.rs
@@ -2,6 +2,7 @@
pub mod nsworkspace;
+use anyhow::Context;
use serde::{Deserialize, Serialize};
use std::process::Command;
@@ -98,7 +99,7 @@ pub fn launch_app(bundle_id: &str) -> anyhow::Result<i32> {
..Default::default()
};
let running = nsworkspace::open_application(bundle_id, &cfg)
- .map_err(|e| anyhow::anyhow!("Failed to launch {bundle_id}: {e}"))?;
+ .with_context(|| format!("Failed to launch {bundle_id}"))?;
let pid: i32 = unsafe { running.processIdentifier() };
Ok(pid)
}
@@ -118,7 +119,7 @@ pub fn launch_app_by_name(name: &str) -> anyhow::Result<i32> {
..Default::default()
};
let running = nsworkspace::open_application(&app_ref, &cfg)
- .map_err(|e| anyhow::anyhow!("Failed to launch '{name}': {e}"))?;
+ .with_context(|| format!("Failed to launch '{name}'"))?;
let pid: i32 = unsafe { running.processIdentifier() };
Ok(pid)
}
@@ -159,7 +160,7 @@ pub fn launch_with_urls_by_bundle(
} else {
nsworkspace::open_urls_with_application(urls, bundle_id, &cfg)
}
- .map_err(|e| anyhow::anyhow!("Failed to launch {bundle_id}: {e}"))?;
+ .with_context(|| format!("Failed to launch {bundle_id}"))?;
let pid: i32 = unsafe { running.processIdentifier() };
Ok(pid)
}
@@ -190,7 +191,7 @@ pub fn launch_with_urls_by_name(
} else {
nsworkspace::open_urls_with_application(urls, &app_ref, &cfg)
}
- .map_err(|e| anyhow::anyhow!("Failed to launch '{name}': {e}"))?;
+ .with_context(|| format!("Failed to launch '{name}'"))?;
let pid: i32 = unsafe { running.processIdentifier() };
Ok(pid)
}
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/apps/nsworkspace.rs b/libs/cua-driver/rust/crates/platform-macos/src/apps/nsworkspace.rs
index 5dfeab60f7..7893a01322 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/apps/nsworkspace.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/apps/nsworkspace.rs
@@ -27,9 +27,10 @@
//! selector which `objc2-foundation 0.2.2` does not bind natively — we
//! hand-roll the binding in [`apple_event`] via `extern_methods!`.
+use std::collections::HashSet;
use std::ptr::NonNull;
use std::sync::Arc;
-use std::time::Duration;
+use std::time::{Duration, Instant};
use block2::RcBlock;
use objc2::rc::Retained;
@@ -72,6 +73,7 @@ pub struct OpenConfig {
/// `open(...)` call returns `Err(LaunchError::Timeout)` instead of hanging
/// the calling thread forever.
const COMPLETION_TIMEOUT: Duration = Duration::from_secs(30);
+const COMPLETION_POLL_INTERVAL: Duration = Duration::from_millis(50);
/// Errors returned by the NSWorkspace launch helpers.
#[derive(Debug, thiserror::Error)]
@@ -80,7 +82,10 @@ pub enum LaunchError {
Cocoa(String),
#[error("NSWorkspace launch returned no NSRunningApplication and no NSError")]
NoApp,
- #[error("NSWorkspace launch did not complete within {:?}", COMPLETION_TIMEOUT)]
+ #[error(
+ "NSWorkspace launch callback did not complete within {:?}, and no matching running application was observed",
+ COMPLETION_TIMEOUT
+ )]
Timeout,
#[error("invalid url: {0}")]
BadUrl(String),
@@ -112,6 +117,7 @@ pub fn open_application(
let ws = unsafe { NSWorkspace::sharedWorkspace() };
let url = resolve_application_url(&ws, app_url)?;
let config = build_configuration(cfg);
+ let reconciliation = Reconciliation::new(cfg);
let (tx, rx) = std::sync::mpsc::sync_channel::<CompletionResult>(1);
// NSWorkspace may invoke its completion on another thread; retain atomic
@@ -128,7 +134,7 @@ pub fn open_application(
ws.openApplicationAtURL_configuration_completionHandler(&url, &config, Some(&block));
}
- wait_for_completion(rx)
+ wait_for_completion(rx, reconciliation)
}
/// Launch the application bundle at `app_url` and hand it `urls` via
@@ -147,6 +153,7 @@ pub fn open_urls_with_application(
let ws = unsafe { NSWorkspace::sharedWorkspace() };
let url = resolve_application_url(&ws, app_url)?;
let config = build_configuration(cfg);
+ let reconciliation = Reconciliation::new(cfg);
let ns_urls: Vec<Retained<NSURL>> = urls
.iter()
@@ -174,7 +181,7 @@ pub fn open_urls_with_application(
);
}
- wait_for_completion(rx)
+ wait_for_completion(rx, reconciliation)
}
// ── Internal helpers ─────────────────────────────────────────────────────────
@@ -287,6 +294,75 @@ fn file_or_app_url(s: &str) -> Result<Retained<NSURL>, LaunchError> {
type CompletionResult = Result<Retained<NSRunningApplication>, LaunchError>;
+/// Actual-process fallback for daemons where LaunchServices accepts the
+/// request but never delivers the completion block. A bundle id is required
+/// because it is the only stable identity shared by the request and
+/// `NSRunningApplication`.
+struct Reconciliation {
+ bundle_id: Option<String>,
+ pids_before_request: HashSet<i32>,
+ require_new_pid: bool,
+}
+
+impl Reconciliation {
+ fn new(cfg: &OpenConfig) -> Self {
+ let bundle_id = cfg.apple_event_bundle_id.clone();
+ let pids_before_request = bundle_id
+ .as_deref()
+ .map(running_pids_for_bundle)
+ .unwrap_or_default();
+ Self {
+ bundle_id,
+ pids_before_request,
+ require_new_pid: cfg.creates_new_instance,
+ }
+ }
+
+ fn running_application(&self) -> Option<Retained<NSRunningApplication>> {
+ let bundle_id = self.bundle_id.as_deref()?;
+ running_application_for_bundle(bundle_id, &self.pids_before_request, self.require_new_pid)
+ }
+}
+
+fn running_pids_for_bundle(bundle_id: &str) -> HashSet<i32> {
+ let bundle_id = NSString::from_str(bundle_id);
+ let running =
+ unsafe { NSRunningApplication::runningApplicationsWithBundleIdentifier(&bundle_id) };
+ let mut pids = HashSet::new();
+ unsafe {
+ for index in 0..running.count() {
+ let app = running.objectAtIndex(index);
+ let pid = app.processIdentifier();
+ if pid > 0 && !app.isTerminated() {
+ pids.insert(pid);
+ }
+ }
+ }
+ pids
+}
+
+fn running_application_for_bundle(
+ bundle_id: &str,
+ pids_before_request: &HashSet<i32>,
+ require_new_pid: bool,
+) -> Option<Retained<NSRunningApplication>> {
+ let current_pids = running_pids_for_bundle(bundle_id);
+ reconciliation_pid(&current_pids, pids_before_request, require_new_pid).and_then(|pid| unsafe {
+ NSRunningApplication::runningApplicationWithProcessIdentifier(pid)
+ })
+}
+
+fn reconciliation_pid(
+ current_pids: &HashSet<i32>,
+ pids_before_request: &HashSet<i32>,
+ require_new_pid: bool,
+) -> Option<i32> {
+ current_pids
+ .iter()
+ .copied()
+ .find(|pid| !require_new_pid || !pids_before_request.contains(pid))
+}
+
/// Build the `(NSRunningApplication?, NSError?) -> Void` completion block.
///
/// Sends exactly one result through `tx`. Late completions (after timeout)
@@ -321,15 +397,135 @@ fn make_completion_block(
)
}
-/// Block on `rx` for up to `COMPLETION_TIMEOUT`. Returns `Err(Timeout)` if
-/// the completion handler never fires (wedged LaunchServices).
+/// Wait for either the completion callback or actual process registration.
+///
+/// LaunchServices can accept an `openApplication...` request and register an
+/// `NSRunningApplication` without delivering the callback to a background
+/// daemon. Treating the callback as the sole success signal turns that state
+/// into a false 30-second timeout. Poll the authoritative running-app state
+/// while retaining the callback as the preferred source of Cocoa errors.
fn wait_for_completion(
rx: std::sync::mpsc::Receiver<CompletionResult>,
+ reconciliation: Reconciliation,
) -> Result<Retained<NSRunningApplication>, LaunchError> {
- match rx.recv_timeout(COMPLETION_TIMEOUT) {
- Ok(r) => r,
- Err(std::sync::mpsc::RecvTimeoutError::Timeout) => Err(LaunchError::Timeout),
- Err(std::sync::mpsc::RecvTimeoutError::Disconnected) => Err(LaunchError::NoApp),
+ wait_for_launch_signal(rx, COMPLETION_TIMEOUT, COMPLETION_POLL_INTERVAL, || {
+ reconciliation.running_application()
+ })
+}
+
+fn wait_for_launch_signal<T>(
+ rx: std::sync::mpsc::Receiver<Result<T, LaunchError>>,
+ timeout: Duration,
+ poll_interval: Duration,
+ mut reconcile: impl FnMut() -> Option<T>,
+) -> Result<T, LaunchError> {
+ let deadline = Instant::now() + timeout;
+ let mut callback_disconnected = false;
+
+ loop {
+ if !callback_disconnected {
+ match rx.try_recv() {
+ Ok(result) => return result,
+ Err(std::sync::mpsc::TryRecvError::Empty) => {}
+ Err(std::sync::mpsc::TryRecvError::Disconnected) => {
+ callback_disconnected = true;
+ }
+ }
+ }
+
+ if let Some(running) = reconcile() {
+ return Ok(running);
+ }
+
+ let now = Instant::now();
+ if now >= deadline {
+ return Err(LaunchError::Timeout);
+ }
+ let wait = poll_interval.min(deadline.saturating_duration_since(now));
+
+ if callback_disconnected {
+ std::thread::sleep(wait);
+ continue;
+ }
+
+ match rx.recv_timeout(wait) {
+ Ok(result) => return result,
+ Err(std::sync::mpsc::RecvTimeoutError::Timeout) => {}
+ Err(std::sync::mpsc::RecvTimeoutError::Disconnected) => {
+ callback_disconnected = true;
+ }
+ }
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::{reconciliation_pid, wait_for_launch_signal, LaunchError};
+ use std::collections::HashSet;
+ use std::sync::mpsc;
+ use std::time::Duration;
+
+ #[test]
+ fn missing_callback_reconciles_registered_process() {
+ let (_tx, rx) = mpsc::sync_channel(1);
+ let mut polls = 0;
+ let result = wait_for_launch_signal(
+ rx,
+ Duration::from_millis(100),
+ Duration::from_millis(1),
+ || {
+ polls += 1;
+ (polls == 3).then_some(4242)
+ },
+ );
+
+ assert_eq!(result.unwrap(), 4242);
+ }
+
+ #[test]
+ fn queued_callback_error_wins_over_process_reconciliation() {
+ let (tx, rx) = mpsc::sync_channel(1);
+ tx.send(Err(LaunchError::Cocoa("denied".to_owned())))
+ .unwrap();
+
+ let result = wait_for_launch_signal(
+ rx,
+ Duration::from_millis(100),
+ Duration::from_millis(1),
+ || Some(4242),
+ );
+
+ assert!(matches!(result, Err(LaunchError::Cocoa(message)) if message == "denied"));
+ }
+
+ #[test]
+ fn no_callback_or_process_returns_precise_bounded_timeout() {
+ let (_tx, rx) = mpsc::sync_channel::<Result<i32, LaunchError>>(1);
+ let result = wait_for_launch_signal(
+ rx,
+ Duration::from_millis(5),
+ Duration::from_millis(1),
+ || None,
+ );
+
+ assert!(matches!(result, Err(LaunchError::Timeout)));
+ }
+
+ #[test]
+ fn new_instance_reconciliation_rejects_preexisting_pid() {
+ let before = HashSet::from([100]);
+ assert_eq!(
+ reconciliation_pid(&HashSet::from([100]), &before, true),
+ None
+ );
+ assert_eq!(
+ reconciliation_pid(&HashSet::from([100, 200]), &before, true),
+ Some(200)
+ );
+ assert_eq!(
+ reconciliation_pid(&HashSet::from([100]), &before, false),
+ Some(100)
+ );
}
}
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/launch_app.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/launch_app.rs
index 5eefcebde1..0d50c35a86 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/launch_app.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/launch_app.rs
@@ -33,7 +33,9 @@ fn def() -> &'static ToolDef {
Optional `additional_arguments`: extra argv strings appended after --args.\n\n\
Returns the launched app's pid, bundle_id, name, and a `windows` array \
(same shape as `list_windows`) so callers can skip an extra round-trip before \
- `get_window_state(pid, window_id)`. When the focus-steal belt-and-braces \
+ `get_window_state(pid, window_id)`. `launch_state` distinguishes whether the \
+ request was sent, the process is running, and a window is ready. When the \
+ focus-steal belt-and-braces \
demotion check ran (target pid ≠ prior frontmost), the response also includes \
`self_activation_suppressed: bool` — true if focus stayed with the prior \
frontmost, false if the launched app held focus despite the re-demote attempt."
@@ -87,6 +89,8 @@ impl Tool for LaunchAppTool {
use cua_driver_core::tool_args::ArgsExt;
let bundle_id = args.opt_str("bundle_id");
let name = args.opt_str("name");
+ let mut response_bundle_id = bundle_id.clone();
+ let response_requested_name = name.clone();
let urls: Vec<String> = args.str_array("urls");
if args.get("cdp_debugging_port").is_some() {
return ToolResult::error(
@@ -136,6 +140,7 @@ impl Tool for LaunchAppTool {
);
};
let (_, resolved_bundle_id) = locator.app_ref_and_bundle_id();
+ response_bundle_id = resolved_bundle_id.clone();
if resolved_bundle_id
.as_deref()
.is_some_and(is_cua_driver_bundle_id)
@@ -426,11 +431,11 @@ impl Tool for LaunchAppTool {
match launch_result {
Ok(Ok((pid, app_info, windows))) => {
- let app_name = app_info.as_ref().map(|a| a.name.as_str()).unwrap_or("?");
- let bid = app_info
- .as_ref()
- .and_then(|a| a.bundle_id.as_deref())
- .unwrap_or("?");
+ let (app_name, bid) = response_identity(
+ app_info.as_ref(),
+ response_bundle_id.as_deref(),
+ response_requested_name.as_deref(),
+ );
let mut summary =
format!("Launched {app_name} (pid {pid}) in background.{port_summary}");
@@ -472,6 +477,7 @@ impl Tool for LaunchAppTool {
"bundle_id": bid,
"name": app_name,
"windows": windows_json,
+ "launch_state": launch_state(true, true, !windows.is_empty()),
});
// Only emit `self_activation_suppressed` when the
// belt-and-braces demotion check actually ran. `None`
@@ -483,7 +489,7 @@ impl Tool for LaunchAppTool {
}
ToolResult::text(summary).with_structured(structured)
}
- Ok(Err(e)) => ToolResult::error(format!("Launch failed: {e}")),
+ Ok(Err(e)) => structured_launch_failure(&e),
Err(e) => ToolResult::error(format!("Task error: {e}")),
}
}
@@ -548,6 +554,77 @@ fn structured_launch_error(code: &str, message: String, details: serde_json::Val
ToolResult::error(message).with_structured(payload)
}
+fn launch_state(requested: bool, process_running: bool, window_ready: bool) -> serde_json::Value {
+ serde_json::json!({
+ "requested": requested,
+ "process_running": process_running,
+ "window_ready": window_ready,
+ })
+}
+
+fn response_identity(
+ app_info: Option<&crate::apps::AppInfo>,
+ requested_bundle_id: Option<&str>,
+ requested_name: Option<&str>,
+) -> (String, String) {
+ let bundle_id = app_info
+ .and_then(|app| app.bundle_id.as_deref())
+ .filter(|value| !value.is_empty())
+ .or(requested_bundle_id)
+ .unwrap_or("?")
+ .to_owned();
+
+ let name = app_info
+ .map(|app| app.name.as_str())
+ .filter(|value| !value.is_empty())
+ .map(str::to_owned)
+ .unwrap_or_else(|| requested_app_name(requested_name, requested_bundle_id));
+
+ (name, bundle_id)
+}
+
+fn requested_app_name(requested_name: Option<&str>, requested_bundle_id: Option<&str>) -> String {
+ if let Some(name) = requested_name.filter(|name| Some(*name) != requested_bundle_id) {
+ let file_name = std::path::Path::new(name)
+ .file_name()
+ .and_then(|value| value.to_str())
+ .unwrap_or(name);
+ return file_name
+ .strip_suffix(".app")
+ .unwrap_or(file_name)
+ .to_owned();
+ }
+
+ requested_bundle_id
+ .and_then(|bundle_id| bundle_id.rsplit('.').next())
+ .filter(|name| !name.is_empty())
+ .unwrap_or("?")
+ .to_owned()
+}
+
+fn structured_launch_failure(error: &anyhow::Error) -> ToolResult {
+ use crate::apps::nsworkspace::LaunchError;
+
+ let (code, requested) = if let Some(launch_error) = error.downcast_ref::<LaunchError>() {
+ match launch_error {
+ LaunchError::Cocoa(_) => ("NSWORKSPACE_LAUNCH_FAILED", true),
+ LaunchError::NoApp => ("LAUNCH_RESULT_MISSING", true),
+ LaunchError::Timeout => ("LAUNCH_CALLBACK_TIMEOUT", true),
+ LaunchError::BadUrl(_) => ("APP_URL_INVALID", false),
+ }
+ } else {
+ ("LAUNCH_FAILED", false)
+ };
+
+ structured_launch_error(
+ code,
+ format!("Launch failed: {error:#}"),
+ serde_json::json!({
+ "launch_state": launch_state(requested, false, false),
+ }),
+ )
+}
+
fn preflight_file_urls(urls: &[String]) -> Option<ToolResult> {
for raw in urls {
let Some(path) = local_file_target(raw) else {
@@ -633,7 +710,7 @@ fn hex_value(byte: u8) -> Option<u8> {
mod tests {
use super::{
contains_remote_debugging_flag, is_cua_driver_bundle_id, local_file_target,
- preflight_file_urls, LaunchAppTool,
+ preflight_file_urls, response_identity, structured_launch_failure, LaunchAppTool,
};
use cua_driver_core::tool::Tool;
use serde_json::json;
@@ -700,6 +777,51 @@ mod tests {
assert!(!is_cua_driver_bundle_id("com.trycua.harness.tauri"));
}
+ #[test]
+ fn launch_timeout_reports_requested_without_process_or_window() {
+ let error = anyhow::Error::new(crate::apps::nsworkspace::LaunchError::Timeout)
+ .context("Failed to launch com.example.App");
+ let result = structured_launch_failure(&error);
+ let structured = result.structured_content.expect("structured error");
+
+ assert_eq!(result.is_error, Some(true));
+ assert_eq!(structured["error"], "LAUNCH_CALLBACK_TIMEOUT");
+ assert_eq!(structured["launch_state"]["requested"], true);
+ assert_eq!(structured["launch_state"]["process_running"], false);
+ assert_eq!(structured["launch_state"]["window_ready"], false);
+ }
+
+ #[test]
+ fn invalid_url_reports_request_was_not_sent() {
+ let error = anyhow::Error::new(crate::apps::nsworkspace::LaunchError::BadUrl(
+ "bad url".to_owned(),
+ ))
+ .context("Failed to launch com.example.App");
+ let result = structured_launch_failure(&error);
+ let structured = result.structured_content.expect("structured error");
+
+ assert_eq!(structured["error"], "APP_URL_INVALID");
+ assert_eq!(structured["launch_state"]["requested"], false);
+ assert_eq!(structured["launch_state"]["process_running"], false);
+ assert_eq!(structured["launch_state"]["window_ready"], false);
+ }
+
+ #[test]
+ fn process_only_response_falls_back_to_requested_identity() {
+ assert_eq!(
+ response_identity(None, Some("com.apple.Safari"), None),
+ ("Safari".to_owned(), "com.apple.Safari".to_owned())
+ );
+ assert_eq!(
+ response_identity(
+ None,
+ Some("com.example.Editor"),
+ Some("/Applications/Example Editor.app"),
+ ),
+ ("Example Editor".to_owned(), "com.example.Editor".to_owned())
+ );
+ }
+
#[tokio::test]
async fn launch_app_cannot_reach_private_permission_host_entrypoint() {
let result = LaunchAppTool