Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
feat(cua-driver): optional brief foreground step for native drag-and-drop on macOS (#5481)
Francesco Bonacci5033e7ed3fd2
diff --git a/docs/content/docs/cua-driver/guides/operate.mdx b/docs/content/docs/cua-driver/guides/operate.mdxindex 644bfeb0f4..4cbe94d32a 100644--- a/docs/content/docs/cua-driver/guides/operate.mdx+++ b/docs/content/docs/cua-driver/guides/operate.mdx@@ -149,6 +149,22 @@ A bad configuration fails startup instead of falling back to `standard`. Auser policy file works the same way: set `CUA_DRIVER_POLICY_FILE` in theplist's `EnvironmentVariables`, the unit, or the User environment.+### Allow a brief foreground step (macOS)++Some actions can't finish in the background. Today that means a drag that+starts a macOS drag-and-drop session in a native window, such as reordering a+list, because macOS follows the real pointer for those. By default the driver+refuses them with `background_unavailable`.++To let the driver finish them, set `CUA_DRIVER_ALLOW_BRIEF_FOREGROUND=1` in+the daemon's environment (the plist's `EnvironmentVariables`). Then only that+one step runs in the foreground: the window comes forward and the pointer moves+for the drag, and afterwards the previous frontmost app and the pointer go+back. The result carries a `foreground_step` object that says whether both were+restored, and the daemon log records the step. A call that passes+`delivery_mode: "background"` explicitly is never escalated. Leave the+variable unset on a Mac someone is using.+## Update```bash test="cli-shape" id="cua-driver-check-update"diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/delivery.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/delivery.rsindex 3046c0083c..a040bc3bb1 100644--- a/libs/cua-driver/rust/crates/cua-driver-core/src/delivery.rs+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/delivery.rs@@ -48,6 +48,40 @@ impl DeliveryMode {}}+/// Environment variable an operator sets when starting the daemon to allow a+/// brief foreground step (trycua/cua#5478). Off by default.+pub const BRIEF_FOREGROUND_ENV: &str = "CUA_DRIVER_ALLOW_BRIEF_FOREGROUND";++/// Whether the operator allowed a brief foreground step for actions that+/// cannot finish in the background.+///+/// When it is on, a tool whose background attempt hits a known platform limit+/// (today: a macOS drag-and-drop session in a native window) repeats only that+/// step in the foreground, then hands the frontmost app and the pointer back,+/// and says so in its result. It is read from the daemon's environment, which+/// the agent can't change, and a call that passes `delivery_mode:"background"`+/// explicitly is never escalated.+pub fn brief_foreground_allowed() -> bool {+ brief_foreground_flag(std::env::var(BRIEF_FOREGROUND_ENV).ok().as_deref())+}++/// Parse [`BRIEF_FOREGROUND_ENV`]: only `1`, `true`, `yes` or `on`+/// (case-insensitive) turn it on.+pub fn brief_foreground_flag(value: Option<&str>) -> bool {+ matches!(+ value.map(|v| v.trim().to_ascii_lowercase()).as_deref(),+ Some("1" | "true" | "yes" | "on")+ )+}++/// Whether the call itself asked for background delivery by name, which opts+/// it out of the brief foreground step.+pub fn background_requested_explicitly(args: &Value) -> bool {+ args.get("delivery_mode")+ .and_then(Value::as_str)+ .is_some_and(|mode| mode.eq_ignore_ascii_case("background"))+}+/// The `suggestion` every background refusal carries.pub const FOREGROUND_RETRY_SUGGESTION: &str ="Retry this action with delivery_mode:\"foreground\".";@@ -103,6 +137,35 @@ pub fn background_unavailable_result_with_suggestion(#[cfg(test)]mod tests {+ #[test]+ fn brief_foreground_is_off_unless_explicitly_enabled() {+ for off in [+ None,+ Some(""),+ Some("0"),+ Some("false"),+ Some("no"),+ Some("maybe"),+ ] {+ assert!(!super::brief_foreground_flag(off), "{off:?}");+ }+ for on in ["1", "true", "TRUE", " yes ", "on"] {+ assert!(super::brief_foreground_flag(Some(on)), "{on:?}");+ }+ }++ #[test]+ fn explicit_background_opts_out_of_the_brief_foreground_step() {+ use serde_json::json;+ assert!(super::background_requested_explicitly(+ &json!({"delivery_mode": "Background"})+ ));+ assert!(!super::background_requested_explicitly(&json!({})));+ assert!(!super::background_requested_explicitly(+ &json!({"delivery_mode": "foreground"})+ ));+ }+use super::*;#[test]diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/drag.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/drag.rsindex 54461d8cda..8e8d8a2a7e 100644--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/drag.rs+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/drag.rs@@ -450,7 +450,12 @@ impl Tool for DragTool {button,true,observe,- )+ )?;+ // Hold the front until AppKit finishes the+ // drop: restoring the previous app while the+ // session is still open cancels it.+ await_drag_session_end(pid);+ Ok(())},).map(|()| false);@@ -542,6 +547,13 @@ impl Tool for DragTool {)};match result {+ Ok(Ok(true))+ if cua_driver_core::delivery::brief_foreground_allowed()+ && !cua_driver_core::delivery::background_requested_explicitly(&args) =>+ {+ self.brief_foreground_drag(&args, pid, wid, prior_front)+ .await+ }Ok(Ok(true)) => drag_and_drop_session_result(pid, wid, changes.result_suffix()),Ok(Ok(false)) => ToolResult::text(format!("✅ Posted drag{btn_suffix}{mod_suffix} to pid {pid} window {wid} \@@ -573,6 +585,76 @@ impl Tool for DragTool {}impl DragTool {+ /// The operator allowed a brief foreground step (trycua/cua#5478) and the+ /// background drag started a native drag-and-drop session, which macOS+ /// tracks with the real pointer. Repeat only this drag in the foreground,+ /// which fronts the window, drives the pointer, then hands the frontmost+ /// app and the pointer back. Check both and report the step in the result.+ async fn brief_foreground_drag(+ &self,+ args: &Value,+ pid: i32,+ wid: u32,+ prior_front: Option<i32>,+ ) -> ToolResult {+ let pointer_before = tokio::task::spawn_blocking(pointer_location)+ .await+ .ok()+ .flatten();+ tracing::info!(+ pid,+ window_id = wid,+ ?prior_front,+ "drag: background drop impossible (drag-and-drop session); \+ operator allowed a brief foreground step"+ );+ let mut foreground_args = args.clone();+ foreground_args["delivery_mode"] = serde_json::json!("foreground");+ let mut result = self.invoke(foreground_args).await;++ // The foreground drag leaves the pointer on the drop point (after the+ // drop has finished, see `await_drag_session_end`); this step promised+ // to hand it back, so put it where the user left it.+ if let Some((x, y)) = pointer_before {+ let _ =+ tokio::task::spawn_blocking(move || crate::input::mouse::move_cursor_desktop(x, y))+ .await;+ }+ let pointer_after = tokio::task::spawn_blocking(pointer_location)+ .await+ .ok()+ .flatten();+ let front_after = apps::frontmost_pid();+ let restored_frontmost = prior_front.is_none() || front_after == prior_front;+ let restored_pointer = match (pointer_before, pointer_after) {+ (Some((bx, by)), Some((ax, ay))) => Some((bx - ax).hypot(by - ay) <= 1.0),+ _ => None,+ };+ tracing::info!(+ pid,+ window_id = wid,+ restored_frontmost,+ ?restored_pointer,+ failed = result.is_error == Some(true),+ "drag: brief foreground step finished"+ );++ let step = brief_foreground_step(restored_frontmost, restored_pointer);+ let note = brief_foreground_note(restored_frontmost, restored_pointer);+ let mut structured = result+ .structured_content+ .take()+ .unwrap_or_else(|| serde_json::json!({}));+ if let Some(fields) = structured.as_object_mut() {+ fields.insert("foreground_step".to_owned(), step);+ }+ result.structured_content = Some(structured);+ result+ .content+ .insert(0, cua_driver_core::protocol::Content::text(note));+ result+ }+/// Background drag in an Electron window through its main process/// ([`crate::input::electron_drag`]). Refuses when the app's inspector/// is unavailable (for example the inspector fuse is off).@@ -688,6 +770,58 @@ impl DragTool {}}+/// Wait (up to 2 s) until `pid` shows no drag image, so a drag-and-drop+/// session started by a foreground drag has finished dropping, then let AppKit+/// settle briefly. A drag that started no session returns almost at once.+fn await_drag_session_end(pid: i32) {+ let deadline = std::time::Instant::now() + std::time::Duration::from_secs(2);+ while !drag_image_windows(pid).is_empty() && std::time::Instant::now() < deadline {+ std::thread::sleep(std::time::Duration::from_millis(50));+ }+ std::thread::sleep(std::time::Duration::from_millis(150));+}++/// Current hardware pointer location in global display points.+fn pointer_location() -> Option<(f64, f64)> {+ use core_graphics::event::CGEvent;+ use core_graphics::event_source::{CGEventSource, CGEventSourceStateID};+ let source = CGEventSource::new(CGEventSourceStateID::HIDSystemState).ok()?;+ let location = CGEvent::new(source).ok()?.location();+ Some((location.x, location.y))+}++/// Structured record of a brief foreground step, for the agent and the log.+fn brief_foreground_step(restored_frontmost: bool, restored_pointer: Option<bool>) -> Value {+ serde_json::json!({+ "used": true,+ "reason": "drag_and_drop_session",+ "allowed_by": cua_driver_core::delivery::BRIEF_FOREGROUND_ENV,+ "background_attempt": "no_drop",+ "restored_frontmost": restored_frontmost,+ "restored_pointer": restored_pointer,+ })+}++fn brief_foreground_note(restored_frontmost: bool, restored_pointer: Option<bool>) -> String {+ let mut note = String::from(+ "Brief foreground step: the background drag started a macOS drag-and-drop session, \+ which follows the real pointer, so the driver repeated this one drag in the \+ foreground because the operator allowed it. The window came forward and the \+ pointer moved for the drag only.",+ );+ note.push_str(if restored_frontmost {+ " The previous frontmost app is back in front."+ } else {+ " The previous frontmost app did NOT come back to the front; tell the user."+ });+ note.push_str(match restored_pointer {+ Some(true) => " The pointer is back where it was.",+ Some(false) => " The pointer did NOT return to where it was; tell the user.",+ None => " The pointer position could not be checked.",+ });+ note+}+/// `kCGDraggingWindowLevel`: AppKit shows the image of a live (or sliding/// back) drag-and-drop session in a window of the source app at this level.const DRAGGING_WINDOW_LEVEL: i32 = 500;@@ -767,6 +901,27 @@ mod tests {}}+ #[test]+ fn brief_foreground_step_reports_what_was_restored() {+ let step = super::brief_foreground_step(true, Some(true));+ assert_eq!(step["used"], true);+ assert_eq!(step["reason"], "drag_and_drop_session");+ assert_eq!(step["allowed_by"], "CUA_DRIVER_ALLOW_BRIEF_FOREGROUND");+ assert_eq!(step["restored_frontmost"], true);+ assert_eq!(step["restored_pointer"], true);+ let unknown = super::brief_foreground_step(false, None);+ assert_eq!(unknown["restored_frontmost"], false);+ assert!(unknown["restored_pointer"].is_null());++ let note = super::brief_foreground_note(true, Some(true));+ assert!(note.contains("Brief foreground step"));+ assert!(note.contains("frontmost app is back"));+ assert!(note.contains("pointer is back"));+ let failed = super::brief_foreground_note(false, Some(false));+ assert!(failed.contains("did NOT come back"));+ assert!(failed.contains("did NOT return"));+ }+#[test]fn schema_says_macos_drag_runs_in_the_background() {let def = def();