Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
fix(cua-spacesd): restore Windows viewer input through driver tools (#4528)
zae 𓂃˚₊⊹6b95048b048d
diff --git a/libs/cua-spacesd/crates/cua-spacesd-desktop/src/lib.rs b/libs/cua-spacesd/crates/cua-spacesd-desktop/src/lib.rsindex eaffa16065..8606f9585d 100644--- a/libs/cua-spacesd/crates/cua-spacesd-desktop/src/lib.rs+++ b/libs/cua-spacesd/crates/cua-spacesd-desktop/src/lib.rs@@ -328,14 +328,30 @@ fn hyprland_background_admission(native: NativeTarget) -> Result<(), ProviderErr/// session ([`human_input_session`]): no agent cursor in the guest, and none/// in presence. An agent's runs in the provider's coordinate session and/// keeps the agent cursor.-#[cfg(target_os = "linux")]+#[cfg(any(target_os = "linux", target_os = "windows"))]struct RegistryGestureTools {inner: Arc<CuaProviderInner>,/// The human-input session, `None` for an agent.human_session: Option<String>,}-#[cfg(target_os = "linux")]+#[cfg(any(target_os = "linux", target_os = "windows", test))]+fn gesture_tool_error(tool: &str, result: &ToolResult) -> ProviderError {+ let code = match result+ .structured_content+ .as_ref()+ .and_then(|value| value.get("code"))+ .and_then(serde_json::Value::as_str)+ {+ Some("background_unavailable" | "would_require_activation") => {+ ProviderErrorCode::WouldRequireActivation+ }+ _ => ProviderErrorCode::DeliveryFailed,+ };+ ProviderError::new(code, format!("{tool}: {}", tool_result_text(result)))+}++#[cfg(any(target_os = "linux", target_os = "windows"))]impl tool_input::GestureTools for RegistryGestureTools {fn invoke(&self,@@ -373,10 +389,7 @@ impl tool_input::GestureTools for RegistryGestureTools {}).map_err(|error| ProviderError::new(ProviderErrorCode::Internal, error))??;if result.is_error == Some(true) {- return Err(ProviderError::new(- ProviderErrorCode::DeliveryFailed,- format!("{tool}: {}", tool_result_text(&result)),- ));+ return Err(gesture_tool_error(tool, &result));}Ok(())}@@ -1223,10 +1236,51 @@ impl InteractiveInputProvider for CuaInteractiveInputProvider {Ok(Some(Arc::new(driver_input::MacosLease(session))))}- #[cfg(not(any(target_os = "linux", target_os = "macos")))]+ #[cfg(target_os = "windows")]+ {+ // Windows already exposes qualified gesture tools. Adapt media+ // batches to those tools, as on Hyprland, instead of advertising+ // interactive input and then returning no input lease.+ let inner = self.0.clone();+ let lease = tool_input::open(+ target,+ delivery_mode,+ |target| {+ let native = inner.native(target)?;+ let geometry = inner.clone();+ Ok(tool_input::ToolInputTarget::Window {+ pid: native.pid,+ window_id: native.window_id,+ extent: Box::new(move || {+ geometry+ .native_pixel_geometry(native)+ .map(|pixels| (pixels.width_px, pixels.height_px))+ }),+ })+ },+ |display_id| {+ windows_display::displays()+ .into_iter()+ .find(|display| display.id == display_id)+ .ok_or_else(|| {+ ProviderError::new(+ ProviderErrorCode::TargetUnavailable,+ "display is gone",+ )+ })+ },+ // Desktop-scope keys address the current foreground app.+ Box::new(|| None),+ Arc::new(RegistryGestureTools {+ inner: self.0.clone(),+ human_session: (!owner.agent).then(|| human_input_session(&owner.id)),+ }),+ )?;+ Ok(Some(Arc::new(lease.with_desktop_hover())))+ }++ #[cfg(not(any(target_os = "linux", target_os = "macos", target_os = "windows")))]{- // cua-driver has no Windows interactive session yet; one-shot- // actions go through its tools (desktop scope for a display).let _ = delivery_mode;if display_key(target).is_none() {self.0.native(target)?;@@ -2991,6 +3045,26 @@ mod tests {));}+ #[test]+ fn gesture_tools_preserve_activation_required_errors() {+ let result = ToolResult::error("background keyboard input is unavailable")+ .with_structured(serde_json::json!({"code":"background_unavailable"}));+ let error = gesture_tool_error("press_key", &result);+ assert_eq!(error.code, ProviderErrorCode::WouldRequireActivation);+ assert_eq!(+ error.message,+ "press_key: background keyboard input is unavailable"+ );+ }++ #[test]+ fn gesture_tools_do_not_retry_security_refusals_by_activating() {+ let result = ToolResult::error("Windows blocked input to an elevated application")+ .with_structured(serde_json::json!({"code":"background_uipi_blocked"}));+ let error = gesture_tool_error("click", &result);+ assert_eq!(error.code, ProviderErrorCode::DeliveryFailed);+ }+#[test]fn adapter_newtypes_are_send_and_sync() {fn assert_send_sync<T: Send + Sync>() {}diff --git a/libs/cua-spacesd/crates/cua-spacesd-desktop/src/tool_input.rs b/libs/cua-spacesd/crates/cua-spacesd-desktop/src/tool_input.rsindex 1d5c325f0b..c4cc837527 100644--- a/libs/cua-spacesd/crates/cua-spacesd-desktop/src/tool_input.rs+++ b/libs/cua-spacesd/crates/cua-spacesd-desktop/src/tool_input.rs@@ -3,15 +3,12 @@//! Interactive input through the cua-driver tools.//!-//! On Hyprland cua-driver has no stateful interactive session: its input-//! goes through the driver tools (`click`, `drag`, `scroll`, `press_key`,-//! `type_text`), which route the desktop through the virtual pointer and-//! keyboard and background window input through the cua-hyprland-plugin-//! seats. This lease folds each interactive batch into whole gestures with-//! the shared [`GestureFolder`] and invokes one tool per gesture, so a-//! streaming viewer's clicks, drags, scrolls and keys reach a Hyprland-//! desktop or window exactly as the same actions from an agent would. Nothing-//! here injects input.+//! Windows and Hyprland interactive batches are folded into whole gestures+//! with the shared [`GestureFolder`] and passed to the existing cua-driver+//! tools (`click`, `drag`, `scroll`, `press_key`, `type_text`). Hyprland tools+//! use virtual pointer/keyboard input and plugin seats for background windows;+//! Windows tools use their qualified native delivery routes. This adapter+//! does not inject input itself.//!//! Addressing follows the one-shot action path: a window gets `pid`,//! `window_id` and window-local capture pixels with the session's delivery@@ -20,23 +17,25 @@//! gRPC backend's Linux keyboard rule; the desktop scope only when nothing is//! focused). A display has no window to address background input to, so a//! background session on one is refused with `WouldRequireActivation`, as on-//! X11 and macOS; so is a background session on a window the driver's-//! background route does not reach (the provider checks that when it-//! resolves the window).+//! X11 and macOS. A window gesture also preserves an activation-required+//! refusal from the driver when its background route cannot reach the target.//!-//! What a gesture tool cannot carry is not reported as delivered: hover moves-//! produce no tool call, and a key's down edge is its whole press.+//! Windows desktop leases also forward hover (a move while no button is held)+//! through `move_cursor`. Other hover produces no tool call. A key's down edge+//! is a whole press; clicks and drags are folded until release rather than+//! delivered as held edges.-// Used by the Linux (Hyprland) provider; compiled everywhere so its+// Used by Windows and Linux (Hyprland); compiled everywhere so its// behavior is tested on every host.-#![cfg_attr(not(target_os = "linux"), allow(dead_code))]+#![cfg_attr(not(any(target_os = "linux", target_os = "windows")), allow(dead_code))]use std::sync::{Arc, Mutex};use std::time::Instant;use cua_driver_core::interactive_input::gestures::{Gesture, GestureFolder};use cua_driver_core::interactive_input::{- validate_batch, InteractiveDeliveryMode, Modifier, PointerButton,+ validate_batch, InteractiveDeliveryMode, InteractiveInputEvent, Modifier, PointerButton,+ PointerPhase,};use cua_media_protocol::InteractiveInputBatch;use cua_spacesd_provider_api::{@@ -118,7 +117,8 @@ pub(crate) fn open(target,mode,tools,- folder: Mutex::new(GestureFolder::new()),+ fold: Mutex::new(Fold::default()),+ desktop_hover: false,})}@@ -126,7 +126,45 @@ pub(crate) struct ToolInputLease {target: ToolInputTarget,mode: InteractiveDeliveryMode,tools: Arc<dyn GestureTools>,- folder: Mutex<GestureFolder>,+ fold: Mutex<Fold>,+ desktop_hover: bool,+}++/// The gesture fold, and the button a viewer holds for desktop hover.+#[derive(Default)]+struct Fold {+ folder: GestureFolder,+ held: Option<PointerButton>,+}++impl Fold {+ /// The point of a hover move. Viewers send drag samples as button-less+ /// moves too, so a move is hover only while no button is held; the held+ /// button is tracked as [`GestureFolder`] tracks its press.+ fn hover(&mut self, event: &InteractiveInputEvent) -> Option<(f64, f64)> {+ let InteractiveInputEvent::Pointer {+ phase,+ button,+ x_normalized,+ y_normalized,+ ..+ } = event+ else {+ return None;+ };+ let pressed = button.unwrap_or(PointerButton::Left);+ match phase {+ PointerPhase::Move if button.is_none() && self.held.is_none() => {+ return Some((*x_normalized, *y_normalized));+ }+ PointerPhase::Move => {}+ PointerPhase::Down => self.held = Some(pressed),+ PointerPhase::Up if self.held == Some(pressed) => self.held = None,+ PointerPhase::Up => {}+ PointerPhase::Cancel => self.held = None,+ }+ None+ }}fn button_name(button: PointerButton) -> &'static str {@@ -189,6 +227,15 @@ fn desktop_arguments() -> Map<String, Value> {}impl ToolInputLease {+ /// The Windows driver can move the real desktop pointer. Window scope+ /// moves only its agent overlay, so never opt that route into hover.+ #[cfg_attr(not(target_os = "windows"), allow(dead_code))]+ pub(crate) fn with_desktop_hover(mut self) -> Self {+ self.desktop_hover = matches!(self.target, ToolInputTarget::Display { .. })+ && self.mode == InteractiveDeliveryMode::PersistentForeground;+ self+ }+fn extent(&self) -> Result<(u32, u32), ProviderError> {let extent = match &self.target {ToolInputTarget::Window { extent, .. } => extent(),@@ -337,16 +384,37 @@ impl InteractiveInputLease for ToolInputLease {let batch = driver_batch(batch)?;let through = validate_batch(&batch).map_err(interactive_error)?;let extent = self.extent()?;- let gestures = self- .folder+ let mut fold = self+ .fold.lock()- .unwrap_or_else(std::sync::PoisonError::into_inner)- .fold(&batch.events, extent);- for gesture in gestures {- for (tool, arguments) in self.call(gesture, extent) {- self.tools.invoke(tool, arguments)?;+ .unwrap_or_else(std::sync::PoisonError::into_inner);+ let mut calls = Vec::new();+ // The events between hover moves fold together, so consecutive+ // scrolls and text stay one tool call each.+ let mut run = 0;+ if self.desktop_hover {+ for (index, event) in batch.events.iter().enumerate() {+ let Some((x, y)) = fold.hover(event) else {+ continue;+ };+ for gesture in fold.folder.fold(&batch.events[run..index], extent) {+ calls.extend(self.call(gesture, extent));+ }+ let (x, y) = self.point(x, y, extent);+ let mut arguments = desktop_arguments();+ arguments.insert("x".into(), x.into());+ arguments.insert("y".into(), y.into());+ calls.push(("move_cursor", arguments));+ run = index + 1;}}+ for gesture in fold.folder.fold(&batch.events[run..], extent) {+ calls.extend(self.call(gesture, extent));+ }+ drop(fold);+ for (tool, arguments) in calls {+ self.tools.invoke(tool, arguments)?;+ }Ok(InteractiveInputOutcome {through_sequence: through,event_count: batch.events.len(),@@ -357,10 +425,10 @@ impl InteractiveInputLease for ToolInputLease {/// Gestures are delivered whole, so nothing is held down; forget a/// half-finished press.fn release_all(&self) {- self.folder+ *self+ .fold.lock()- .unwrap_or_else(std::sync::PoisonError::into_inner)- .reset();+ .unwrap_or_else(std::sync::PoisonError::into_inner) = Fold::default();}}@@ -431,6 +499,29 @@ mod tests {}}+ /// A button-less move: hover, or a drag sample while a button is held.+ fn moved(x: f64, y: f64) -> InteractiveInputEvent {+ InteractiveInputEvent::Pointer {+ phase: InputPointerPhase::Move,+ button: None,+ x_normalized: x,+ y_normalized: y,+ modifiers: Vec::new(),+ }+ }++ fn wheel(x: f64, y: f64, delta_y: f64) -> InteractiveInputEvent {+ InteractiveInputEvent::Scroll {+ x_normalized: x,+ y_normalized: y,+ delta_x: 0.0,+ delta_y,+ phase: InputGesturePhase::None,+ momentum_phase: InputGesturePhase::None,+ precise: false,+ }+ }+fn key(name: &str, state: InputKeyState) -> InteractiveInputEvent {InteractiveInputEvent::Key {key: name.into(),@@ -440,6 +531,19 @@ mod tests {}}+ fn hover_lease(tools: Arc<Recorded>) -> ToolInputLease {+ open(+ &display_target(),+ InteractiveDeliveryMode::PersistentForeground,+ unknown_window,+ |_| Ok(display()),+ Box::new(|| None),+ tools,+ )+ .unwrap()+ .with_desktop_hover()+ }+/// The provider's window catalog, which never enumerates a display.fn unknown_window(target: &ProviderTargetId) -> Result<ToolInputTarget, ProviderError> {let native = crate::TargetCatalog::default().native(target)?;@@ -519,6 +623,136 @@ mod tests {);}+ #[test]+ fn desktop_hover_preserves_gesture_order_and_scaled_monitor_origin() {+ let tools = Arc::new(Recorded::default());+ let mut monitor = display();+ monitor.scale_factor = 1.5;+ monitor.bounds = (-1600.0 / 1.5, -300.0 / 1.5, 1280.0 / 1.5, 800.0 / 1.5);+ let lease = open(+ &display_target(),+ InteractiveDeliveryMode::PersistentForeground,+ unknown_window,+ |_| Ok(monitor),+ Box::new(|| None),+ tools.clone(),+ )+ .unwrap()+ .with_desktop_hover();+ lease+ .dispatch(&batch(+ 5,+ vec![+ moved(0.25, 0.5),+ key("escape", InputKeyState::Down),+ pointer(InputPointerPhase::Down, 0.5, 0.5),+ pointer(InputPointerPhase::Up, 0.5, 0.5),+ moved(1.0, 1.0),+ ],+ ))+ .unwrap();+ assert_eq!(+ tools.0.lock().unwrap().clone(),+ vec![+ (+ "move_cursor".into(),+ json!({"scope":"desktop", "x":-1280.0, "y":100.0})+ ),+ (+ "press_key".into(),+ json!({"scope":"desktop", "key":"escape"})+ ),+ (+ "click".into(),+ json!({"scope":"desktop", "x":-960.0, "y":100.0, "button":"left"})+ ),+ (+ "move_cursor".into(),+ json!({"scope":"desktop", "x":-321.0, "y":499.0})+ ),+ ]+ );+ }++ /// Regression: viewers send drag samples as button-less moves. Moving+ /// the real pointer for them would sweep it across the desktop with no+ /// button held, then jump back when the folded drag replays on release.+ #[test]+ fn desktop_hover_ignores_drag_samples_across_batches() {+ let tools = Arc::new(Recorded::default());+ let lease = hover_lease(tools.clone());+ for (first_sequence, events) in [+ (+ 1,+ vec![+ moved(0.25, 0.25),+ pointer(InputPointerPhase::Down, 0.25, 0.25),+ ],+ ),+ (3, vec![moved(0.5, 0.5)]),+ (+ 4,+ vec![pointer(InputPointerPhase::Up, 0.75, 0.75), moved(1.0, 1.0)],+ ),+ ] {+ lease.dispatch(&batch(first_sequence, events)).unwrap();+ }+ assert_eq!(+ tools.0.lock().unwrap().clone(),+ vec![+ (+ "move_cursor".into(),+ json!({"scope":"desktop", "x":320.0, "y":200.0})+ ),+ (+ "drag".into(),+ json!({+ "scope":"desktop", "from_x":320.0, "from_y":200.0,+ "to_x":959.0, "to_y":599.0, "button":"left",+ })+ ),+ (+ "move_cursor".into(),+ json!({"scope":"desktop", "x":1279.0, "y":799.0})+ ),+ ]+ );+ }++ #[test]+ fn desktop_hover_keeps_consecutive_scrolls_one_tool_call() {+ let tools = Arc::new(Recorded::default());+ let lease = hover_lease(tools.clone());+ lease+ .dispatch(&batch(+ 1,+ vec![+ wheel(0.25, 0.25, 1.0),+ wheel(0.5, 0.5, 2.0),+ moved(0.75, 0.75),+ wheel(0.75, 0.75, 1.0),+ ],+ ))+ .unwrap();+ assert_eq!(+ tools.0.lock().unwrap().clone(),+ vec![+ (+ "scroll".into(),+ json!({"scope":"desktop", "x":640.0, "y":400.0, "direction":"down", "amount":3})+ ),+ (+ "move_cursor".into(),+ json!({"scope":"desktop", "x":959.0, "y":599.0})+ ),+ (+ "scroll".into(),+ json!({"scope":"desktop", "x":959.0, "y":599.0, "direction":"down", "amount":1})+ ),+ ]+ );+ }+#[test]fn desktop_keys_without_a_focused_window_use_the_desktop_scope() {let tools = Arc::new(Recorded::default());diff --git a/libs/cua-spacesd/crates/cua-spacesd-server/src/process/spawn.rs b/libs/cua-spacesd/crates/cua-spacesd-server/src/process/spawn.rsindex 8ed9dbfe24..4b90d2eaf9 100644--- a/libs/cua-spacesd/crates/cua-spacesd-server/src/process/spawn.rs+++ b/libs/cua-spacesd/crates/cua-spacesd-server/src/process/spawn.rs@@ -221,6 +221,12 @@ pub fn spawn_pipes(spec: &SpawnSpec) -> io::Result<Spawned> {command.process_group(0);install_user_switch(&mut command, spec.switch_user.clone(), false);}+ #[cfg(windows)]+ {+ // Pipes already carry all shell I/O. A background host must not+ // allocate a visible console for each ProcessService command.+ command.creation_flags(0x08000000); // CREATE_NO_WINDOW+ }let mut child = command.spawn()?;let pid = child.id().unwrap_or(0);let stdin = childdiff --git a/libs/cua-spacesd/crates/cua-spacesd-session/src/media/viewer.rs b/libs/cua-spacesd/crates/cua-spacesd-session/src/media/viewer.rsindex 05b995265a..0782197ede 100644--- a/libs/cua-spacesd/crates/cua-spacesd-session/src/media/viewer.rs+++ b/libs/cua-spacesd/crates/cua-spacesd-session/src/media/viewer.rs@@ -581,6 +581,8 @@ async fn run_input_job(let started = Instant::now();let events = batch.events.len() as u64;let nack = |code: ActionErrorCode, message: String| {+ tracing::warn!(target: "cua_spacesd_client::host_input", ?code,+ "viewer input dispatch rejected");ServerMessage::InteractiveInputAcknowledgement(InteractiveInputAcknowledgement {session_id: session_id.clone(),through_sequence: through,@@ -832,6 +834,10 @@ impl Viewer {async fn dispatch_input(&mut self, batch: InteractiveInputBatch) -> Option<ServerMessage> {let session_id = self.session_id();let nack = |through: u64, code: ActionErrorCode, message: String| {+ // Admission errors contain permission/lease state, never the+ // contents of a user's keyboard or pointer batch.+ tracing::warn!(target: "cua_spacesd_client::host_input", ?code,+ reason = %message, "viewer input admission rejected");ServerMessage::InteractiveInputAcknowledgement(InteractiveInputAcknowledgement {session_id: session_id.clone(),through_sequence: through,