Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

fix(cua-spacesd): restore Windows viewer input through driver tools (#4528)

zae 𓂃˚₊⊹6b95048b048d
diff --git a/libs/cua-spacesd/crates/cua-spacesd-desktop/src/lib.rs b/libs/cua-spacesd/crates/cua-spacesd-desktop/src/lib.rs
index eaffa16065..8606f9585d 100644
--- a/libs/cua-spacesd/crates/cua-spacesd-desktop/src/lib.rs
+++ b/libs/cua-spacesd/crates/cua-spacesd-desktop/src/lib.rs
@@ -328,14 +328,30 @@ fn hyprland_background_admission(native: NativeTarget) -> Result<(), ProviderErr
/// session ([`human_input_session`]): no agent cursor in the guest, and none
/// in presence. An agent's runs in the provider's coordinate session and
/// keeps the agent cursor.
-#[cfg(target_os = "linux")]
+#[cfg(any(target_os = "linux", target_os = "windows"))]
struct RegistryGestureTools {
inner: Arc<CuaProviderInner>,
/// The human-input session, `None` for an agent.
human_session: Option<String>,
}
-#[cfg(target_os = "linux")]
+#[cfg(any(target_os = "linux", target_os = "windows", test))]
+fn gesture_tool_error(tool: &str, result: &ToolResult) -> ProviderError {
+ let code = match result
+ .structured_content
+ .as_ref()
+ .and_then(|value| value.get("code"))
+ .and_then(serde_json::Value::as_str)
+ {
+ Some("background_unavailable" | "would_require_activation") => {
+ ProviderErrorCode::WouldRequireActivation
+ }
+ _ => ProviderErrorCode::DeliveryFailed,
+ };
+ ProviderError::new(code, format!("{tool}: {}", tool_result_text(result)))
+}
+
+#[cfg(any(target_os = "linux", target_os = "windows"))]
impl tool_input::GestureTools for RegistryGestureTools {
fn invoke(
&self,
@@ -373,10 +389,7 @@ impl tool_input::GestureTools for RegistryGestureTools {
})
.map_err(|error| ProviderError::new(ProviderErrorCode::Internal, error))??;
if result.is_error == Some(true) {
- return Err(ProviderError::new(
- ProviderErrorCode::DeliveryFailed,
- format!("{tool}: {}", tool_result_text(&result)),
- ));
+ return Err(gesture_tool_error(tool, &result));
}
Ok(())
}
@@ -1223,10 +1236,51 @@ impl InteractiveInputProvider for CuaInteractiveInputProvider {
Ok(Some(Arc::new(driver_input::MacosLease(session))))
}
- #[cfg(not(any(target_os = "linux", target_os = "macos")))]
+ #[cfg(target_os = "windows")]
+ {
+ // Windows already exposes qualified gesture tools. Adapt media
+ // batches to those tools, as on Hyprland, instead of advertising
+ // interactive input and then returning no input lease.
+ let inner = self.0.clone();
+ let lease = tool_input::open(
+ target,
+ delivery_mode,
+ |target| {
+ let native = inner.native(target)?;
+ let geometry = inner.clone();
+ Ok(tool_input::ToolInputTarget::Window {
+ pid: native.pid,
+ window_id: native.window_id,
+ extent: Box::new(move || {
+ geometry
+ .native_pixel_geometry(native)
+ .map(|pixels| (pixels.width_px, pixels.height_px))
+ }),
+ })
+ },
+ |display_id| {
+ windows_display::displays()
+ .into_iter()
+ .find(|display| display.id == display_id)
+ .ok_or_else(|| {
+ ProviderError::new(
+ ProviderErrorCode::TargetUnavailable,
+ "display is gone",
+ )
+ })
+ },
+ // Desktop-scope keys address the current foreground app.
+ Box::new(|| None),
+ Arc::new(RegistryGestureTools {
+ inner: self.0.clone(),
+ human_session: (!owner.agent).then(|| human_input_session(&owner.id)),
+ }),
+ )?;
+ Ok(Some(Arc::new(lease.with_desktop_hover())))
+ }
+
+ #[cfg(not(any(target_os = "linux", target_os = "macos", target_os = "windows")))]
{
- // cua-driver has no Windows interactive session yet; one-shot
- // actions go through its tools (desktop scope for a display).
let _ = delivery_mode;
if display_key(target).is_none() {
self.0.native(target)?;
@@ -2991,6 +3045,26 @@ mod tests {
));
}
+ #[test]
+ fn gesture_tools_preserve_activation_required_errors() {
+ let result = ToolResult::error("background keyboard input is unavailable")
+ .with_structured(serde_json::json!({"code":"background_unavailable"}));
+ let error = gesture_tool_error("press_key", &result);
+ assert_eq!(error.code, ProviderErrorCode::WouldRequireActivation);
+ assert_eq!(
+ error.message,
+ "press_key: background keyboard input is unavailable"
+ );
+ }
+
+ #[test]
+ fn gesture_tools_do_not_retry_security_refusals_by_activating() {
+ let result = ToolResult::error("Windows blocked input to an elevated application")
+ .with_structured(serde_json::json!({"code":"background_uipi_blocked"}));
+ let error = gesture_tool_error("click", &result);
+ assert_eq!(error.code, ProviderErrorCode::DeliveryFailed);
+ }
+
#[test]
fn adapter_newtypes_are_send_and_sync() {
fn assert_send_sync<T: Send + Sync>() {}
diff --git a/libs/cua-spacesd/crates/cua-spacesd-desktop/src/tool_input.rs b/libs/cua-spacesd/crates/cua-spacesd-desktop/src/tool_input.rs
index 1d5c325f0b..c4cc837527 100644
--- a/libs/cua-spacesd/crates/cua-spacesd-desktop/src/tool_input.rs
+++ b/libs/cua-spacesd/crates/cua-spacesd-desktop/src/tool_input.rs
@@ -3,15 +3,12 @@
//! Interactive input through the cua-driver tools.
//!
-//! On Hyprland cua-driver has no stateful interactive session: its input
-//! goes through the driver tools (`click`, `drag`, `scroll`, `press_key`,
-//! `type_text`), which route the desktop through the virtual pointer and
-//! keyboard and background window input through the cua-hyprland-plugin
-//! seats. This lease folds each interactive batch into whole gestures with
-//! the shared [`GestureFolder`] and invokes one tool per gesture, so a
-//! streaming viewer's clicks, drags, scrolls and keys reach a Hyprland
-//! desktop or window exactly as the same actions from an agent would. Nothing
-//! here injects input.
+//! Windows and Hyprland interactive batches are folded into whole gestures
+//! with the shared [`GestureFolder`] and passed to the existing cua-driver
+//! tools (`click`, `drag`, `scroll`, `press_key`, `type_text`). Hyprland tools
+//! use virtual pointer/keyboard input and plugin seats for background windows;
+//! Windows tools use their qualified native delivery routes. This adapter
+//! does not inject input itself.
//!
//! Addressing follows the one-shot action path: a window gets `pid`,
//! `window_id` and window-local capture pixels with the session's delivery
@@ -20,23 +17,25 @@
//! gRPC backend's Linux keyboard rule; the desktop scope only when nothing is
//! focused). A display has no window to address background input to, so a
//! background session on one is refused with `WouldRequireActivation`, as on
-//! X11 and macOS; so is a background session on a window the driver's
-//! background route does not reach (the provider checks that when it
-//! resolves the window).
+//! X11 and macOS. A window gesture also preserves an activation-required
+//! refusal from the driver when its background route cannot reach the target.
//!
-//! What a gesture tool cannot carry is not reported as delivered: hover moves
-//! produce no tool call, and a key's down edge is its whole press.
+//! Windows desktop leases also forward hover (a move while no button is held)
+//! through `move_cursor`. Other hover produces no tool call. A key's down edge
+//! is a whole press; clicks and drags are folded until release rather than
+//! delivered as held edges.
-// Used by the Linux (Hyprland) provider; compiled everywhere so its
+// Used by Windows and Linux (Hyprland); compiled everywhere so its
// behavior is tested on every host.
-#![cfg_attr(not(target_os = "linux"), allow(dead_code))]
+#![cfg_attr(not(any(target_os = "linux", target_os = "windows")), allow(dead_code))]
use std::sync::{Arc, Mutex};
use std::time::Instant;
use cua_driver_core::interactive_input::gestures::{Gesture, GestureFolder};
use cua_driver_core::interactive_input::{
- validate_batch, InteractiveDeliveryMode, Modifier, PointerButton,
+ validate_batch, InteractiveDeliveryMode, InteractiveInputEvent, Modifier, PointerButton,
+ PointerPhase,
};
use cua_media_protocol::InteractiveInputBatch;
use cua_spacesd_provider_api::{
@@ -118,7 +117,8 @@ pub(crate) fn open(
target,
mode,
tools,
- folder: Mutex::new(GestureFolder::new()),
+ fold: Mutex::new(Fold::default()),
+ desktop_hover: false,
})
}
@@ -126,7 +126,45 @@ pub(crate) struct ToolInputLease {
target: ToolInputTarget,
mode: InteractiveDeliveryMode,
tools: Arc<dyn GestureTools>,
- folder: Mutex<GestureFolder>,
+ fold: Mutex<Fold>,
+ desktop_hover: bool,
+}
+
+/// The gesture fold, and the button a viewer holds for desktop hover.
+#[derive(Default)]
+struct Fold {
+ folder: GestureFolder,
+ held: Option<PointerButton>,
+}
+
+impl Fold {
+ /// The point of a hover move. Viewers send drag samples as button-less
+ /// moves too, so a move is hover only while no button is held; the held
+ /// button is tracked as [`GestureFolder`] tracks its press.
+ fn hover(&mut self, event: &InteractiveInputEvent) -> Option<(f64, f64)> {
+ let InteractiveInputEvent::Pointer {
+ phase,
+ button,
+ x_normalized,
+ y_normalized,
+ ..
+ } = event
+ else {
+ return None;
+ };
+ let pressed = button.unwrap_or(PointerButton::Left);
+ match phase {
+ PointerPhase::Move if button.is_none() && self.held.is_none() => {
+ return Some((*x_normalized, *y_normalized));
+ }
+ PointerPhase::Move => {}
+ PointerPhase::Down => self.held = Some(pressed),
+ PointerPhase::Up if self.held == Some(pressed) => self.held = None,
+ PointerPhase::Up => {}
+ PointerPhase::Cancel => self.held = None,
+ }
+ None
+ }
}
fn button_name(button: PointerButton) -> &'static str {
@@ -189,6 +227,15 @@ fn desktop_arguments() -> Map<String, Value> {
}
impl ToolInputLease {
+ /// The Windows driver can move the real desktop pointer. Window scope
+ /// moves only its agent overlay, so never opt that route into hover.
+ #[cfg_attr(not(target_os = "windows"), allow(dead_code))]
+ pub(crate) fn with_desktop_hover(mut self) -> Self {
+ self.desktop_hover = matches!(self.target, ToolInputTarget::Display { .. })
+ && self.mode == InteractiveDeliveryMode::PersistentForeground;
+ self
+ }
+
fn extent(&self) -> Result<(u32, u32), ProviderError> {
let extent = match &self.target {
ToolInputTarget::Window { extent, .. } => extent(),
@@ -337,16 +384,37 @@ impl InteractiveInputLease for ToolInputLease {
let batch = driver_batch(batch)?;
let through = validate_batch(&batch).map_err(interactive_error)?;
let extent = self.extent()?;
- let gestures = self
- .folder
+ let mut fold = self
+ .fold
.lock()
- .unwrap_or_else(std::sync::PoisonError::into_inner)
- .fold(&batch.events, extent);
- for gesture in gestures {
- for (tool, arguments) in self.call(gesture, extent) {
- self.tools.invoke(tool, arguments)?;
+ .unwrap_or_else(std::sync::PoisonError::into_inner);
+ let mut calls = Vec::new();
+ // The events between hover moves fold together, so consecutive
+ // scrolls and text stay one tool call each.
+ let mut run = 0;
+ if self.desktop_hover {
+ for (index, event) in batch.events.iter().enumerate() {
+ let Some((x, y)) = fold.hover(event) else {
+ continue;
+ };
+ for gesture in fold.folder.fold(&batch.events[run..index], extent) {
+ calls.extend(self.call(gesture, extent));
+ }
+ let (x, y) = self.point(x, y, extent);
+ let mut arguments = desktop_arguments();
+ arguments.insert("x".into(), x.into());
+ arguments.insert("y".into(), y.into());
+ calls.push(("move_cursor", arguments));
+ run = index + 1;
}
}
+ for gesture in fold.folder.fold(&batch.events[run..], extent) {
+ calls.extend(self.call(gesture, extent));
+ }
+ drop(fold);
+ for (tool, arguments) in calls {
+ self.tools.invoke(tool, arguments)?;
+ }
Ok(InteractiveInputOutcome {
through_sequence: through,
event_count: batch.events.len(),
@@ -357,10 +425,10 @@ impl InteractiveInputLease for ToolInputLease {
/// Gestures are delivered whole, so nothing is held down; forget a
/// half-finished press.
fn release_all(&self) {
- self.folder
+ *self
+ .fold
.lock()
- .unwrap_or_else(std::sync::PoisonError::into_inner)
- .reset();
+ .unwrap_or_else(std::sync::PoisonError::into_inner) = Fold::default();
}
}
@@ -431,6 +499,29 @@ mod tests {
}
}
+ /// A button-less move: hover, or a drag sample while a button is held.
+ fn moved(x: f64, y: f64) -> InteractiveInputEvent {
+ InteractiveInputEvent::Pointer {
+ phase: InputPointerPhase::Move,
+ button: None,
+ x_normalized: x,
+ y_normalized: y,
+ modifiers: Vec::new(),
+ }
+ }
+
+ fn wheel(x: f64, y: f64, delta_y: f64) -> InteractiveInputEvent {
+ InteractiveInputEvent::Scroll {
+ x_normalized: x,
+ y_normalized: y,
+ delta_x: 0.0,
+ delta_y,
+ phase: InputGesturePhase::None,
+ momentum_phase: InputGesturePhase::None,
+ precise: false,
+ }
+ }
+
fn key(name: &str, state: InputKeyState) -> InteractiveInputEvent {
InteractiveInputEvent::Key {
key: name.into(),
@@ -440,6 +531,19 @@ mod tests {
}
}
+ fn hover_lease(tools: Arc<Recorded>) -> ToolInputLease {
+ open(
+ &display_target(),
+ InteractiveDeliveryMode::PersistentForeground,
+ unknown_window,
+ |_| Ok(display()),
+ Box::new(|| None),
+ tools,
+ )
+ .unwrap()
+ .with_desktop_hover()
+ }
+
/// The provider's window catalog, which never enumerates a display.
fn unknown_window(target: &ProviderTargetId) -> Result<ToolInputTarget, ProviderError> {
let native = crate::TargetCatalog::default().native(target)?;
@@ -519,6 +623,136 @@ mod tests {
);
}
+ #[test]
+ fn desktop_hover_preserves_gesture_order_and_scaled_monitor_origin() {
+ let tools = Arc::new(Recorded::default());
+ let mut monitor = display();
+ monitor.scale_factor = 1.5;
+ monitor.bounds = (-1600.0 / 1.5, -300.0 / 1.5, 1280.0 / 1.5, 800.0 / 1.5);
+ let lease = open(
+ &display_target(),
+ InteractiveDeliveryMode::PersistentForeground,
+ unknown_window,
+ |_| Ok(monitor),
+ Box::new(|| None),
+ tools.clone(),
+ )
+ .unwrap()
+ .with_desktop_hover();
+ lease
+ .dispatch(&batch(
+ 5,
+ vec![
+ moved(0.25, 0.5),
+ key("escape", InputKeyState::Down),
+ pointer(InputPointerPhase::Down, 0.5, 0.5),
+ pointer(InputPointerPhase::Up, 0.5, 0.5),
+ moved(1.0, 1.0),
+ ],
+ ))
+ .unwrap();
+ assert_eq!(
+ tools.0.lock().unwrap().clone(),
+ vec![
+ (
+ "move_cursor".into(),
+ json!({"scope":"desktop", "x":-1280.0, "y":100.0})
+ ),
+ (
+ "press_key".into(),
+ json!({"scope":"desktop", "key":"escape"})
+ ),
+ (
+ "click".into(),
+ json!({"scope":"desktop", "x":-960.0, "y":100.0, "button":"left"})
+ ),
+ (
+ "move_cursor".into(),
+ json!({"scope":"desktop", "x":-321.0, "y":499.0})
+ ),
+ ]
+ );
+ }
+
+ /// Regression: viewers send drag samples as button-less moves. Moving
+ /// the real pointer for them would sweep it across the desktop with no
+ /// button held, then jump back when the folded drag replays on release.
+ #[test]
+ fn desktop_hover_ignores_drag_samples_across_batches() {
+ let tools = Arc::new(Recorded::default());
+ let lease = hover_lease(tools.clone());
+ for (first_sequence, events) in [
+ (
+ 1,
+ vec![
+ moved(0.25, 0.25),
+ pointer(InputPointerPhase::Down, 0.25, 0.25),
+ ],
+ ),
+ (3, vec![moved(0.5, 0.5)]),
+ (
+ 4,
+ vec![pointer(InputPointerPhase::Up, 0.75, 0.75), moved(1.0, 1.0)],
+ ),
+ ] {
+ lease.dispatch(&batch(first_sequence, events)).unwrap();
+ }
+ assert_eq!(
+ tools.0.lock().unwrap().clone(),
+ vec![
+ (
+ "move_cursor".into(),
+ json!({"scope":"desktop", "x":320.0, "y":200.0})
+ ),
+ (
+ "drag".into(),
+ json!({
+ "scope":"desktop", "from_x":320.0, "from_y":200.0,
+ "to_x":959.0, "to_y":599.0, "button":"left",
+ })
+ ),
+ (
+ "move_cursor".into(),
+ json!({"scope":"desktop", "x":1279.0, "y":799.0})
+ ),
+ ]
+ );
+ }
+
+ #[test]
+ fn desktop_hover_keeps_consecutive_scrolls_one_tool_call() {
+ let tools = Arc::new(Recorded::default());
+ let lease = hover_lease(tools.clone());
+ lease
+ .dispatch(&batch(
+ 1,
+ vec![
+ wheel(0.25, 0.25, 1.0),
+ wheel(0.5, 0.5, 2.0),
+ moved(0.75, 0.75),
+ wheel(0.75, 0.75, 1.0),
+ ],
+ ))
+ .unwrap();
+ assert_eq!(
+ tools.0.lock().unwrap().clone(),
+ vec![
+ (
+ "scroll".into(),
+ json!({"scope":"desktop", "x":640.0, "y":400.0, "direction":"down", "amount":3})
+ ),
+ (
+ "move_cursor".into(),
+ json!({"scope":"desktop", "x":959.0, "y":599.0})
+ ),
+ (
+ "scroll".into(),
+ json!({"scope":"desktop", "x":959.0, "y":599.0, "direction":"down", "amount":1})
+ ),
+ ]
+ );
+ }
+
#[test]
fn desktop_keys_without_a_focused_window_use_the_desktop_scope() {
let tools = Arc::new(Recorded::default());
diff --git a/libs/cua-spacesd/crates/cua-spacesd-server/src/process/spawn.rs b/libs/cua-spacesd/crates/cua-spacesd-server/src/process/spawn.rs
index 8ed9dbfe24..4b90d2eaf9 100644
--- a/libs/cua-spacesd/crates/cua-spacesd-server/src/process/spawn.rs
+++ b/libs/cua-spacesd/crates/cua-spacesd-server/src/process/spawn.rs
@@ -221,6 +221,12 @@ pub fn spawn_pipes(spec: &SpawnSpec) -> io::Result<Spawned> {
command.process_group(0);
install_user_switch(&mut command, spec.switch_user.clone(), false);
}
+ #[cfg(windows)]
+ {
+ // Pipes already carry all shell I/O. A background host must not
+ // allocate a visible console for each ProcessService command.
+ command.creation_flags(0x08000000); // CREATE_NO_WINDOW
+ }
let mut child = command.spawn()?;
let pid = child.id().unwrap_or(0);
let stdin = child
diff --git a/libs/cua-spacesd/crates/cua-spacesd-session/src/media/viewer.rs b/libs/cua-spacesd/crates/cua-spacesd-session/src/media/viewer.rs
index 05b995265a..0782197ede 100644
--- a/libs/cua-spacesd/crates/cua-spacesd-session/src/media/viewer.rs
+++ b/libs/cua-spacesd/crates/cua-spacesd-session/src/media/viewer.rs
@@ -581,6 +581,8 @@ async fn run_input_job(
let started = Instant::now();
let events = batch.events.len() as u64;
let nack = |code: ActionErrorCode, message: String| {
+ tracing::warn!(target: "cua_spacesd_client::host_input", ?code,
+ "viewer input dispatch rejected");
ServerMessage::InteractiveInputAcknowledgement(InteractiveInputAcknowledgement {
session_id: session_id.clone(),
through_sequence: through,
@@ -832,6 +834,10 @@ impl Viewer {
async fn dispatch_input(&mut self, batch: InteractiveInputBatch) -> Option<ServerMessage> {
let session_id = self.session_id();
let nack = |through: u64, code: ActionErrorCode, message: String| {
+ // Admission errors contain permission/lease state, never the
+ // contents of a user's keyboard or pointer batch.
+ tracing::warn!(target: "cua_spacesd_client::host_input", ?code,
+ reason = %message, "viewer input admission rejected");
ServerMessage::InteractiveInputAcknowledgement(InteractiveInputAcknowledgement {
session_id: session_id.clone(),
through_sequence: through,