Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
fix(cua-driver): report native Wayland modified-drag refusals with a stable code (#4360)
Francesco Bonacci62cdfbcd9356
diff --git a/libs/cua-driver/docs/test-harnesses-guide.md b/libs/cua-driver/docs/test-harnesses-guide.mdindex 99fd714661..e798c17965 100644--- a/libs/cua-driver/docs/test-harnesses-guide.md+++ b/libs/cua-driver/docs/test-harnesses-guide.md@@ -410,7 +410,10 @@ The shared and native action matrices should test left click, right click,double click, typing, keys, hotkeys, scroll, child windows, and drag acrossAX/PX and foreground/background combinations where the driver supports them.Unsupported background routes require an explicit refusal contract with an-allowed structured code and desktop-side-effect oracles. A refusal fails a+allowed structured code and desktop-side-effect oracles. A desktop-scope+gesture that the platform's pointer route cannot carry uses the same contract+with the fixture under the gesture as its oracle; today that is only a modified+drag on native Wayland (`modified_pointer_unavailable`). A refusal fails acell that requires delivery. There should not be a separate "delivery" familywhose only purpose is to repeat those same actions in the background.diff --git a/libs/cua-driver/rust/crates/cua-driver-e2e/tests/desktop_scope_linux_test.rs b/libs/cua-driver/rust/crates/cua-driver-e2e/tests/desktop_scope_linux_test.rsindex 260610d928..7ad7b3c25d 100644--- a/libs/cua-driver/rust/crates/cua-driver-e2e/tests/desktop_scope_linux_test.rs+++ b/libs/cua-driver/rust/crates/cua-driver-e2e/tests/desktop_scope_linux_test.rs@@ -24,7 +24,7 @@ use std::time::{Duration, Instant};use cua_driver_testkit::e2e::{execute_case, recording_evidence, CaseSpec, Delivery, DriverRoute, Evidence, Observation,- OracleKind, Scope, Targeting,+ OracleKind, RefusalCode, Scope, Targeting,};use cua_driver_testkit::{harness_app, Driver, McpDriver};@@ -455,10 +455,29 @@ fn desktop_scope_hotkey_releases_modifiers() {});}+/// Route for a modified desktop drag. X11 holds the modifier through XTest.+/// Every native Wayland session, including the nested cua-compositor lane,+/// reaches desktop drags through the Wayland desktop pointer: the+/// cua-compositor socket has no drag command, and its pointer commands carry+/// no modifier state (its only modifier mask is transient inside a hotkey).+fn desktop_drag_route(native_wayland: bool) -> DriverRoute {+ if native_wayland {+ DriverRoute::LinuxWaylandVirtualPointer+ } else {+ DriverRoute::LinuxXTest+ }+}++/// X11 holds the modifier for the whole desktop drag and releases it after.+/// Native Wayland pointer routes cannot carry keyboard modifier state, so the+/// driver must refuse with `modified_pointer_unavailable` before dispatch: the+/// fixture under the gesture sees no press, drag, or click, and a following+/// plain key still arrives unmodified.#[test]#[ignore]fn desktop_scope_drag_holds_and_releases_modifier() {let cell_id = "linux-gtk3-desktop-drag-modifier-px-foreground";+ let native_wayland = platform_linux::wayland::wayland_input_enabled();let case = CaseSpec::delivered(cell_id,"gtk3",@@ -467,9 +486,14 @@ fn desktop_scope_drag_holds_and_releases_modifier() {Targeting::Px,Delivery::Foreground,Scope::Desktop,- desktop_input_route(),+ desktop_drag_route(native_wayland),vec![OracleKind::FixtureState],);+ let case = if native_wayland {+ case.expecting_refusal(vec![RefusalCode::ModifiedPointerUnavailable])+ } else {+ case+ };execute_case(case, |evidence| {let mut driver = McpDriver::spawn_named(cell_id).expect("start source-built Linux driver");*evidence = recording_evidence(driver.recording_dir());@@ -493,6 +517,8 @@ fn desktop_scope_drag_holds_and_releases_modifier() {let (x, y) = element_center_containing(&snap, "btn-clicktarget").expect("GTK3 drag target frame not found");let before_keys = marker_value(&snap, "key_presses=").unwrap_or(0);+ let before_drags = marker_value(&snap, "drag_events=").unwrap_or(0);+ let before_clicks = marker_value(&snap, "clicks=").unwrap_or(0);driver.start_behavior_recording();let drag = driver.call("drag",@@ -502,26 +528,64 @@ fn desktop_scope_drag_holds_and_releases_modifier() {"duration_ms": 250, "steps": 8, "modifier": ["ctrl"]}),);- assert!(- !drag.is_error(),- "modified desktop drag failed: {}",- drag.text()- );- let deadline = Instant::now() + Duration::from_secs(3);- loop {- let state = ax_snapshot(&mut driver, &window_session, pid, wid);- if tree_contains(&state, "drag_modifier=ctrl") {- break;- }+ let refusal = if native_wayland {assert!(- Instant::now() < deadline,- "GTK3 did not observe ctrl during drag: {}",+ drag.is_error(),+ "native Wayland accepted a modified desktop drag it cannot hold: {}",drag.text());- std::thread::sleep(Duration::from_millis(100));- }+ let code = drag.structured()["code"]+ .as_str()+ .and_then(RefusalCode::from_driver_code);+ assert_eq!(+ code,+ Some(RefusalCode::ModifiedPointerUnavailable),+ "native Wayland returned the wrong refusal: {}; raw={}",+ drag.text(),+ drag.raw+ );+ assert_eq!(+ drag.text(),+ "modified drags are unavailable on native Wayland: \+ the pointer route cannot carry keyboard modifier state"+ );+ assert_eq!(drag.structured()["effect"], "refused");+ std::thread::sleep(Duration::from_millis(250));+ let after = ax_snapshot(&mut driver, &window_session, pid, wid);+ assert_eq!(+ (+ marker_value(&after, "drag_events=").unwrap_or(before_drags),+ marker_value(&after, "clicks=").unwrap_or(before_clicks),+ tree_contains(&after, "drag_modifier=none"),+ ),+ (before_drags, before_clicks, true),+ "refused modified drag reached the GTK3 fixture"+ );+ code+ } else {+ assert!(+ !drag.is_error(),+ "modified desktop drag failed: {}",+ drag.text()+ );+ let deadline = Instant::now() + Duration::from_secs(3);+ loop {+ let state = ax_snapshot(&mut driver, &window_session, pid, wid);+ if tree_contains(&state, "drag_modifier=ctrl") {+ break;+ }+ assert!(+ Instant::now() < deadline,+ "GTK3 did not observe ctrl during drag: {}",+ drag.text()+ );+ std::thread::sleep(Duration::from_millis(100));+ }+ None+ };+ // Delivered or refused, no modifier may be left latched.let plain_key = driver.call("press_key",serde_json::json!({"session": desktop_session, "scope": "desktop", "key": "f5"}),@@ -543,7 +607,15 @@ fn desktop_scope_drag_holds_and_releases_modifier() {);std::thread::sleep(Duration::from_millis(100));}- Observation::delivered_with_fixture_state(Vec::new())+ match refusal {+ Some(code) => Observation::refused(+ code,+ vec![OracleKind::FixtureState],+ "native Wayland refused the modified desktop drag before dispatch",+ Evidence::default(),+ ),+ None => Observation::delivered_with_fixture_state(Vec::new()),+ }});}diff --git a/libs/cua-driver/rust/crates/cua-driver-testkit/src/e2e.rs b/libs/cua-driver/rust/crates/cua-driver-testkit/src/e2e.rsindex 4c4680cfaf..242294f1e4 100644--- a/libs/cua-driver/rust/crates/cua-driver-testkit/src/e2e.rs+++ b/libs/cua-driver/rust/crates/cua-driver-testkit/src/e2e.rs@@ -316,6 +316,7 @@ pub enum RefusalCode {BrowserReconnectExhausted,BrowserInputIncomplete,BrowserActionUnavailable,+ ModifiedPointerUnavailable,WindowMinimized,}@@ -341,6 +342,7 @@ impl RefusalCode {"browser_reconnect_exhausted" => Some(Self::BrowserReconnectExhausted),"browser_input_incomplete" => Some(Self::BrowserInputIncomplete),"browser_action_unavailable" => Some(Self::BrowserActionUnavailable),+ "modified_pointer_unavailable" => Some(Self::ModifiedPointerUnavailable),"window_minimized" => Some(Self::WindowMinimized),_ => None,}@@ -580,24 +582,35 @@ impl CaseSpec {&& self.scope == Scope::Window&& self.driver_route == DriverRoute::WindowState&& allowed_codes == &[RefusalCode::BringToFrontExactWindowUnverified];- if self.delivery != Delivery::Background && !exact_activation_refusal {+ // A desktop pointer route that cannot carry keyboard modifier+ // state (native Wayland) must refuse a modified gesture before+ // dispatch. The fixture under the gesture is the oracle that+ // nothing reached it.+ let desktop_modifier_refusal = self.delivery == Delivery::Foreground+ && self.scope == Scope::Desktop+ && allowed_codes == &[RefusalCode::ModifiedPointerUnavailable];+ if self.delivery != Delivery::Background+ && !exact_activation_refusal+ && !desktop_modifier_refusal+ {return Err(format!(- "{}: only background delivery or exact-window activation may declare refusal",+ "{}: only background delivery, exact-window activation, or a desktop modified-pointer limitation may declare refusal",self.cell_id));}if allowed_codes.is_empty() {return Err(format!("{}: refusal has no allowed code", self.cell_id));}- let required_oracles: &[OracleKind] = if exact_activation_refusal {- &[OracleKind::FixtureState]- } else {- &[- OracleKind::Focus,- OracleKind::ZOrder,- OracleKind::NoLeakedInput,- ]- };+ let required_oracles: &[OracleKind] =+ if exact_activation_refusal || desktop_modifier_refusal {+ &[OracleKind::FixtureState]+ } else {+ &[+ OracleKind::Focus,+ OracleKind::ZOrder,+ OracleKind::NoLeakedInput,+ ]+ };for required in required_oracles {if !self.oracles.contains(required) {return Err(format!(@@ -2322,6 +2335,61 @@ mod tests {assert_eq!(RefusalCode::from_driver_code("background_timeout"), None);}+ #[test]+ fn desktop_refusal_is_limited_to_the_modified_pointer_limitation() {+ assert_eq!(+ RefusalCode::from_driver_code("modified_pointer_unavailable"),+ Some(RefusalCode::ModifiedPointerUnavailable)+ );+ let desktop_drag = |code| {+ CaseSpec::delivered(+ "desktop-modified-drag",+ "gtk3",+ "gtk3",+ "drag",+ Targeting::Px,+ Delivery::Foreground,+ Scope::Desktop,+ DriverRoute::LinuxWaylandVirtualPointer,+ vec![OracleKind::FixtureState],+ )+ .expecting_refusal(vec![code])+ };+ let case = desktop_drag(RefusalCode::ModifiedPointerUnavailable);+ case.validate()+ .expect("a desktop modified-pointer limitation may be declared");+ assert!(desktop_drag(RefusalCode::BackgroundUnavailable)+ .validate()+ .is_err());+ let mut window_scoped = case.clone();+ window_scoped.scope = Scope::Window;+ assert!(window_scoped.validate().is_err());+ let mut without_fixture = case.clone();+ without_fixture.oracles = vec![OracleKind::Protocol];+ assert!(without_fixture.validate().is_err());++ let refused = CaseResult::evaluate(+ case.clone(),+ Observation::refused(+ RefusalCode::ModifiedPointerUnavailable,+ vec![OracleKind::FixtureState],+ "",+ Evidence::default(),+ ),+ Duration::from_millis(1),+ );+ assert_eq!(refused.test_status, TestStatus::Pass);+ let delivered = CaseResult::evaluate(+ case,+ Observation::delivered_with_fixture_state(Vec::new()),+ Duration::from_millis(1),+ );+ assert_eq!(delivered.test_status, TestStatus::Fail);+ assert!(delivered+ .message+ .contains("unexpected delivery requires contract review"));+ }+#[test]fn validator_rejects_missing_and_duplicate_results() {let case = delivered_case("one");diff --git a/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs b/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rsindex a56354bd21..c2017c7afb 100644--- a/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs+++ b/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs@@ -10347,6 +10347,24 @@ impl Tool for RightClickTool {// ── drag ─────────────────────────────────────────────────────────────────────+/// Typed refusal for a modified drag on native Wayland. The Wayland pointer+/// routes (virtual pointer, libei, and the nested cua-compositor socket) do not+/// hold keyboard modifier state across a pointer gesture, so the drag is+/// refused before any input is dispatched rather than degraded to an+/// unmodified drag. The stable `code` lets callers branch without parsing text.+fn wayland_modified_drag_refusal() -> ToolResult {+ const DETAIL: &str = "the pointer route cannot carry keyboard modifier state";+ ToolResult::error(format!(+ "modified drags are unavailable on native Wayland: {DETAIL}"+ ))+ .with_structured(json!({+ "code": "modified_pointer_unavailable",+ "effect": "refused",+ "verified": false,+ "detail": DETAIL,+ }))+}+pub struct DragTool {state: Arc<ToolState>,}@@ -10416,9 +10434,7 @@ impl Tool for DragTool {let modifiers = input.modifier.unwrap_or_default();let wayland = crate::wayland::wayland_input_enabled();if wayland && !modifiers.is_empty() {- return ToolResult::error(- "modified drags are unavailable on native Wayland: the pointer route cannot carry keyboard modifier state",- );+ return wayland_modified_drag_refusal();}let path = if wayland { "wayland_desktop" } else { "xtest" };let result = tokio::task::spawn_blocking(move || {@@ -10479,9 +10495,7 @@ impl Tool for DragTool {let isolated_background = isolated_hyprland_background(delivery);let native_refusal = if !isolated_background {if crate::wayland::wayland_input_enabled() && !modifiers.is_empty() {- return ToolResult::error(- "modified drags are unavailable on native Wayland: the pointer route cannot carry keyboard modifier state",- );+ return wayland_modified_drag_refusal();}if let Some(refusal) = unavailable_chromium_background(pid, delivery) {return refusal;