Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

fix(cua-driver): report native Wayland modified-drag refusals with a stable code (#4360)

Francesco Bonacci62cdfbcd9356
diff --git a/libs/cua-driver/docs/test-harnesses-guide.md b/libs/cua-driver/docs/test-harnesses-guide.md
index 99fd714661..e798c17965 100644
--- a/libs/cua-driver/docs/test-harnesses-guide.md
+++ b/libs/cua-driver/docs/test-harnesses-guide.md
@@ -410,7 +410,10 @@ The shared and native action matrices should test left click, right click,
double click, typing, keys, hotkeys, scroll, child windows, and drag across
AX/PX and foreground/background combinations where the driver supports them.
Unsupported background routes require an explicit refusal contract with an
-allowed structured code and desktop-side-effect oracles. A refusal fails a
+allowed structured code and desktop-side-effect oracles. A desktop-scope
+gesture that the platform's pointer route cannot carry uses the same contract
+with the fixture under the gesture as its oracle; today that is only a modified
+drag on native Wayland (`modified_pointer_unavailable`). A refusal fails a
cell that requires delivery. There should not be a separate "delivery" family
whose only purpose is to repeat those same actions in the background.
diff --git a/libs/cua-driver/rust/crates/cua-driver-e2e/tests/desktop_scope_linux_test.rs b/libs/cua-driver/rust/crates/cua-driver-e2e/tests/desktop_scope_linux_test.rs
index 260610d928..7ad7b3c25d 100644
--- a/libs/cua-driver/rust/crates/cua-driver-e2e/tests/desktop_scope_linux_test.rs
+++ b/libs/cua-driver/rust/crates/cua-driver-e2e/tests/desktop_scope_linux_test.rs
@@ -24,7 +24,7 @@ use std::time::{Duration, Instant};
use cua_driver_testkit::e2e::{
execute_case, recording_evidence, CaseSpec, Delivery, DriverRoute, Evidence, Observation,
- OracleKind, Scope, Targeting,
+ OracleKind, RefusalCode, Scope, Targeting,
};
use cua_driver_testkit::{harness_app, Driver, McpDriver};
@@ -455,10 +455,29 @@ fn desktop_scope_hotkey_releases_modifiers() {
});
}
+/// Route for a modified desktop drag. X11 holds the modifier through XTest.
+/// Every native Wayland session, including the nested cua-compositor lane,
+/// reaches desktop drags through the Wayland desktop pointer: the
+/// cua-compositor socket has no drag command, and its pointer commands carry
+/// no modifier state (its only modifier mask is transient inside a hotkey).
+fn desktop_drag_route(native_wayland: bool) -> DriverRoute {
+ if native_wayland {
+ DriverRoute::LinuxWaylandVirtualPointer
+ } else {
+ DriverRoute::LinuxXTest
+ }
+}
+
+/// X11 holds the modifier for the whole desktop drag and releases it after.
+/// Native Wayland pointer routes cannot carry keyboard modifier state, so the
+/// driver must refuse with `modified_pointer_unavailable` before dispatch: the
+/// fixture under the gesture sees no press, drag, or click, and a following
+/// plain key still arrives unmodified.
#[test]
#[ignore]
fn desktop_scope_drag_holds_and_releases_modifier() {
let cell_id = "linux-gtk3-desktop-drag-modifier-px-foreground";
+ let native_wayland = platform_linux::wayland::wayland_input_enabled();
let case = CaseSpec::delivered(
cell_id,
"gtk3",
@@ -467,9 +486,14 @@ fn desktop_scope_drag_holds_and_releases_modifier() {
Targeting::Px,
Delivery::Foreground,
Scope::Desktop,
- desktop_input_route(),
+ desktop_drag_route(native_wayland),
vec![OracleKind::FixtureState],
);
+ let case = if native_wayland {
+ case.expecting_refusal(vec![RefusalCode::ModifiedPointerUnavailable])
+ } else {
+ case
+ };
execute_case(case, |evidence| {
let mut driver = McpDriver::spawn_named(cell_id).expect("start source-built Linux driver");
*evidence = recording_evidence(driver.recording_dir());
@@ -493,6 +517,8 @@ fn desktop_scope_drag_holds_and_releases_modifier() {
let (x, y) = element_center_containing(&snap, "btn-clicktarget")
.expect("GTK3 drag target frame not found");
let before_keys = marker_value(&snap, "key_presses=").unwrap_or(0);
+ let before_drags = marker_value(&snap, "drag_events=").unwrap_or(0);
+ let before_clicks = marker_value(&snap, "clicks=").unwrap_or(0);
driver.start_behavior_recording();
let drag = driver.call(
"drag",
@@ -502,26 +528,64 @@ fn desktop_scope_drag_holds_and_releases_modifier() {
"duration_ms": 250, "steps": 8, "modifier": ["ctrl"]
}),
);
- assert!(
- !drag.is_error(),
- "modified desktop drag failed: {}",
- drag.text()
- );
- let deadline = Instant::now() + Duration::from_secs(3);
- loop {
- let state = ax_snapshot(&mut driver, &window_session, pid, wid);
- if tree_contains(&state, "drag_modifier=ctrl") {
- break;
- }
+ let refusal = if native_wayland {
assert!(
- Instant::now() < deadline,
- "GTK3 did not observe ctrl during drag: {}",
+ drag.is_error(),
+ "native Wayland accepted a modified desktop drag it cannot hold: {}",
drag.text()
);
- std::thread::sleep(Duration::from_millis(100));
- }
+ let code = drag.structured()["code"]
+ .as_str()
+ .and_then(RefusalCode::from_driver_code);
+ assert_eq!(
+ code,
+ Some(RefusalCode::ModifiedPointerUnavailable),
+ "native Wayland returned the wrong refusal: {}; raw={}",
+ drag.text(),
+ drag.raw
+ );
+ assert_eq!(
+ drag.text(),
+ "modified drags are unavailable on native Wayland: \
+ the pointer route cannot carry keyboard modifier state"
+ );
+ assert_eq!(drag.structured()["effect"], "refused");
+ std::thread::sleep(Duration::from_millis(250));
+ let after = ax_snapshot(&mut driver, &window_session, pid, wid);
+ assert_eq!(
+ (
+ marker_value(&after, "drag_events=").unwrap_or(before_drags),
+ marker_value(&after, "clicks=").unwrap_or(before_clicks),
+ tree_contains(&after, "drag_modifier=none"),
+ ),
+ (before_drags, before_clicks, true),
+ "refused modified drag reached the GTK3 fixture"
+ );
+ code
+ } else {
+ assert!(
+ !drag.is_error(),
+ "modified desktop drag failed: {}",
+ drag.text()
+ );
+ let deadline = Instant::now() + Duration::from_secs(3);
+ loop {
+ let state = ax_snapshot(&mut driver, &window_session, pid, wid);
+ if tree_contains(&state, "drag_modifier=ctrl") {
+ break;
+ }
+ assert!(
+ Instant::now() < deadline,
+ "GTK3 did not observe ctrl during drag: {}",
+ drag.text()
+ );
+ std::thread::sleep(Duration::from_millis(100));
+ }
+ None
+ };
+ // Delivered or refused, no modifier may be left latched.
let plain_key = driver.call(
"press_key",
serde_json::json!({"session": desktop_session, "scope": "desktop", "key": "f5"}),
@@ -543,7 +607,15 @@ fn desktop_scope_drag_holds_and_releases_modifier() {
);
std::thread::sleep(Duration::from_millis(100));
}
- Observation::delivered_with_fixture_state(Vec::new())
+ match refusal {
+ Some(code) => Observation::refused(
+ code,
+ vec![OracleKind::FixtureState],
+ "native Wayland refused the modified desktop drag before dispatch",
+ Evidence::default(),
+ ),
+ None => Observation::delivered_with_fixture_state(Vec::new()),
+ }
});
}
diff --git a/libs/cua-driver/rust/crates/cua-driver-testkit/src/e2e.rs b/libs/cua-driver/rust/crates/cua-driver-testkit/src/e2e.rs
index 4c4680cfaf..242294f1e4 100644
--- a/libs/cua-driver/rust/crates/cua-driver-testkit/src/e2e.rs
+++ b/libs/cua-driver/rust/crates/cua-driver-testkit/src/e2e.rs
@@ -316,6 +316,7 @@ pub enum RefusalCode {
BrowserReconnectExhausted,
BrowserInputIncomplete,
BrowserActionUnavailable,
+ ModifiedPointerUnavailable,
WindowMinimized,
}
@@ -341,6 +342,7 @@ impl RefusalCode {
"browser_reconnect_exhausted" => Some(Self::BrowserReconnectExhausted),
"browser_input_incomplete" => Some(Self::BrowserInputIncomplete),
"browser_action_unavailable" => Some(Self::BrowserActionUnavailable),
+ "modified_pointer_unavailable" => Some(Self::ModifiedPointerUnavailable),
"window_minimized" => Some(Self::WindowMinimized),
_ => None,
}
@@ -580,24 +582,35 @@ impl CaseSpec {
&& self.scope == Scope::Window
&& self.driver_route == DriverRoute::WindowState
&& allowed_codes == &[RefusalCode::BringToFrontExactWindowUnverified];
- if self.delivery != Delivery::Background && !exact_activation_refusal {
+ // A desktop pointer route that cannot carry keyboard modifier
+ // state (native Wayland) must refuse a modified gesture before
+ // dispatch. The fixture under the gesture is the oracle that
+ // nothing reached it.
+ let desktop_modifier_refusal = self.delivery == Delivery::Foreground
+ && self.scope == Scope::Desktop
+ && allowed_codes == &[RefusalCode::ModifiedPointerUnavailable];
+ if self.delivery != Delivery::Background
+ && !exact_activation_refusal
+ && !desktop_modifier_refusal
+ {
return Err(format!(
- "{}: only background delivery or exact-window activation may declare refusal",
+ "{}: only background delivery, exact-window activation, or a desktop modified-pointer limitation may declare refusal",
self.cell_id
));
}
if allowed_codes.is_empty() {
return Err(format!("{}: refusal has no allowed code", self.cell_id));
}
- let required_oracles: &[OracleKind] = if exact_activation_refusal {
- &[OracleKind::FixtureState]
- } else {
- &[
- OracleKind::Focus,
- OracleKind::ZOrder,
- OracleKind::NoLeakedInput,
- ]
- };
+ let required_oracles: &[OracleKind] =
+ if exact_activation_refusal || desktop_modifier_refusal {
+ &[OracleKind::FixtureState]
+ } else {
+ &[
+ OracleKind::Focus,
+ OracleKind::ZOrder,
+ OracleKind::NoLeakedInput,
+ ]
+ };
for required in required_oracles {
if !self.oracles.contains(required) {
return Err(format!(
@@ -2322,6 +2335,61 @@ mod tests {
assert_eq!(RefusalCode::from_driver_code("background_timeout"), None);
}
+ #[test]
+ fn desktop_refusal_is_limited_to_the_modified_pointer_limitation() {
+ assert_eq!(
+ RefusalCode::from_driver_code("modified_pointer_unavailable"),
+ Some(RefusalCode::ModifiedPointerUnavailable)
+ );
+ let desktop_drag = |code| {
+ CaseSpec::delivered(
+ "desktop-modified-drag",
+ "gtk3",
+ "gtk3",
+ "drag",
+ Targeting::Px,
+ Delivery::Foreground,
+ Scope::Desktop,
+ DriverRoute::LinuxWaylandVirtualPointer,
+ vec![OracleKind::FixtureState],
+ )
+ .expecting_refusal(vec![code])
+ };
+ let case = desktop_drag(RefusalCode::ModifiedPointerUnavailable);
+ case.validate()
+ .expect("a desktop modified-pointer limitation may be declared");
+ assert!(desktop_drag(RefusalCode::BackgroundUnavailable)
+ .validate()
+ .is_err());
+ let mut window_scoped = case.clone();
+ window_scoped.scope = Scope::Window;
+ assert!(window_scoped.validate().is_err());
+ let mut without_fixture = case.clone();
+ without_fixture.oracles = vec![OracleKind::Protocol];
+ assert!(without_fixture.validate().is_err());
+
+ let refused = CaseResult::evaluate(
+ case.clone(),
+ Observation::refused(
+ RefusalCode::ModifiedPointerUnavailable,
+ vec![OracleKind::FixtureState],
+ "",
+ Evidence::default(),
+ ),
+ Duration::from_millis(1),
+ );
+ assert_eq!(refused.test_status, TestStatus::Pass);
+ let delivered = CaseResult::evaluate(
+ case,
+ Observation::delivered_with_fixture_state(Vec::new()),
+ Duration::from_millis(1),
+ );
+ assert_eq!(delivered.test_status, TestStatus::Fail);
+ assert!(delivered
+ .message
+ .contains("unexpected delivery requires contract review"));
+ }
+
#[test]
fn validator_rejects_missing_and_duplicate_results() {
let case = delivered_case("one");
diff --git a/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs b/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs
index a56354bd21..c2017c7afb 100644
--- a/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs
+++ b/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs
@@ -10347,6 +10347,24 @@ impl Tool for RightClickTool {
// ── drag ─────────────────────────────────────────────────────────────────────
+/// Typed refusal for a modified drag on native Wayland. The Wayland pointer
+/// routes (virtual pointer, libei, and the nested cua-compositor socket) do not
+/// hold keyboard modifier state across a pointer gesture, so the drag is
+/// refused before any input is dispatched rather than degraded to an
+/// unmodified drag. The stable `code` lets callers branch without parsing text.
+fn wayland_modified_drag_refusal() -> ToolResult {
+ const DETAIL: &str = "the pointer route cannot carry keyboard modifier state";
+ ToolResult::error(format!(
+ "modified drags are unavailable on native Wayland: {DETAIL}"
+ ))
+ .with_structured(json!({
+ "code": "modified_pointer_unavailable",
+ "effect": "refused",
+ "verified": false,
+ "detail": DETAIL,
+ }))
+}
+
pub struct DragTool {
state: Arc<ToolState>,
}
@@ -10416,9 +10434,7 @@ impl Tool for DragTool {
let modifiers = input.modifier.unwrap_or_default();
let wayland = crate::wayland::wayland_input_enabled();
if wayland && !modifiers.is_empty() {
- return ToolResult::error(
- "modified drags are unavailable on native Wayland: the pointer route cannot carry keyboard modifier state",
- );
+ return wayland_modified_drag_refusal();
}
let path = if wayland { "wayland_desktop" } else { "xtest" };
let result = tokio::task::spawn_blocking(move || {
@@ -10479,9 +10495,7 @@ impl Tool for DragTool {
let isolated_background = isolated_hyprland_background(delivery);
let native_refusal = if !isolated_background {
if crate::wayland::wayland_input_enabled() && !modifiers.is_empty() {
- return ToolResult::error(
- "modified drags are unavailable on native Wayland: the pointer route cannot carry keyboard modifier state",
- );
+ return wayland_modified_drag_refusal();
}
if let Some(refusal) = unavailable_chromium_background(pid, delivery) {
return refusal;