Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

fix(cua-driver): cover the desktop lane for invoke_menu and type_text_chars on Linux (#6653)

injaneity373346436ff9
diff --git a/libs/cua-driver/rust/crates/cua-driver-e2e/tests/harness_gtk3_test.rs b/libs/cua-driver/rust/crates/cua-driver-e2e/tests/harness_gtk3_test.rs
index 0c42a4ff7d..f9d2c87b67 100644
--- a/libs/cua-driver/rust/crates/cua-driver-e2e/tests/harness_gtk3_test.rs
+++ b/libs/cua-driver/rust/crates/cua-driver-e2e/tests/harness_gtk3_test.rs
@@ -582,6 +582,137 @@ fn harness_gtk3_invoke_menu_live_path() {
);
}
+/// Send `tool` while another client's drag holds the slider and the desktop
+/// lane, and require that its effect lands only after the drag releases
+/// (#6638). The fixture records an effect that arrives while its slider
+/// button is still held.
+fn assert_waits_for_desktop_lane(
+ pid: u32,
+ window_id: u64,
+ driver: &mut McpDriver,
+ tool: &str,
+ args: impl FnOnce(&ToolResponse) -> serde_json::Value,
+ expected: &str,
+) -> Observation {
+ let mut peer = driver
+ .spawn_peer_unrecorded()
+ .expect("start a second client on the same daemon");
+ // Screenshot pixels are bound to the client that captured them.
+ let peer_state = snapshot(&mut peer, pid, window_id);
+ let (x, y, width, height) = element_rect(&mut peer, pid, window_id, &peer_state, "sld-value");
+ let (drag, response) = std::thread::scope(|scope| {
+ let drag = scope.spawn(move || {
+ peer.call(
+ "drag",
+ serde_json::json!({
+ "pid": pid as i64, "window_id": window_id,
+ "from_x": x + width * 0.05, "from_y": y + height / 2.0,
+ "to_x": x + width * 0.90, "to_y": y + height / 2.0,
+ "duration_ms": 3000, "steps": 60, "delivery_mode": "foreground"
+ }),
+ )
+ });
+ // A read-only snapshot never waits for the lane, so the slider moving
+ // proves the drag holds it now.
+ let (state, slider) = loop {
+ let state = snapshot(driver, pid, window_id);
+ if let Some(slider) =
+ state_number(state.tree_text(), "slider_value=").filter(|value| *value > 0)
+ {
+ break (state, slider);
+ }
+ assert!(
+ !drag.is_finished(),
+ "the peer's drag never moved the slider"
+ );
+ std::thread::sleep(Duration::from_millis(50));
+ };
+ // Over half of the drag (1.5 s) is left, so the call reaches the
+ // daemon while the drag holds the lane unless this thread stalls.
+ assert!(
+ slider < 45 && !drag.is_finished(),
+ "too little of the drag was left to send {tool} under it (slider_value={slider})"
+ );
+ let response = driver.call(tool, args(&state));
+ (drag.join().expect("drag thread"), response)
+ });
+ assert!(!drag.is_error(), "peer drag failed: {}", drag.text());
+ assert!(!response.is_error(), "{tool} failed: {}", response.text());
+ let state = wait_for_state(driver, pid, window_id, expected);
+ // Without exactly one observed press and release the marker below could
+ // not see an overlap.
+ assert!(
+ state.contains("slider_presses=1 slider_releases=1 "),
+ "the fixture did not observe the drag's press and release: {state}"
+ );
+ assert!(
+ state.contains("effect_during_drag=none"),
+ "{tool} took effect while the drag held the slider: it ran beside the drag instead of waiting for the desktop lane"
+ );
+ assert!(
+ state_number(&state, "slider_value=").is_some_and(|value| value >= 80),
+ "the drag did not reach its end: {state}"
+ );
+ Observation::delivered_with_fixture_state(Vec::new())
+}
+
+#[test]
+#[ignore]
+fn harness_gtk3_invoke_menu_waits_for_desktop_lane() {
+ run_case(
+ native_foreground_case(
+ "gtk3",
+ "invoke_menu_desktop_lane",
+ Targeting::Ax,
+ DriverRoute::LinuxAtSpiAction,
+ ),
+ |pid, window_id, driver| {
+ assert_waits_for_desktop_lane(
+ pid,
+ window_id,
+ driver,
+ "invoke_menu",
+ |_| {
+ serde_json::json!({
+ "pid": pid, "window_id": window_id,
+ "path": ["Window", "Arrange", "Left"]
+ })
+ },
+ "menu_action=window_arrange_left",
+ )
+ },
+ );
+}
+
+#[test]
+#[ignore]
+fn harness_gtk3_type_text_chars_waits_for_desktop_lane() {
+ run_case(
+ native_foreground_case(
+ "gtk3",
+ "type_text_chars_desktop_lane",
+ Targeting::Ax,
+ DriverRoute::LinuxAtSpiValue,
+ ),
+ |pid, window_id, driver| {
+ assert_waits_for_desktop_lane(
+ pid,
+ window_id,
+ driver,
+ "type_text_chars",
+ |state| {
+ serde_json::json!({
+ "pid": pid, "window_id": window_id,
+ "element_token": element_token(state, "txt-input"),
+ "text": "lane-chars", "delivery_mode": "foreground"
+ })
+ },
+ "mirror=lane-chars",
+ )
+ },
+ );
+}
+
#[derive(Clone, Copy, Debug)]
enum Operation {
AxClick {
diff --git a/libs/cua-driver/tests/fixtures/apps/linux/gtk3/main.py b/libs/cua-driver/tests/fixtures/apps/linux/gtk3/main.py
index fa4b5ad328..958700666d 100755
--- a/libs/cua-driver/tests/fixtures/apps/linux/gtk3/main.py
+++ b/libs/cua-driver/tests/fixtures/apps/linux/gtk3/main.py
@@ -51,6 +51,10 @@ class HarnessWindow(Gtk.Window):
self.hotkeys = 0
self._drag_modifier_seen = False
self.drag_events = 0
+ self._slider_held = False
+ self.slider_presses = 0
+ self.slider_releases = 0
+ self._effect_during_drag = "none"
# Top-level scroller so every control is reachable even on a short window.
scroller = Gtk.ScrolledWindow()
@@ -127,9 +131,17 @@ class HarnessWindow(Gtk.Window):
self.scale = aid(Gtk.Scale(orientation=Gtk.Orientation.HORIZONTAL, adjustment=adj), "sld-value")
self.scale.set_draw_value(False)
self.scale.connect("value-changed", self.on_scale)
+ self.scale.connect("button-press-event", self.on_scale_button, True)
+ self.scale.connect("button-release-event", self.on_scale_button, False)
root.pack_start(self.scale, False, False, 0)
self.scale_status = Gtk.Label(label="slider_value=0", xalign=0)
root.pack_start(self.scale_status, False, False, 0)
+ # Desktop-lane E2E: names an effect that landed while a slider drag
+ # still held the pointer button.
+ self.lane_status = Gtk.Label(
+ label="slider_presses=0 slider_releases=0 effect_during_drag=none", xalign=0
+ )
+ root.pack_start(self.lane_status, False, False, 0)
# ── checkable_controls ────────────────────────────────────────────
section(root, "checkable")
@@ -220,6 +232,7 @@ class HarnessWindow(Gtk.Window):
def on_entry_changed(self, entry):
self.mirror.set_text(f"mirror={entry.get_text()}")
+ self.note_effect_during_drag("text")
def on_click_target(self, *_):
self.clicks += 1
@@ -261,6 +274,27 @@ class HarnessWindow(Gtk.Window):
def on_scale(self, s):
self.scale_status.set_text(f"slider_value={int(s.get_value())}")
+ def on_scale_button(self, _w, ev, held):
+ if ev.button == 1:
+ self._slider_held = held
+ if held:
+ self.slider_presses += 1
+ else:
+ self.slider_releases += 1
+ self.publish_lane_status()
+ return False
+
+ def note_effect_during_drag(self, effect):
+ if self._slider_held:
+ self._effect_during_drag = effect
+ self.publish_lane_status()
+
+ def publish_lane_status(self):
+ self.lane_status.set_text(
+ f"slider_presses={self.slider_presses} slider_releases={self.slider_releases} "
+ f"effect_during_drag={self._effect_during_drag}"
+ )
+
def on_chk(self, c):
self.chk_status.set_text(f"agreed={c.get_active()}")
@@ -283,6 +317,7 @@ class HarnessWindow(Gtk.Window):
def on_native_menu_left(self, _w):
self._menu_action = "window_arrange_left"
self.ctx_status.set_text("menu_action=window_arrange_left")
+ self.note_effect_during_drag("menu")
def on_key_press(self, _w, ev):
key = (Gdk.keyval_name(ev.keyval) or "unknown").lower()