Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
fix(cua-driver): cover the desktop lane for invoke_menu and type_text_chars on Linux (#6653)
injaneity373346436ff9
diff --git a/libs/cua-driver/rust/crates/cua-driver-e2e/tests/harness_gtk3_test.rs b/libs/cua-driver/rust/crates/cua-driver-e2e/tests/harness_gtk3_test.rsindex 0c42a4ff7d..f9d2c87b67 100644--- a/libs/cua-driver/rust/crates/cua-driver-e2e/tests/harness_gtk3_test.rs+++ b/libs/cua-driver/rust/crates/cua-driver-e2e/tests/harness_gtk3_test.rs@@ -582,6 +582,137 @@ fn harness_gtk3_invoke_menu_live_path() {);}+/// Send `tool` while another client's drag holds the slider and the desktop+/// lane, and require that its effect lands only after the drag releases+/// (#6638). The fixture records an effect that arrives while its slider+/// button is still held.+fn assert_waits_for_desktop_lane(+ pid: u32,+ window_id: u64,+ driver: &mut McpDriver,+ tool: &str,+ args: impl FnOnce(&ToolResponse) -> serde_json::Value,+ expected: &str,+) -> Observation {+ let mut peer = driver+ .spawn_peer_unrecorded()+ .expect("start a second client on the same daemon");+ // Screenshot pixels are bound to the client that captured them.+ let peer_state = snapshot(&mut peer, pid, window_id);+ let (x, y, width, height) = element_rect(&mut peer, pid, window_id, &peer_state, "sld-value");+ let (drag, response) = std::thread::scope(|scope| {+ let drag = scope.spawn(move || {+ peer.call(+ "drag",+ serde_json::json!({+ "pid": pid as i64, "window_id": window_id,+ "from_x": x + width * 0.05, "from_y": y + height / 2.0,+ "to_x": x + width * 0.90, "to_y": y + height / 2.0,+ "duration_ms": 3000, "steps": 60, "delivery_mode": "foreground"+ }),+ )+ });+ // A read-only snapshot never waits for the lane, so the slider moving+ // proves the drag holds it now.+ let (state, slider) = loop {+ let state = snapshot(driver, pid, window_id);+ if let Some(slider) =+ state_number(state.tree_text(), "slider_value=").filter(|value| *value > 0)+ {+ break (state, slider);+ }+ assert!(+ !drag.is_finished(),+ "the peer's drag never moved the slider"+ );+ std::thread::sleep(Duration::from_millis(50));+ };+ // Over half of the drag (1.5 s) is left, so the call reaches the+ // daemon while the drag holds the lane unless this thread stalls.+ assert!(+ slider < 45 && !drag.is_finished(),+ "too little of the drag was left to send {tool} under it (slider_value={slider})"+ );+ let response = driver.call(tool, args(&state));+ (drag.join().expect("drag thread"), response)+ });+ assert!(!drag.is_error(), "peer drag failed: {}", drag.text());+ assert!(!response.is_error(), "{tool} failed: {}", response.text());+ let state = wait_for_state(driver, pid, window_id, expected);+ // Without exactly one observed press and release the marker below could+ // not see an overlap.+ assert!(+ state.contains("slider_presses=1 slider_releases=1 "),+ "the fixture did not observe the drag's press and release: {state}"+ );+ assert!(+ state.contains("effect_during_drag=none"),+ "{tool} took effect while the drag held the slider: it ran beside the drag instead of waiting for the desktop lane"+ );+ assert!(+ state_number(&state, "slider_value=").is_some_and(|value| value >= 80),+ "the drag did not reach its end: {state}"+ );+ Observation::delivered_with_fixture_state(Vec::new())+}++#[test]+#[ignore]+fn harness_gtk3_invoke_menu_waits_for_desktop_lane() {+ run_case(+ native_foreground_case(+ "gtk3",+ "invoke_menu_desktop_lane",+ Targeting::Ax,+ DriverRoute::LinuxAtSpiAction,+ ),+ |pid, window_id, driver| {+ assert_waits_for_desktop_lane(+ pid,+ window_id,+ driver,+ "invoke_menu",+ |_| {+ serde_json::json!({+ "pid": pid, "window_id": window_id,+ "path": ["Window", "Arrange", "Left"]+ })+ },+ "menu_action=window_arrange_left",+ )+ },+ );+}++#[test]+#[ignore]+fn harness_gtk3_type_text_chars_waits_for_desktop_lane() {+ run_case(+ native_foreground_case(+ "gtk3",+ "type_text_chars_desktop_lane",+ Targeting::Ax,+ DriverRoute::LinuxAtSpiValue,+ ),+ |pid, window_id, driver| {+ assert_waits_for_desktop_lane(+ pid,+ window_id,+ driver,+ "type_text_chars",+ |state| {+ serde_json::json!({+ "pid": pid, "window_id": window_id,+ "element_token": element_token(state, "txt-input"),+ "text": "lane-chars", "delivery_mode": "foreground"+ })+ },+ "mirror=lane-chars",+ )+ },+ );+}+#[derive(Clone, Copy, Debug)]enum Operation {AxClick {diff --git a/libs/cua-driver/tests/fixtures/apps/linux/gtk3/main.py b/libs/cua-driver/tests/fixtures/apps/linux/gtk3/main.pyindex fa4b5ad328..958700666d 100755--- a/libs/cua-driver/tests/fixtures/apps/linux/gtk3/main.py+++ b/libs/cua-driver/tests/fixtures/apps/linux/gtk3/main.py@@ -51,6 +51,10 @@ class HarnessWindow(Gtk.Window):self.hotkeys = 0self._drag_modifier_seen = Falseself.drag_events = 0+ self._slider_held = False+ self.slider_presses = 0+ self.slider_releases = 0+ self._effect_during_drag = "none"# Top-level scroller so every control is reachable even on a short window.scroller = Gtk.ScrolledWindow()@@ -127,9 +131,17 @@ class HarnessWindow(Gtk.Window):self.scale = aid(Gtk.Scale(orientation=Gtk.Orientation.HORIZONTAL, adjustment=adj), "sld-value")self.scale.set_draw_value(False)self.scale.connect("value-changed", self.on_scale)+ self.scale.connect("button-press-event", self.on_scale_button, True)+ self.scale.connect("button-release-event", self.on_scale_button, False)root.pack_start(self.scale, False, False, 0)self.scale_status = Gtk.Label(label="slider_value=0", xalign=0)root.pack_start(self.scale_status, False, False, 0)+ # Desktop-lane E2E: names an effect that landed while a slider drag+ # still held the pointer button.+ self.lane_status = Gtk.Label(+ label="slider_presses=0 slider_releases=0 effect_during_drag=none", xalign=0+ )+ root.pack_start(self.lane_status, False, False, 0)# ── checkable_controls ────────────────────────────────────────────section(root, "checkable")@@ -220,6 +232,7 @@ class HarnessWindow(Gtk.Window):def on_entry_changed(self, entry):self.mirror.set_text(f"mirror={entry.get_text()}")+ self.note_effect_during_drag("text")def on_click_target(self, *_):self.clicks += 1@@ -261,6 +274,27 @@ class HarnessWindow(Gtk.Window):def on_scale(self, s):self.scale_status.set_text(f"slider_value={int(s.get_value())}")+ def on_scale_button(self, _w, ev, held):+ if ev.button == 1:+ self._slider_held = held+ if held:+ self.slider_presses += 1+ else:+ self.slider_releases += 1+ self.publish_lane_status()+ return False++ def note_effect_during_drag(self, effect):+ if self._slider_held:+ self._effect_during_drag = effect+ self.publish_lane_status()++ def publish_lane_status(self):+ self.lane_status.set_text(+ f"slider_presses={self.slider_presses} slider_releases={self.slider_releases} "+ f"effect_during_drag={self._effect_during_drag}"+ )+def on_chk(self, c):self.chk_status.set_text(f"agreed={c.get_active()}")@@ -283,6 +317,7 @@ class HarnessWindow(Gtk.Window):def on_native_menu_left(self, _w):self._menu_action = "window_arrange_left"self.ctx_status.set_text("menu_action=window_arrange_left")+ self.note_effect_during_drag("menu")def on_key_press(self, _w, ev):key = (Gdk.keyval_name(ev.keyval) or "unknown").lower()