Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
fix(cua-driver): keep a foreground key chord's modifiers on its base key (#3855)
Will Boguszf71044a940b8
diff --git a/libs/cua-driver/rust/crates/cua-driver-e2e/tests/harness_appkit_test.rs b/libs/cua-driver/rust/crates/cua-driver-e2e/tests/harness_appkit_test.rsindex 82c6267b71..ca3e7c992d 100644--- a/libs/cua-driver/rust/crates/cua-driver-e2e/tests/harness_appkit_test.rs+++ b/libs/cua-driver/rust/crates/cua-driver-e2e/tests/harness_appkit_test.rs@@ -884,6 +884,58 @@ fn harness_appkit_element_foreground_press_key_commits_edit() {);}+/// The fixture's accelerator fires only when control and shift are set on the+/// `k` keyDown itself, so a foreground chord that loses them leaves it at 0.+#[test]+#[ignore]+fn harness_appkit_foreground_press_key_chord_carries_its_modifiers() {+ run_case(+ native_foreground_case(+ "appkit",+ "press_key_chord",+ Targeting::Ax,+ DriverRoute::MacosCgEventHid,+ ),+ |pid, wid, driver| {+ let before = snapshot_elements(driver, pid, wid);+ assert!(+ before.tree_text().contains("accel_fired=0"),+ "fixture did not start with an unfired accelerator:\n{}",+ before.tree_text()+ );+ let chord = driver.call(+ "press_key",+ serde_json::json!({+ "pid": pid as i64,+ "window_id": wid,+ "key": "k",+ "modifiers": ["ctrl", "shift"],+ "delivery_mode": "foreground"+ }),+ );+ assert!(+ !chord.is_error(),+ "foreground press_key chord failed: {}",+ chord.text()+ );+ let deadline = std::time::Instant::now() + Duration::from_secs(2);+ loop {+ let after = snapshot_elements(driver, pid, wid);+ if after.tree_text().contains("accel_fired=1") {+ break;+ }+ assert!(+ std::time::Instant::now() < deadline,+ "ctrl+shift+k arrived without its modifiers:\n{}",+ after.tree_text()+ );+ std::thread::sleep(Duration::from_millis(100));+ }+ Observation::delivered_with_fixture_state(Vec::new())+ },+ );+}+#[test]#[ignore]fn harness_appkit_px_background_press_key_reports_honest_delivery_truth() {diff --git a/libs/cua-driver/rust/crates/platform-macos/src/input/keyboard.rs b/libs/cua-driver/rust/crates/platform-macos/src/input/keyboard.rsindex 2b9baf8e55..dca9973e37 100644--- a/libs/cua-driver/rust/crates/platform-macos/src/input/keyboard.rs+++ b/libs/cua-driver/rust/crates/platform-macos/src/input/keyboard.rs@@ -414,28 +414,38 @@ pub fn type_text_physical_global(text: &str, inter_char_delay_ms: u64) -> anyhowOk(())}-/// Send a physical key chord using the exact bare-event sequence documented by-/// Apple for `CGEventCreateKeyboardEvent`: NULL source, modifier downs, base-/// down/up, then modifier ups in reverse order. No flags, Unicode payload, or-/// event-type overrides are applied; CoreGraphics derives those from the-/// virtual key transitions and its default source state.+/// Send a physical key chord as the virtual-key transition sequence Apple+/// documents for `CGEventCreateKeyboardEvent`: NULL source, modifier downs,+/// base down/up, then modifier ups in reverse order.+///+/// Each transition also carries the chord's accumulated `CGEventFlags`, as the+/// PID-routed and global rungs do. A keyboard event created from the default+/// source starts with no flags, and posting a modifier keycode does not+/// retro-fit them onto events that were already created, so a chord whose base+/// key carries no flags arrives as the bare key: `cmd+a` inserts a literal `a`.+/// Remote-input clients such as Screen Sharing re-derive modifiers from the+/// keycode transitions and ignore the flags, so both consumers are served.pub fn press_key_bare_global(key: &str, modifiers: &[&str]) -> anyhow::Result<()> {use core_graphics::event::CGEventTapLocation;let key_code = key_name_to_code(key)?;- let mut modifier_codes = Vec::new();+ let mut modifier_keys = Vec::new();for modifier in modifiers {- let Some((modifier_code, _)) = modifier_key_code_and_flag(modifier) else {+ let Some((modifier_code, flag)) = modifier_key_code_and_flag(modifier) else {continue;};- if !modifier_codes.contains(&modifier_code) {- modifier_codes.push(modifier_code);+ if !modifier_keys.iter().any(|&(code, _)| code == modifier_code) {+ modifier_keys.push((modifier_code, flag));}}- let events = bare_chord_transitions(key_code, &modifier_codes)+ let events = bare_chord_transitions(key_code, &modifier_keys).into_iter()- .map(|(code, down)| create_bare_keyboard_event(code, down))+ .map(|(code, down, flags)| {+ let event = create_bare_keyboard_event(code, down)?;+ event.set_flags(flags);+ Ok(event)+ }).collect::<anyhow::Result<Vec<_>>>()?;for event in events {event.post(CGEventTapLocation::HID);@@ -444,12 +454,25 @@ pub fn press_key_bare_global(key: &str, modifiers: &[&str]) -> anyhow::Result<()Ok(())}-fn bare_chord_transitions(key_code: u16, modifier_codes: &[u16]) -> Vec<(u16, bool)> {- let mut transitions = Vec::with_capacity(modifier_codes.len() * 2 + 2);- transitions.extend(modifier_codes.iter().map(|&code| (code, true)));- transitions.push((key_code, true));- transitions.push((key_code, false));- transitions.extend(modifier_codes.iter().rev().map(|&code| (code, false)));+/// The chord's transitions with the flags each one carries. A modifier's own+/// down is the first event that holds its flag and its up is the first that has+/// dropped it again, so the base key always sees the full chord.+fn bare_chord_transitions(+ key_code: u16,+ modifier_keys: &[(u16, CGEventFlags)],+) -> Vec<(u16, bool, CGEventFlags)> {+ let mut transitions = Vec::with_capacity(modifier_keys.len() * 2 + 2);+ let mut flags = CGEventFlags::CGEventFlagNull;+ let mut releases = Vec::with_capacity(modifier_keys.len());+ for &(code, flag) in modifier_keys {+ let previous = flags;+ flags |= flag;+ transitions.push((code, true, flags));+ releases.push((code, false, previous));+ }+ transitions.push((key_code, true, flags));+ transitions.push((key_code, false, flags));+ transitions.extend(releases.into_iter().rev());transitions}@@ -836,10 +859,23 @@ mod tests {#[test]fn bare_command_chord_orders_modifier_base_and_reverse_release() {+ let command = (55, CGEventFlags::CGEventFlagCommand);assert_eq!(- bare_chord_transitions(9, &[55]),- vec![(55, true), (9, true), (9, false), (55, false)]+ bare_chord_transitions(9, &[command]),+ vec![+ (55, true, CGEventFlags::CGEventFlagCommand),+ (9, true, CGEventFlags::CGEventFlagCommand),+ (9, false, CGEventFlags::CGEventFlagCommand),+ (55, false, CGEventFlags::CGEventFlagNull),+ ]);+ }++ /// The default source derives a modifier's own flag from its keycode but+ /// has nothing to derive from for the base key, which is what turned+ /// `cmd+a` into a literal `a` on the foreground rung.+ #[test]+ fn bare_base_key_carries_the_chord_flags_the_default_source_cannot_derive() {let command_down = create_bare_keyboard_event(55, true).unwrap();assert_eq!(command_down.get_type() as u32,@@ -851,6 +887,25 @@ mod tests {.contains(CGEventFlags::CGEventFlagCommand),"bare Command down must derive the active Command flag");++ let bare_base = create_bare_keyboard_event(0, true).unwrap();+ assert!(+ !bare_base+ .get_flags()+ .contains(CGEventFlags::CGEventFlagCommand),+ "the default source cannot derive Command for a non-modifier keycode"+ );++ let (code, down, flags) =+ bare_chord_transitions(0, &[(55, CGEventFlags::CGEventFlagCommand)])[1];+ let carried = create_bare_keyboard_event(code, down).unwrap();+ carried.set_flags(flags);+ assert!(+ carried+ .get_flags()+ .contains(CGEventFlags::CGEventFlagCommand),+ "the chord's base key must carry Command"+ );}#[test]diff --git a/scripts/ci/macos/run-rust-e2e.sh b/scripts/ci/macos/run-rust-e2e.shindex 0ab5b0d6c2..19006f5c33 100755--- a/scripts/ci/macos/run-rust-e2e.sh+++ b/scripts/ci/macos/run-rust-e2e.sh@@ -350,6 +350,7 @@ if [[ "${SUITE}" == native || "${SUITE}" == all ]]; thenharness_appkit_invoke_menu_live_path \harness_appkit_text_input \harness_appkit_element_foreground_press_key_commits_edit \+ harness_appkit_foreground_press_key_chord_carries_its_modifiers \harness_appkit_modified_click_preserves_selection \harness_appkit_type_text_background \harness_appkit_scroll_foreground \