Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

fix(cua-driver): keep a foreground key chord's modifiers on its base key (#3855)

Will Boguszf71044a940b8
diff --git a/libs/cua-driver/rust/crates/cua-driver-e2e/tests/harness_appkit_test.rs b/libs/cua-driver/rust/crates/cua-driver-e2e/tests/harness_appkit_test.rs
index 82c6267b71..ca3e7c992d 100644
--- a/libs/cua-driver/rust/crates/cua-driver-e2e/tests/harness_appkit_test.rs
+++ b/libs/cua-driver/rust/crates/cua-driver-e2e/tests/harness_appkit_test.rs
@@ -884,6 +884,58 @@ fn harness_appkit_element_foreground_press_key_commits_edit() {
);
}
+/// The fixture's accelerator fires only when control and shift are set on the
+/// `k` keyDown itself, so a foreground chord that loses them leaves it at 0.
+#[test]
+#[ignore]
+fn harness_appkit_foreground_press_key_chord_carries_its_modifiers() {
+ run_case(
+ native_foreground_case(
+ "appkit",
+ "press_key_chord",
+ Targeting::Ax,
+ DriverRoute::MacosCgEventHid,
+ ),
+ |pid, wid, driver| {
+ let before = snapshot_elements(driver, pid, wid);
+ assert!(
+ before.tree_text().contains("accel_fired=0"),
+ "fixture did not start with an unfired accelerator:\n{}",
+ before.tree_text()
+ );
+ let chord = driver.call(
+ "press_key",
+ serde_json::json!({
+ "pid": pid as i64,
+ "window_id": wid,
+ "key": "k",
+ "modifiers": ["ctrl", "shift"],
+ "delivery_mode": "foreground"
+ }),
+ );
+ assert!(
+ !chord.is_error(),
+ "foreground press_key chord failed: {}",
+ chord.text()
+ );
+ let deadline = std::time::Instant::now() + Duration::from_secs(2);
+ loop {
+ let after = snapshot_elements(driver, pid, wid);
+ if after.tree_text().contains("accel_fired=1") {
+ break;
+ }
+ assert!(
+ std::time::Instant::now() < deadline,
+ "ctrl+shift+k arrived without its modifiers:\n{}",
+ after.tree_text()
+ );
+ std::thread::sleep(Duration::from_millis(100));
+ }
+ Observation::delivered_with_fixture_state(Vec::new())
+ },
+ );
+}
+
#[test]
#[ignore]
fn harness_appkit_px_background_press_key_reports_honest_delivery_truth() {
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/input/keyboard.rs b/libs/cua-driver/rust/crates/platform-macos/src/input/keyboard.rs
index 2b9baf8e55..dca9973e37 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/input/keyboard.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/input/keyboard.rs
@@ -414,28 +414,38 @@ pub fn type_text_physical_global(text: &str, inter_char_delay_ms: u64) -> anyhow
Ok(())
}
-/// Send a physical key chord using the exact bare-event sequence documented by
-/// Apple for `CGEventCreateKeyboardEvent`: NULL source, modifier downs, base
-/// down/up, then modifier ups in reverse order. No flags, Unicode payload, or
-/// event-type overrides are applied; CoreGraphics derives those from the
-/// virtual key transitions and its default source state.
+/// Send a physical key chord as the virtual-key transition sequence Apple
+/// documents for `CGEventCreateKeyboardEvent`: NULL source, modifier downs,
+/// base down/up, then modifier ups in reverse order.
+///
+/// Each transition also carries the chord's accumulated `CGEventFlags`, as the
+/// PID-routed and global rungs do. A keyboard event created from the default
+/// source starts with no flags, and posting a modifier keycode does not
+/// retro-fit them onto events that were already created, so a chord whose base
+/// key carries no flags arrives as the bare key: `cmd+a` inserts a literal `a`.
+/// Remote-input clients such as Screen Sharing re-derive modifiers from the
+/// keycode transitions and ignore the flags, so both consumers are served.
pub fn press_key_bare_global(key: &str, modifiers: &[&str]) -> anyhow::Result<()> {
use core_graphics::event::CGEventTapLocation;
let key_code = key_name_to_code(key)?;
- let mut modifier_codes = Vec::new();
+ let mut modifier_keys = Vec::new();
for modifier in modifiers {
- let Some((modifier_code, _)) = modifier_key_code_and_flag(modifier) else {
+ let Some((modifier_code, flag)) = modifier_key_code_and_flag(modifier) else {
continue;
};
- if !modifier_codes.contains(&modifier_code) {
- modifier_codes.push(modifier_code);
+ if !modifier_keys.iter().any(|&(code, _)| code == modifier_code) {
+ modifier_keys.push((modifier_code, flag));
}
}
- let events = bare_chord_transitions(key_code, &modifier_codes)
+ let events = bare_chord_transitions(key_code, &modifier_keys)
.into_iter()
- .map(|(code, down)| create_bare_keyboard_event(code, down))
+ .map(|(code, down, flags)| {
+ let event = create_bare_keyboard_event(code, down)?;
+ event.set_flags(flags);
+ Ok(event)
+ })
.collect::<anyhow::Result<Vec<_>>>()?;
for event in events {
event.post(CGEventTapLocation::HID);
@@ -444,12 +454,25 @@ pub fn press_key_bare_global(key: &str, modifiers: &[&str]) -> anyhow::Result<()
Ok(())
}
-fn bare_chord_transitions(key_code: u16, modifier_codes: &[u16]) -> Vec<(u16, bool)> {
- let mut transitions = Vec::with_capacity(modifier_codes.len() * 2 + 2);
- transitions.extend(modifier_codes.iter().map(|&code| (code, true)));
- transitions.push((key_code, true));
- transitions.push((key_code, false));
- transitions.extend(modifier_codes.iter().rev().map(|&code| (code, false)));
+/// The chord's transitions with the flags each one carries. A modifier's own
+/// down is the first event that holds its flag and its up is the first that has
+/// dropped it again, so the base key always sees the full chord.
+fn bare_chord_transitions(
+ key_code: u16,
+ modifier_keys: &[(u16, CGEventFlags)],
+) -> Vec<(u16, bool, CGEventFlags)> {
+ let mut transitions = Vec::with_capacity(modifier_keys.len() * 2 + 2);
+ let mut flags = CGEventFlags::CGEventFlagNull;
+ let mut releases = Vec::with_capacity(modifier_keys.len());
+ for &(code, flag) in modifier_keys {
+ let previous = flags;
+ flags |= flag;
+ transitions.push((code, true, flags));
+ releases.push((code, false, previous));
+ }
+ transitions.push((key_code, true, flags));
+ transitions.push((key_code, false, flags));
+ transitions.extend(releases.into_iter().rev());
transitions
}
@@ -836,10 +859,23 @@ mod tests {
#[test]
fn bare_command_chord_orders_modifier_base_and_reverse_release() {
+ let command = (55, CGEventFlags::CGEventFlagCommand);
assert_eq!(
- bare_chord_transitions(9, &[55]),
- vec![(55, true), (9, true), (9, false), (55, false)]
+ bare_chord_transitions(9, &[command]),
+ vec![
+ (55, true, CGEventFlags::CGEventFlagCommand),
+ (9, true, CGEventFlags::CGEventFlagCommand),
+ (9, false, CGEventFlags::CGEventFlagCommand),
+ (55, false, CGEventFlags::CGEventFlagNull),
+ ]
);
+ }
+
+ /// The default source derives a modifier's own flag from its keycode but
+ /// has nothing to derive from for the base key, which is what turned
+ /// `cmd+a` into a literal `a` on the foreground rung.
+ #[test]
+ fn bare_base_key_carries_the_chord_flags_the_default_source_cannot_derive() {
let command_down = create_bare_keyboard_event(55, true).unwrap();
assert_eq!(
command_down.get_type() as u32,
@@ -851,6 +887,25 @@ mod tests {
.contains(CGEventFlags::CGEventFlagCommand),
"bare Command down must derive the active Command flag"
);
+
+ let bare_base = create_bare_keyboard_event(0, true).unwrap();
+ assert!(
+ !bare_base
+ .get_flags()
+ .contains(CGEventFlags::CGEventFlagCommand),
+ "the default source cannot derive Command for a non-modifier keycode"
+ );
+
+ let (code, down, flags) =
+ bare_chord_transitions(0, &[(55, CGEventFlags::CGEventFlagCommand)])[1];
+ let carried = create_bare_keyboard_event(code, down).unwrap();
+ carried.set_flags(flags);
+ assert!(
+ carried
+ .get_flags()
+ .contains(CGEventFlags::CGEventFlagCommand),
+ "the chord's base key must carry Command"
+ );
}
#[test]
diff --git a/scripts/ci/macos/run-rust-e2e.sh b/scripts/ci/macos/run-rust-e2e.sh
index 0ab5b0d6c2..19006f5c33 100755
--- a/scripts/ci/macos/run-rust-e2e.sh
+++ b/scripts/ci/macos/run-rust-e2e.sh
@@ -350,6 +350,7 @@ if [[ "${SUITE}" == native || "${SUITE}" == all ]]; then
harness_appkit_invoke_menu_live_path \
harness_appkit_text_input \
harness_appkit_element_foreground_press_key_commits_edit \
+ harness_appkit_foreground_press_key_chord_carries_its_modifiers \
harness_appkit_modified_click_preserves_selection \
harness_appkit_type_text_background \
harness_appkit_scroll_foreground \