Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

fix(cua-driver): preserve Screen Sharing keyboard input (#2698)

Francesco Bonacci61b1a84c9ede
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/input/keyboard.rs b/libs/cua-driver/rust/crates/platform-macos/src/input/keyboard.rs
index 152071db6b..857ef2400f 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/input/keyboard.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/input/keyboard.rs
@@ -11,6 +11,25 @@ use core_graphics::{
};
use foreign_types::ForeignType;
+const SCREEN_SHARING_BUNDLE_ID: &str = "com.apple.ScreenSharing";
+const SHIFT_KEY_CODE: u16 = 56;
+
+fn is_screen_sharing_bundle_id(bundle_id: &str) -> bool {
+ bundle_id == SCREEN_SHARING_BUNDLE_ID
+}
+
+/// Whether `pid` is Apple's Screen Sharing client.
+///
+/// Screen Sharing is an input forwarder rather than a text consumer: it relays
+/// physical virtual-key transitions to the guest and ignores the Unicode string
+/// attached to a synthetic keycode-0 event. Keep that special case explicit so
+/// ordinary PID-routed text input retains its layout-independent Unicode path.
+pub fn is_screen_sharing_pid(pid: i32) -> bool {
+ crate::apps::bundle_id_for_pid(pid)
+ .as_deref()
+ .is_some_and(is_screen_sharing_bundle_id)
+}
+
/// Press and release a single key, delivered to `pid` without stealing focus.
pub fn press_key(pid: i32, key: &str, modifiers: &[&str]) -> anyhow::Result<()> {
// Handle "+" / "plus" → Shift+= (US keyboard layout).
@@ -264,6 +283,213 @@ pub fn type_text_global(text: &str, inter_char_delay_ms: u64) -> anyhow::Result<
Ok(())
}
+/// Type ASCII text through the global HID queue as physical US-keyboard
+/// transitions.
+///
+/// This is reserved for a caller that has guarded the exact target with
+/// `with_foreground_hid_activation`. Remote-input clients such as Screen
+/// Sharing forward virtual keycodes and modifier transitions, not the Unicode
+/// payload carried by keycode 0, so the ordinary text synthesis path cannot be
+/// used for them.
+pub fn type_text_physical_global(text: &str, inter_char_delay_ms: u64) -> anyhow::Result<()> {
+ use core_graphics::event::CGEventTapLocation;
+
+ // Validate the complete payload before posting its first event. A string
+ // containing an unsupported character must fail without partially typing.
+ let event_groups = text
+ .chars()
+ .map(physical_text_events)
+ .collect::<anyhow::Result<Vec<_>>>()?;
+ for events in event_groups {
+ let native_events = events
+ .iter()
+ .map(|event| create_bare_keyboard_event(event.key_code, event.key_down))
+ .collect::<anyhow::Result<Vec<_>>>()?;
+ for cg_event in native_events {
+ cg_event.post(CGEventTapLocation::HID);
+ std::thread::sleep(std::time::Duration::from_millis(8));
+ }
+ if inter_char_delay_ms > 8 {
+ std::thread::sleep(std::time::Duration::from_millis(inter_char_delay_ms - 8));
+ }
+ }
+ Ok(())
+}
+
+/// Send a physical key chord using the exact bare-event sequence documented by
+/// Apple for `CGEventCreateKeyboardEvent`: NULL source, modifier downs, base
+/// down/up, then modifier ups in reverse order. No flags, Unicode payload, or
+/// event-type overrides are applied; CoreGraphics derives those from the
+/// virtual key transitions and its default source state.
+pub fn press_key_bare_global(key: &str, modifiers: &[&str]) -> anyhow::Result<()> {
+ use core_graphics::event::CGEventTapLocation;
+
+ let key_code = key_name_to_code(key)?;
+ let mut modifier_codes = Vec::new();
+ for modifier in modifiers {
+ let Some((modifier_code, _)) = modifier_key_code_and_flag(modifier) else {
+ continue;
+ };
+ if !modifier_codes.contains(&modifier_code) {
+ modifier_codes.push(modifier_code);
+ }
+ }
+
+ let events = bare_chord_transitions(key_code, &modifier_codes)
+ .into_iter()
+ .map(|(code, down)| create_bare_keyboard_event(code, down))
+ .collect::<anyhow::Result<Vec<_>>>()?;
+ for event in events {
+ event.post(CGEventTapLocation::HID);
+ std::thread::sleep(std::time::Duration::from_millis(8));
+ }
+ Ok(())
+}
+
+fn bare_chord_transitions(key_code: u16, modifier_codes: &[u16]) -> Vec<(u16, bool)> {
+ let mut transitions = Vec::with_capacity(modifier_codes.len() * 2 + 2);
+ transitions.extend(modifier_codes.iter().map(|&code| (code, true)));
+ transitions.push((key_code, true));
+ transitions.push((key_code, false));
+ transitions.extend(modifier_codes.iter().rev().map(|&code| (code, false)));
+ transitions
+}
+
+fn create_bare_keyboard_event(key_code: u16, key_down: bool) -> anyhow::Result<CGEvent> {
+ unsafe {
+ let event_ref = CGEventCreateKeyboardEvent(std::ptr::null_mut(), key_code, key_down);
+ if event_ref.is_null() {
+ anyhow::bail!("CGEventCreateKeyboardEvent with default source failed");
+ }
+ Ok(CGEvent::from_ptr(event_ref))
+ }
+}
+
+#[link(name = "CoreGraphics", kind = "framework")]
+extern "C" {
+ fn CGEventCreateKeyboardEvent(
+ source: core_graphics::sys::CGEventSourceRef,
+ key_code: u16,
+ key_down: bool,
+ ) -> core_graphics::sys::CGEventRef;
+}
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+struct PhysicalTextEvent {
+ key_code: u16,
+ key_down: bool,
+ event_type: PhysicalEventType,
+ shift: bool,
+ text: Option<char>,
+}
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+enum PhysicalEventType {
+ KeyDown,
+ KeyUp,
+ FlagsChanged,
+}
+
+fn physical_text_events(ch: char) -> anyhow::Result<Vec<PhysicalTextEvent>> {
+ let (key_code, shift) = physical_key_for_char(ch).ok_or_else(|| {
+ anyhow::anyhow!(
+ "Screen Sharing physical text delivery does not support character U+{:04X}",
+ ch as u32
+ )
+ })?;
+ let mut events = Vec::with_capacity(if shift { 4 } else { 2 });
+ if shift {
+ events.push(PhysicalTextEvent {
+ key_code: SHIFT_KEY_CODE,
+ key_down: true,
+ event_type: PhysicalEventType::FlagsChanged,
+ shift: true,
+ text: None,
+ });
+ }
+ events.push(PhysicalTextEvent {
+ key_code,
+ key_down: true,
+ event_type: PhysicalEventType::KeyDown,
+ shift,
+ text: Some(ch),
+ });
+ events.push(PhysicalTextEvent {
+ key_code,
+ key_down: false,
+ event_type: PhysicalEventType::KeyUp,
+ shift,
+ text: Some(ch),
+ });
+ if shift {
+ events.push(PhysicalTextEvent {
+ key_code: SHIFT_KEY_CODE,
+ key_down: false,
+ event_type: PhysicalEventType::FlagsChanged,
+ shift: false,
+ text: None,
+ });
+ }
+ Ok(events)
+}
+
+/// Map printable ASCII to the physical key that produces it on the standard US
+/// layout. The Screen Sharing path intentionally sends only the returned
+/// keycode and the required bare Shift transitions.
+fn physical_key_for_char(ch: char) -> Option<(u16, bool)> {
+ let lower = ch.to_ascii_lowercase();
+ if ch.is_ascii_alphabetic() {
+ return key_name_to_code(&lower.to_string())
+ .ok()
+ .map(|code| (code, ch.is_ascii_uppercase()));
+ }
+ if ch.is_ascii_digit() {
+ return key_name_to_code(&ch.to_string())
+ .ok()
+ .map(|code| (code, false));
+ }
+
+ let (base, shift) = match ch {
+ ' ' => ("space", false),
+ '\t' => ("tab", false),
+ '\n' | '\r' => ("return", false),
+ '-' => ("-", false),
+ '_' => ("-", true),
+ '=' => ("=", false),
+ '+' => ("=", true),
+ '[' => ("[", false),
+ '{' => ("[", true),
+ ']' => ("]", false),
+ '}' => ("]", true),
+ '\\' => ("\\", false),
+ '|' => ("\\", true),
+ ';' => (";", false),
+ ':' => (";", true),
+ '\'' => ("'", false),
+ '"' => ("'", true),
+ ',' => (",", false),
+ '<' => (",", true),
+ '.' => (".", false),
+ '>' => (".", true),
+ '/' => ("/", false),
+ '?' => ("/", true),
+ '`' => ("`", false),
+ '~' => ("`", true),
+ '!' => ("1", true),
+ '@' => ("2", true),
+ '#' => ("3", true),
+ '$' => ("4", true),
+ '%' => ("5", true),
+ '^' => ("6", true),
+ '&' => ("7", true),
+ '*' => ("8", true),
+ '(' => ("9", true),
+ ')' => ("0", true),
+ _ => return None,
+ };
+ key_name_to_code(base).ok().map(|code| (code, shift))
+}
+
/// Post a keyboard event to `pid` via SLEventPostToPid (with auth message for
/// Chromium/Electron support) or fall back to CGEvent::post_to_pid.
pub(super) fn post_keyboard_event(pid: i32, event: &CGEvent) {
@@ -406,3 +632,156 @@ pub(super) fn key_name_to_code(key: &str) -> anyhow::Result<u16> {
};
Ok(code)
}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use core_graphics::event::CGEventType;
+
+ #[test]
+ fn physical_text_uses_flags_changed_for_balanced_shift_transitions() {
+ assert_eq!(
+ physical_text_events('a').unwrap(),
+ vec![
+ PhysicalTextEvent {
+ key_code: 0,
+ key_down: true,
+ event_type: PhysicalEventType::KeyDown,
+ shift: false,
+ text: Some('a'),
+ },
+ PhysicalTextEvent {
+ key_code: 0,
+ key_down: false,
+ event_type: PhysicalEventType::KeyUp,
+ shift: false,
+ text: Some('a'),
+ },
+ ]
+ );
+ assert_eq!(
+ physical_text_events('Z').unwrap(),
+ vec![
+ PhysicalTextEvent {
+ key_code: 56,
+ key_down: true,
+ event_type: PhysicalEventType::FlagsChanged,
+ shift: true,
+ text: None,
+ },
+ PhysicalTextEvent {
+ key_code: 6,
+ key_down: true,
+ event_type: PhysicalEventType::KeyDown,
+ shift: true,
+ text: Some('Z'),
+ },
+ PhysicalTextEvent {
+ key_code: 6,
+ key_down: false,
+ event_type: PhysicalEventType::KeyUp,
+ shift: true,
+ text: Some('Z'),
+ },
+ PhysicalTextEvent {
+ key_code: 56,
+ key_down: false,
+ event_type: PhysicalEventType::FlagsChanged,
+ shift: false,
+ text: None,
+ },
+ ]
+ );
+ assert_eq!(physical_key_for_char('1'), Some((18, false)));
+ assert_eq!(physical_key_for_char('!'), Some((18, true)));
+ assert_eq!(physical_key_for_char('/'), Some((44, false)));
+ assert_eq!(physical_key_for_char('?'), Some((44, true)));
+ }
+
+ #[test]
+ fn bare_modifier_events_derive_type_and_flags_from_default_source() {
+ let shift_down = create_bare_keyboard_event(SHIFT_KEY_CODE, true).unwrap();
+ assert_eq!(
+ shift_down.get_type() as u32,
+ CGEventType::FlagsChanged as u32
+ );
+ assert!(
+ shift_down
+ .get_flags()
+ .contains(CGEventFlags::CGEventFlagShift),
+ "bare Shift down must derive the active Shift flag"
+ );
+ assert_eq!(
+ shift_down
+ .get_integer_value_field(core_graphics::event::EventField::EVENT_SOURCE_STATE_ID),
+ CGEventSourceStateID::CombinedSessionState as i64,
+ "NULL source must resolve to the default combined-session state, not HID state"
+ );
+
+ let shift_up = create_bare_keyboard_event(SHIFT_KEY_CODE, false).unwrap();
+ assert_eq!(shift_up.get_type() as u32, CGEventType::FlagsChanged as u32);
+ assert!(
+ !shift_up
+ .get_flags()
+ .contains(CGEventFlags::CGEventFlagShift),
+ "bare Shift up must derive cleared Shift state"
+ );
+
+ let z_down = create_bare_keyboard_event(6, true).unwrap();
+ assert_eq!(z_down.get_type() as u32, CGEventType::KeyDown as u32);
+ assert_eq!(
+ z_down
+ .get_integer_value_field(core_graphics::event::EventField::KEYBOARD_EVENT_KEYCODE),
+ 6
+ );
+ }
+
+ #[test]
+ fn bare_command_chord_orders_modifier_base_and_reverse_release() {
+ assert_eq!(
+ bare_chord_transitions(9, &[55]),
+ vec![(55, true), (9, true), (9, false), (55, false)]
+ );
+ let command_down = create_bare_keyboard_event(55, true).unwrap();
+ assert_eq!(
+ command_down.get_type() as u32,
+ CGEventType::FlagsChanged as u32
+ );
+ assert!(
+ command_down
+ .get_flags()
+ .contains(CGEventFlags::CGEventFlagCommand),
+ "bare Command down must derive the active Command flag"
+ );
+ }
+
+ #[test]
+ fn physical_text_rejects_characters_without_a_lossless_keycode() {
+ let error = physical_text_events('🙂').unwrap_err().to_string();
+ assert!(error.contains("U+1F642"), "{error}");
+ }
+
+ #[test]
+ fn physical_text_covers_printable_ascii_and_common_text_whitespace() {
+ for byte in 0x20_u8..=0x7e {
+ let ch = char::from(byte);
+ assert!(
+ physical_key_for_char(ch).is_some(),
+ "missing physical key for ASCII {ch:?}"
+ );
+ }
+ for ch in ['\t', '\n', '\r'] {
+ assert!(
+ physical_key_for_char(ch).is_some(),
+ "missing physical key for whitespace {ch:?}"
+ );
+ }
+ }
+
+ #[test]
+ fn screen_sharing_detection_is_exact_and_case_sensitive() {
+ assert!(is_screen_sharing_bundle_id("com.apple.ScreenSharing"));
+ assert!(!is_screen_sharing_bundle_id("com.apple.screensharing"));
+ assert!(!is_screen_sharing_bundle_id("com.microsoft.rdc.macos"));
+ }
+}
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/hotkey.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/hotkey.rs
index 1f74f3c63a..fa1b4474b3 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/hotkey.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/hotkey.rs
@@ -93,6 +93,33 @@ fn is_modifier(k: &str) -> bool {
)
}
+fn screen_sharing_modifier_delivery_error(
+ is_screen_sharing: bool,
+ has_modifiers: bool,
+ foreground: bool,
+ window_id: Option<u32>,
+) -> Option<ToolResult> {
+ if !is_screen_sharing || !has_modifiers || (foreground && window_id.is_some()) {
+ return None;
+ }
+ Some(
+ ToolResult::error(
+ "Screen Sharing modifier hotkeys require delivery_mode:\"foreground\" and window_id \
+ so Cua Driver can deliver physical modifier transitions safely.",
+ )
+ .with_structured(serde_json::json!({
+ "code": "SCREEN_SHARING_REQUIRES_FOREGROUND_HID",
+ "effect": "refused",
+ "escalation": {
+ "recommended": "foreground",
+ "reason": "Screen Sharing does not forward modifier state from background \
+ PID-routed base-key events.",
+ "requires": ["window_id"]
+ }
+ })),
+ )
+}
+
#[async_trait]
impl Tool for HotkeyTool {
fn def(&self) -> &ToolDef {
@@ -187,6 +214,15 @@ impl Tool for HotkeyTool {
// background combo (matches click/type_text).
let delivery_mode = super::DeliveryMode::parse(args.opt_str("delivery_mode").as_deref());
let fg = delivery_mode.is_foreground();
+ let screen_sharing_target = crate::input::keyboard::is_screen_sharing_pid(pid);
+ if let Some(error) = screen_sharing_modifier_delivery_error(
+ screen_sharing_target,
+ !modifiers.is_empty(),
+ fg,
+ window_id,
+ ) {
+ return error;
+ }
// px form: focus the field before sending the combo. Foreground delivery
// still needs to front the target for the chord itself: the focus helper
@@ -245,7 +281,27 @@ impl Tool for HotkeyTool {
crate::input::skylight::with_foreground_hid_activation(
pid as libc::pid_t,
wid,
- || crate::input::keyboard::press_key_global(&key, &m),
+ || {
+ if screen_sharing_target {
+ crate::input::keyboard::press_key_bare_global(&key, &m)
+ } else {
+ crate::input::keyboard::press_key_global(&key, &m)
+ }
+ },
+ )?;
+ Ok(())
+ }
+ // Screen Sharing is an input forwarder: modifier flags
+ // on a PID-routed base-key event are not relayed to the
+ // guest. Emit the physical modifier down/base/up
+ // sequence through the guarded foreground HID path.
+ (true, false, Some(wid))
+ if crate::input::keyboard::is_screen_sharing_pid(pid) =>
+ {
+ crate::input::skylight::with_foreground_hid_activation(
+ pid as libc::pid_t,
+ wid,
+ || crate::input::keyboard::press_key_bare_global(&key, &m),
)?;
Ok(())
}
@@ -306,3 +362,25 @@ impl Tool for HotkeyTool {
}
}
}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn screen_sharing_modifier_hotkeys_fail_closed_without_foreground_window() {
+ for (foreground, window_id) in [(false, None), (false, Some(7)), (true, None)] {
+ let result = screen_sharing_modifier_delivery_error(true, true, foreground, window_id)
+ .expect("unsafe Screen Sharing modifier route must be refused");
+ assert_eq!(result.is_error, Some(true));
+ let structured = result.structured_content.unwrap();
+ assert_eq!(structured["code"], "SCREEN_SHARING_REQUIRES_FOREGROUND_HID");
+ assert_eq!(structured["effect"], "refused");
+ assert_eq!(structured["escalation"]["recommended"], "foreground");
+ assert_eq!(structured["escalation"]["requires"][0], "window_id");
+ }
+ assert!(screen_sharing_modifier_delivery_error(true, true, true, Some(7)).is_none());
+ assert!(screen_sharing_modifier_delivery_error(true, false, false, None).is_none());
+ assert!(screen_sharing_modifier_delivery_error(false, true, false, None).is_none());
+ }
+}
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/type_text.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/type_text.rs
index d74f5df922..b659d622af 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/type_text.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/type_text.rs
@@ -125,6 +125,32 @@ fn def() -> &'static ToolDef {
})
}
+fn screen_sharing_delivery_error(
+ is_screen_sharing: bool,
+ foreground: bool,
+ window_id: Option<u32>,
+) -> Option<ToolResult> {
+ if !is_screen_sharing || (foreground && window_id.is_some()) {
+ return None;
+ }
+ Some(
+ ToolResult::error(
+ "Screen Sharing text input requires delivery_mode:\"foreground\" and window_id \
+ so Cua Driver can deliver physical HID key transitions safely.",
+ )
+ .with_structured(serde_json::json!({
+ "code": "SCREEN_SHARING_REQUIRES_FOREGROUND_HID",
+ "effect": "refused",
+ "escalation": {
+ "recommended": "foreground",
+ "reason": "Screen Sharing forwards physical keycodes; background PID-routed \
+ Unicode events can corrupt guest text.",
+ "requires": ["window_id"]
+ }
+ })),
+ )
+}
+
#[async_trait]
impl Tool for TypeTextTool {
fn def(&self) -> &ToolDef {
@@ -198,6 +224,13 @@ impl Tool for TypeTextTool {
};
let delay_ms = args.u64_or("delay_ms", 30);
let delivery_mode = super::DeliveryMode::parse(args.opt_str("delivery_mode").as_deref());
+ if let Some(error) = screen_sharing_delivery_error(
+ crate::input::keyboard::is_screen_sharing_pid(pid),
+ delivery_mode.is_foreground(),
+ window_id,
+ ) {
+ return error;
+ }
// ── px form: focus by pixel-click, then type into the focused element ──
// Pass x,y (no element_index) for an *element px action*: pixel-click the
@@ -768,6 +801,7 @@ fn type_text_blocking(
// dropped. 200ms covers that re-grab without penalizing an already
// armed interactive stream on every text chunk.
let foreground_settle_ms = foreground_settle_ms(pid, apps::frontmost_pid());
+ let screen_sharing_target = crate::input::keyboard::is_screen_sharing_pid(pid);
let do_type = || {
cgevent_type_verified(
pid,
@@ -779,6 +813,27 @@ fn type_text_blocking(
)
};
let ((verified, delivered_chars), fronted) = match window_id {
+ Some(wid) if screen_sharing_target => {
+ // Screen Sharing forwards physical HID transitions to the
+ // guest. PID-routed Unicode events all carry keycode 0 (the A
+ // key), so a guest sees "aaaa"; modifier flags alone likewise
+ // turn Cmd+V into plain "v". The explicit foreground rung may
+ // safely use the global HID queue while the exact target is
+ // guarded and restored.
+ crate::input::skylight::with_foreground_hid_activation(
+ pid as libc::pid_t,
+ wid,
+ || {
+ if foreground_settle_ms > 0 {
+ std::thread::sleep(std::time::Duration::from_millis(
+ foreground_settle_ms,
+ ));
+ }
+ crate::input::keyboard::type_text_physical_global(text, delay_ms)
+ },
+ )?;
+ ((false, None), true)
+ }
Some(wid) => {
// Front → type → restore. The closure returns the read-back
// result; with_foreground_assist returns whether it actually
@@ -1065,4 +1120,20 @@ mod tests {
assert_eq!(foreground_settle_ms(42, Some(7)), 200);
assert_eq!(foreground_settle_ms(42, None), 200);
}
+
+ #[test]
+ fn screen_sharing_text_fails_closed_without_foreground_window() {
+ for (foreground, window_id) in [(false, None), (false, Some(7)), (true, None)] {
+ let result = screen_sharing_delivery_error(true, foreground, window_id)
+ .expect("unsafe Screen Sharing route must be refused");
+ assert_eq!(result.is_error, Some(true));
+ let structured = result.structured_content.unwrap();
+ assert_eq!(structured["code"], "SCREEN_SHARING_REQUIRES_FOREGROUND_HID");
+ assert_eq!(structured["effect"], "refused");
+ assert_eq!(structured["escalation"]["recommended"], "foreground");
+ assert_eq!(structured["escalation"]["requires"][0], "window_id");
+ }
+ assert!(screen_sharing_delivery_error(true, true, Some(7)).is_none());
+ assert!(screen_sharing_delivery_error(false, false, None).is_none());
+ }
}