Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
fix(cua-driver): preserve Screen Sharing keyboard input (#2698)
Francesco Bonacci61b1a84c9ede
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/input/keyboard.rs b/libs/cua-driver/rust/crates/platform-macos/src/input/keyboard.rsindex 152071db6b..857ef2400f 100644--- a/libs/cua-driver/rust/crates/platform-macos/src/input/keyboard.rs+++ b/libs/cua-driver/rust/crates/platform-macos/src/input/keyboard.rs@@ -11,6 +11,25 @@ use core_graphics::{};use foreign_types::ForeignType;+const SCREEN_SHARING_BUNDLE_ID: &str = "com.apple.ScreenSharing";+const SHIFT_KEY_CODE: u16 = 56;++fn is_screen_sharing_bundle_id(bundle_id: &str) -> bool {+ bundle_id == SCREEN_SHARING_BUNDLE_ID+}++/// Whether `pid` is Apple's Screen Sharing client.+///+/// Screen Sharing is an input forwarder rather than a text consumer: it relays+/// physical virtual-key transitions to the guest and ignores the Unicode string+/// attached to a synthetic keycode-0 event. Keep that special case explicit so+/// ordinary PID-routed text input retains its layout-independent Unicode path.+pub fn is_screen_sharing_pid(pid: i32) -> bool {+ crate::apps::bundle_id_for_pid(pid)+ .as_deref()+ .is_some_and(is_screen_sharing_bundle_id)+}+/// Press and release a single key, delivered to `pid` without stealing focus.pub fn press_key(pid: i32, key: &str, modifiers: &[&str]) -> anyhow::Result<()> {// Handle "+" / "plus" → Shift+= (US keyboard layout).@@ -264,6 +283,213 @@ pub fn type_text_global(text: &str, inter_char_delay_ms: u64) -> anyhow::Result<Ok(())}+/// Type ASCII text through the global HID queue as physical US-keyboard+/// transitions.+///+/// This is reserved for a caller that has guarded the exact target with+/// `with_foreground_hid_activation`. Remote-input clients such as Screen+/// Sharing forward virtual keycodes and modifier transitions, not the Unicode+/// payload carried by keycode 0, so the ordinary text synthesis path cannot be+/// used for them.+pub fn type_text_physical_global(text: &str, inter_char_delay_ms: u64) -> anyhow::Result<()> {+ use core_graphics::event::CGEventTapLocation;++ // Validate the complete payload before posting its first event. A string+ // containing an unsupported character must fail without partially typing.+ let event_groups = text+ .chars()+ .map(physical_text_events)+ .collect::<anyhow::Result<Vec<_>>>()?;+ for events in event_groups {+ let native_events = events+ .iter()+ .map(|event| create_bare_keyboard_event(event.key_code, event.key_down))+ .collect::<anyhow::Result<Vec<_>>>()?;+ for cg_event in native_events {+ cg_event.post(CGEventTapLocation::HID);+ std::thread::sleep(std::time::Duration::from_millis(8));+ }+ if inter_char_delay_ms > 8 {+ std::thread::sleep(std::time::Duration::from_millis(inter_char_delay_ms - 8));+ }+ }+ Ok(())+}++/// Send a physical key chord using the exact bare-event sequence documented by+/// Apple for `CGEventCreateKeyboardEvent`: NULL source, modifier downs, base+/// down/up, then modifier ups in reverse order. No flags, Unicode payload, or+/// event-type overrides are applied; CoreGraphics derives those from the+/// virtual key transitions and its default source state.+pub fn press_key_bare_global(key: &str, modifiers: &[&str]) -> anyhow::Result<()> {+ use core_graphics::event::CGEventTapLocation;++ let key_code = key_name_to_code(key)?;+ let mut modifier_codes = Vec::new();+ for modifier in modifiers {+ let Some((modifier_code, _)) = modifier_key_code_and_flag(modifier) else {+ continue;+ };+ if !modifier_codes.contains(&modifier_code) {+ modifier_codes.push(modifier_code);+ }+ }++ let events = bare_chord_transitions(key_code, &modifier_codes)+ .into_iter()+ .map(|(code, down)| create_bare_keyboard_event(code, down))+ .collect::<anyhow::Result<Vec<_>>>()?;+ for event in events {+ event.post(CGEventTapLocation::HID);+ std::thread::sleep(std::time::Duration::from_millis(8));+ }+ Ok(())+}++fn bare_chord_transitions(key_code: u16, modifier_codes: &[u16]) -> Vec<(u16, bool)> {+ let mut transitions = Vec::with_capacity(modifier_codes.len() * 2 + 2);+ transitions.extend(modifier_codes.iter().map(|&code| (code, true)));+ transitions.push((key_code, true));+ transitions.push((key_code, false));+ transitions.extend(modifier_codes.iter().rev().map(|&code| (code, false)));+ transitions+}++fn create_bare_keyboard_event(key_code: u16, key_down: bool) -> anyhow::Result<CGEvent> {+ unsafe {+ let event_ref = CGEventCreateKeyboardEvent(std::ptr::null_mut(), key_code, key_down);+ if event_ref.is_null() {+ anyhow::bail!("CGEventCreateKeyboardEvent with default source failed");+ }+ Ok(CGEvent::from_ptr(event_ref))+ }+}++#[link(name = "CoreGraphics", kind = "framework")]+extern "C" {+ fn CGEventCreateKeyboardEvent(+ source: core_graphics::sys::CGEventSourceRef,+ key_code: u16,+ key_down: bool,+ ) -> core_graphics::sys::CGEventRef;+}++#[derive(Clone, Copy, Debug, PartialEq, Eq)]+struct PhysicalTextEvent {+ key_code: u16,+ key_down: bool,+ event_type: PhysicalEventType,+ shift: bool,+ text: Option<char>,+}++#[derive(Clone, Copy, Debug, PartialEq, Eq)]+enum PhysicalEventType {+ KeyDown,+ KeyUp,+ FlagsChanged,+}++fn physical_text_events(ch: char) -> anyhow::Result<Vec<PhysicalTextEvent>> {+ let (key_code, shift) = physical_key_for_char(ch).ok_or_else(|| {+ anyhow::anyhow!(+ "Screen Sharing physical text delivery does not support character U+{:04X}",+ ch as u32+ )+ })?;+ let mut events = Vec::with_capacity(if shift { 4 } else { 2 });+ if shift {+ events.push(PhysicalTextEvent {+ key_code: SHIFT_KEY_CODE,+ key_down: true,+ event_type: PhysicalEventType::FlagsChanged,+ shift: true,+ text: None,+ });+ }+ events.push(PhysicalTextEvent {+ key_code,+ key_down: true,+ event_type: PhysicalEventType::KeyDown,+ shift,+ text: Some(ch),+ });+ events.push(PhysicalTextEvent {+ key_code,+ key_down: false,+ event_type: PhysicalEventType::KeyUp,+ shift,+ text: Some(ch),+ });+ if shift {+ events.push(PhysicalTextEvent {+ key_code: SHIFT_KEY_CODE,+ key_down: false,+ event_type: PhysicalEventType::FlagsChanged,+ shift: false,+ text: None,+ });+ }+ Ok(events)+}++/// Map printable ASCII to the physical key that produces it on the standard US+/// layout. The Screen Sharing path intentionally sends only the returned+/// keycode and the required bare Shift transitions.+fn physical_key_for_char(ch: char) -> Option<(u16, bool)> {+ let lower = ch.to_ascii_lowercase();+ if ch.is_ascii_alphabetic() {+ return key_name_to_code(&lower.to_string())+ .ok()+ .map(|code| (code, ch.is_ascii_uppercase()));+ }+ if ch.is_ascii_digit() {+ return key_name_to_code(&ch.to_string())+ .ok()+ .map(|code| (code, false));+ }++ let (base, shift) = match ch {+ ' ' => ("space", false),+ '\t' => ("tab", false),+ '\n' | '\r' => ("return", false),+ '-' => ("-", false),+ '_' => ("-", true),+ '=' => ("=", false),+ '+' => ("=", true),+ '[' => ("[", false),+ '{' => ("[", true),+ ']' => ("]", false),+ '}' => ("]", true),+ '\\' => ("\\", false),+ '|' => ("\\", true),+ ';' => (";", false),+ ':' => (";", true),+ '\'' => ("'", false),+ '"' => ("'", true),+ ',' => (",", false),+ '<' => (",", true),+ '.' => (".", false),+ '>' => (".", true),+ '/' => ("/", false),+ '?' => ("/", true),+ '`' => ("`", false),+ '~' => ("`", true),+ '!' => ("1", true),+ '@' => ("2", true),+ '#' => ("3", true),+ '$' => ("4", true),+ '%' => ("5", true),+ '^' => ("6", true),+ '&' => ("7", true),+ '*' => ("8", true),+ '(' => ("9", true),+ ')' => ("0", true),+ _ => return None,+ };+ key_name_to_code(base).ok().map(|code| (code, shift))+}+/// Post a keyboard event to `pid` via SLEventPostToPid (with auth message for/// Chromium/Electron support) or fall back to CGEvent::post_to_pid.pub(super) fn post_keyboard_event(pid: i32, event: &CGEvent) {@@ -406,3 +632,156 @@ pub(super) fn key_name_to_code(key: &str) -> anyhow::Result<u16> {};Ok(code)}++#[cfg(test)]+mod tests {+ use super::*;+ use core_graphics::event::CGEventType;++ #[test]+ fn physical_text_uses_flags_changed_for_balanced_shift_transitions() {+ assert_eq!(+ physical_text_events('a').unwrap(),+ vec![+ PhysicalTextEvent {+ key_code: 0,+ key_down: true,+ event_type: PhysicalEventType::KeyDown,+ shift: false,+ text: Some('a'),+ },+ PhysicalTextEvent {+ key_code: 0,+ key_down: false,+ event_type: PhysicalEventType::KeyUp,+ shift: false,+ text: Some('a'),+ },+ ]+ );+ assert_eq!(+ physical_text_events('Z').unwrap(),+ vec![+ PhysicalTextEvent {+ key_code: 56,+ key_down: true,+ event_type: PhysicalEventType::FlagsChanged,+ shift: true,+ text: None,+ },+ PhysicalTextEvent {+ key_code: 6,+ key_down: true,+ event_type: PhysicalEventType::KeyDown,+ shift: true,+ text: Some('Z'),+ },+ PhysicalTextEvent {+ key_code: 6,+ key_down: false,+ event_type: PhysicalEventType::KeyUp,+ shift: true,+ text: Some('Z'),+ },+ PhysicalTextEvent {+ key_code: 56,+ key_down: false,+ event_type: PhysicalEventType::FlagsChanged,+ shift: false,+ text: None,+ },+ ]+ );+ assert_eq!(physical_key_for_char('1'), Some((18, false)));+ assert_eq!(physical_key_for_char('!'), Some((18, true)));+ assert_eq!(physical_key_for_char('/'), Some((44, false)));+ assert_eq!(physical_key_for_char('?'), Some((44, true)));+ }++ #[test]+ fn bare_modifier_events_derive_type_and_flags_from_default_source() {+ let shift_down = create_bare_keyboard_event(SHIFT_KEY_CODE, true).unwrap();+ assert_eq!(+ shift_down.get_type() as u32,+ CGEventType::FlagsChanged as u32+ );+ assert!(+ shift_down+ .get_flags()+ .contains(CGEventFlags::CGEventFlagShift),+ "bare Shift down must derive the active Shift flag"+ );+ assert_eq!(+ shift_down+ .get_integer_value_field(core_graphics::event::EventField::EVENT_SOURCE_STATE_ID),+ CGEventSourceStateID::CombinedSessionState as i64,+ "NULL source must resolve to the default combined-session state, not HID state"+ );++ let shift_up = create_bare_keyboard_event(SHIFT_KEY_CODE, false).unwrap();+ assert_eq!(shift_up.get_type() as u32, CGEventType::FlagsChanged as u32);+ assert!(+ !shift_up+ .get_flags()+ .contains(CGEventFlags::CGEventFlagShift),+ "bare Shift up must derive cleared Shift state"+ );++ let z_down = create_bare_keyboard_event(6, true).unwrap();+ assert_eq!(z_down.get_type() as u32, CGEventType::KeyDown as u32);+ assert_eq!(+ z_down+ .get_integer_value_field(core_graphics::event::EventField::KEYBOARD_EVENT_KEYCODE),+ 6+ );+ }++ #[test]+ fn bare_command_chord_orders_modifier_base_and_reverse_release() {+ assert_eq!(+ bare_chord_transitions(9, &[55]),+ vec![(55, true), (9, true), (9, false), (55, false)]+ );+ let command_down = create_bare_keyboard_event(55, true).unwrap();+ assert_eq!(+ command_down.get_type() as u32,+ CGEventType::FlagsChanged as u32+ );+ assert!(+ command_down+ .get_flags()+ .contains(CGEventFlags::CGEventFlagCommand),+ "bare Command down must derive the active Command flag"+ );+ }++ #[test]+ fn physical_text_rejects_characters_without_a_lossless_keycode() {+ let error = physical_text_events('🙂').unwrap_err().to_string();+ assert!(error.contains("U+1F642"), "{error}");+ }++ #[test]+ fn physical_text_covers_printable_ascii_and_common_text_whitespace() {+ for byte in 0x20_u8..=0x7e {+ let ch = char::from(byte);+ assert!(+ physical_key_for_char(ch).is_some(),+ "missing physical key for ASCII {ch:?}"+ );+ }+ for ch in ['\t', '\n', '\r'] {+ assert!(+ physical_key_for_char(ch).is_some(),+ "missing physical key for whitespace {ch:?}"+ );+ }+ }++ #[test]+ fn screen_sharing_detection_is_exact_and_case_sensitive() {+ assert!(is_screen_sharing_bundle_id("com.apple.ScreenSharing"));+ assert!(!is_screen_sharing_bundle_id("com.apple.screensharing"));+ assert!(!is_screen_sharing_bundle_id("com.microsoft.rdc.macos"));+ }+}diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/hotkey.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/hotkey.rsindex 1f74f3c63a..fa1b4474b3 100644--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/hotkey.rs+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/hotkey.rs@@ -93,6 +93,33 @@ fn is_modifier(k: &str) -> bool {)}+fn screen_sharing_modifier_delivery_error(+ is_screen_sharing: bool,+ has_modifiers: bool,+ foreground: bool,+ window_id: Option<u32>,+) -> Option<ToolResult> {+ if !is_screen_sharing || !has_modifiers || (foreground && window_id.is_some()) {+ return None;+ }+ Some(+ ToolResult::error(+ "Screen Sharing modifier hotkeys require delivery_mode:\"foreground\" and window_id \+ so Cua Driver can deliver physical modifier transitions safely.",+ )+ .with_structured(serde_json::json!({+ "code": "SCREEN_SHARING_REQUIRES_FOREGROUND_HID",+ "effect": "refused",+ "escalation": {+ "recommended": "foreground",+ "reason": "Screen Sharing does not forward modifier state from background \+ PID-routed base-key events.",+ "requires": ["window_id"]+ }+ })),+ )+}+#[async_trait]impl Tool for HotkeyTool {fn def(&self) -> &ToolDef {@@ -187,6 +214,15 @@ impl Tool for HotkeyTool {// background combo (matches click/type_text).let delivery_mode = super::DeliveryMode::parse(args.opt_str("delivery_mode").as_deref());let fg = delivery_mode.is_foreground();+ let screen_sharing_target = crate::input::keyboard::is_screen_sharing_pid(pid);+ if let Some(error) = screen_sharing_modifier_delivery_error(+ screen_sharing_target,+ !modifiers.is_empty(),+ fg,+ window_id,+ ) {+ return error;+ }// px form: focus the field before sending the combo. Foreground delivery// still needs to front the target for the chord itself: the focus helper@@ -245,7 +281,27 @@ impl Tool for HotkeyTool {crate::input::skylight::with_foreground_hid_activation(pid as libc::pid_t,wid,- || crate::input::keyboard::press_key_global(&key, &m),+ || {+ if screen_sharing_target {+ crate::input::keyboard::press_key_bare_global(&key, &m)+ } else {+ crate::input::keyboard::press_key_global(&key, &m)+ }+ },+ )?;+ Ok(())+ }+ // Screen Sharing is an input forwarder: modifier flags+ // on a PID-routed base-key event are not relayed to the+ // guest. Emit the physical modifier down/base/up+ // sequence through the guarded foreground HID path.+ (true, false, Some(wid))+ if crate::input::keyboard::is_screen_sharing_pid(pid) =>+ {+ crate::input::skylight::with_foreground_hid_activation(+ pid as libc::pid_t,+ wid,+ || crate::input::keyboard::press_key_bare_global(&key, &m),)?;Ok(())}@@ -306,3 +362,25 @@ impl Tool for HotkeyTool {}}}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn screen_sharing_modifier_hotkeys_fail_closed_without_foreground_window() {+ for (foreground, window_id) in [(false, None), (false, Some(7)), (true, None)] {+ let result = screen_sharing_modifier_delivery_error(true, true, foreground, window_id)+ .expect("unsafe Screen Sharing modifier route must be refused");+ assert_eq!(result.is_error, Some(true));+ let structured = result.structured_content.unwrap();+ assert_eq!(structured["code"], "SCREEN_SHARING_REQUIRES_FOREGROUND_HID");+ assert_eq!(structured["effect"], "refused");+ assert_eq!(structured["escalation"]["recommended"], "foreground");+ assert_eq!(structured["escalation"]["requires"][0], "window_id");+ }+ assert!(screen_sharing_modifier_delivery_error(true, true, true, Some(7)).is_none());+ assert!(screen_sharing_modifier_delivery_error(true, false, false, None).is_none());+ assert!(screen_sharing_modifier_delivery_error(false, true, false, None).is_none());+ }+}diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/type_text.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/type_text.rsindex d74f5df922..b659d622af 100644--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/type_text.rs+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/type_text.rs@@ -125,6 +125,32 @@ fn def() -> &'static ToolDef {})}+fn screen_sharing_delivery_error(+ is_screen_sharing: bool,+ foreground: bool,+ window_id: Option<u32>,+) -> Option<ToolResult> {+ if !is_screen_sharing || (foreground && window_id.is_some()) {+ return None;+ }+ Some(+ ToolResult::error(+ "Screen Sharing text input requires delivery_mode:\"foreground\" and window_id \+ so Cua Driver can deliver physical HID key transitions safely.",+ )+ .with_structured(serde_json::json!({+ "code": "SCREEN_SHARING_REQUIRES_FOREGROUND_HID",+ "effect": "refused",+ "escalation": {+ "recommended": "foreground",+ "reason": "Screen Sharing forwards physical keycodes; background PID-routed \+ Unicode events can corrupt guest text.",+ "requires": ["window_id"]+ }+ })),+ )+}+#[async_trait]impl Tool for TypeTextTool {fn def(&self) -> &ToolDef {@@ -198,6 +224,13 @@ impl Tool for TypeTextTool {};let delay_ms = args.u64_or("delay_ms", 30);let delivery_mode = super::DeliveryMode::parse(args.opt_str("delivery_mode").as_deref());+ if let Some(error) = screen_sharing_delivery_error(+ crate::input::keyboard::is_screen_sharing_pid(pid),+ delivery_mode.is_foreground(),+ window_id,+ ) {+ return error;+ }// ── px form: focus by pixel-click, then type into the focused element ──// Pass x,y (no element_index) for an *element px action*: pixel-click the@@ -768,6 +801,7 @@ fn type_text_blocking(// dropped. 200ms covers that re-grab without penalizing an already// armed interactive stream on every text chunk.let foreground_settle_ms = foreground_settle_ms(pid, apps::frontmost_pid());+ let screen_sharing_target = crate::input::keyboard::is_screen_sharing_pid(pid);let do_type = || {cgevent_type_verified(pid,@@ -779,6 +813,27 @@ fn type_text_blocking()};let ((verified, delivered_chars), fronted) = match window_id {+ Some(wid) if screen_sharing_target => {+ // Screen Sharing forwards physical HID transitions to the+ // guest. PID-routed Unicode events all carry keycode 0 (the A+ // key), so a guest sees "aaaa"; modifier flags alone likewise+ // turn Cmd+V into plain "v". The explicit foreground rung may+ // safely use the global HID queue while the exact target is+ // guarded and restored.+ crate::input::skylight::with_foreground_hid_activation(+ pid as libc::pid_t,+ wid,+ || {+ if foreground_settle_ms > 0 {+ std::thread::sleep(std::time::Duration::from_millis(+ foreground_settle_ms,+ ));+ }+ crate::input::keyboard::type_text_physical_global(text, delay_ms)+ },+ )?;+ ((false, None), true)+ }Some(wid) => {// Front → type → restore. The closure returns the read-back// result; with_foreground_assist returns whether it actually@@ -1065,4 +1120,20 @@ mod tests {assert_eq!(foreground_settle_ms(42, Some(7)), 200);assert_eq!(foreground_settle_ms(42, None), 200);}++ #[test]+ fn screen_sharing_text_fails_closed_without_foreground_window() {+ for (foreground, window_id) in [(false, None), (false, Some(7)), (true, None)] {+ let result = screen_sharing_delivery_error(true, foreground, window_id)+ .expect("unsafe Screen Sharing route must be refused");+ assert_eq!(result.is_error, Some(true));+ let structured = result.structured_content.unwrap();+ assert_eq!(structured["code"], "SCREEN_SHARING_REQUIRES_FOREGROUND_HID");+ assert_eq!(structured["effect"], "refused");+ assert_eq!(structured["escalation"]["recommended"], "foreground");+ assert_eq!(structured["escalation"]["requires"][0], "window_id");+ }+ assert!(screen_sharing_delivery_error(true, true, Some(7)).is_none());+ assert!(screen_sharing_delivery_error(false, false, None).is_none());+ }}