Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
fix(cua-driver): type long Linux text in spare-keycode batches (#5168)
injaneity8be7052a063a
diff --git a/libs/cua-driver/rust/crates/platform-linux/src/input/mod.rs b/libs/cua-driver/rust/crates/platform-linux/src/input/mod.rsindex 0de7e513d6..c899caddbe 100644--- a/libs/cua-driver/rust/crates/platform-linux/src/input/mod.rs+++ b/libs/cua-driver/rust/crates/platform-linux/src/input/mod.rs@@ -3425,13 +3425,65 @@ fn type_text_with_preflight<G>(Ok(guards)}+fn type_text_in_spare_batches<G>(+ text: &str,+ mapping: &GetKeyboardMappingReply,+ mut remap: impl FnMut(char, &mut GetKeyboardMappingReply) -> Result<(u8, Option<G>)>,+ mut send: impl FnMut(u8, bool) -> Result<()>,+) -> Result<()> {+ let total = text.chars().count();+ let spares = match mapping.keysyms_per_keycode as usize {+ 0 => 0,+ per => mapping+ .keysyms+ .chunks(per)+ .filter(|syms| syms.iter().all(|&s| s == 0))+ .count(),+ };+ let mut rest = text.chars().peekable();+ let mut typed = 0;+ while rest.peek().is_some() {+ let mut mapping = mapping.clone();+ let mut guards = Vec::new();+ let mut resolved = std::collections::HashMap::new();+ let mut remapped = 0;+ let mut keys = Vec::new();+ while let Some(&ch) = rest.peek() {+ let key = match resolved+ .get(&ch)+ .copied()+ .or_else(|| char_to_keycode_shift(&mapping, mpx_keyboard::keysym_for_char(ch)))+ {+ Some(key) => key,+ None if spares > 0 && remapped == spares => break,+ None => {+ let (keycode, guard) = remap(ch, &mut mapping).map_err(|error| {+ anyhow::anyhow!("typed {typed} of {total} characters: {error:#}")+ })?;+ remapped += 1;+ guards.extend(guard);+ resolved.insert(ch, (keycode, false));+ (keycode, false)+ }+ };+ keys.push(key);+ rest.next();+ }+ for (keycode, needs_shift) in keys {+ send(keycode, needs_shift)?;+ typed += 1;+ }+ }+ Ok(())+}+/// Type a string with an additional `inter_char_ms` delay between each character.pub fn send_type_text_with_delay(xid: u64, text: &str, inter_char_ms: u64) -> Result<()> {let (conn, _) = connect_x11_for_input()?;let window = xid as u32;let root = conn.setup().roots[0].root;let mapping = conn.get_keyboard_mapping(8, 248)?.reply()?;- let _guards = type_text_with_preflight(+ type_text_in_spare_batches(text,&mapping,|ch, mapping| {@@ -3516,7 +3568,7 @@ pub fn send_type_text_xtest(text: &str) -> Result<()> {.get(..kpm).and_then(|s| s.iter().copied().find(|&k| k != 0)).unwrap_or(50);- let _guards = type_text_with_preflight(+ type_text_in_spare_batches(text,&mapping,|ch, mapping| {@@ -4135,8 +4187,8 @@ pub(crate) fn key_name_to_keysym(key: &str) -> Result<u32> {#[cfg(test)]mod key_name_alias_tests {use super::{- char_to_keycode_shift, key_name_to_keysym, reserve_spare_keycode, type_text_with_preflight,- GetKeyboardMappingReply,+ char_to_keycode_shift, key_name_to_keysym, reserve_spare_keycode,+ type_text_in_spare_batches, type_text_with_preflight, GetKeyboardMappingReply,};#[test]@@ -4214,6 +4266,86 @@ mod key_name_alias_tests {}}+ #[test]+ fn text_batches_types_more_distinct_unmapped_characters_than_spares() {+ use std::cell::RefCell;+ struct Restore<'a>(u8, &'a RefCell<Vec<(&'static str, u8)>>);+ impl Drop for Restore<'_> {+ fn drop(&mut self) {+ self.1.borrow_mut().push(("restore", self.0));+ }+ }+ let mapping = GetKeyboardMappingReply {+ keysyms_per_keycode: 2,+ sequence: 0,+ keysyms: vec!['a' as u32, 'A' as u32, 0, 0, 0, 0],+ };+ let log = RefCell::new(Vec::new());+ type_text_in_spare_batches(+ "a你好你世a世",+ &mapping,+ |ch, mapping| {+ reserve_spare_keycode(+ mapping,+ super::mpx_keyboard::keysym_for_char(ch),+ |spare, _, _, _| {+ log.borrow_mut().push(("remap", spare));+ Ok(Restore(spare, &log))+ },+ )+ .map(|(keycode, guard)| (keycode, Some(guard)))+ },+ |keycode, _| {+ log.borrow_mut().push(("send", keycode));+ Ok(())+ },+ )+ .unwrap();+ assert_eq!(+ log.into_inner(),+ [+ ("remap", 10),+ ("remap", 9),+ ("send", 8),+ ("send", 10),+ ("send", 9),+ ("send", 10),+ ("restore", 10),+ ("restore", 9),+ ("remap", 10),+ ("send", 10),+ ("send", 8),+ ("send", 10),+ ("restore", 10),+ ]+ );+ }++ #[test]+ fn text_batches_reports_progress_when_a_later_batch_fails() {+ let mapping = GetKeyboardMappingReply {+ keysyms_per_keycode: 2,+ sequence: 0,+ keysyms: vec!['a' as u32, 'A' as u32, 0, 0],+ };+ let mut events = Vec::new();+ let error = type_text_in_spare_batches::<()>(+ "你a好",+ &mapping,+ |ch, _| match ch {+ '你' => Ok((9, None)),+ _ => anyhow::bail!("keymap changed"),+ },+ |keycode, _| {+ events.push(keycode);+ Ok(())+ },+ )+ .unwrap_err();+ assert_eq!(error.to_string(), "typed 2 of 3 characters: keymap changed");+ assert_eq!(events, [9, 8]);+ }+#[test]fn text_preflight_restores_every_remap_after_success_and_failure() {let mapping = GetKeyboardMappingReply {diff --git a/libs/cua-driver/rust/crates/platform-linux/tests/key_input_x11.rs b/libs/cua-driver/rust/crates/platform-linux/tests/key_input_x11.rsindex def6e5615b..918922c1b3 100644--- a/libs/cua-driver/rust/crates/platform-linux/tests/key_input_x11.rs+++ b/libs/cua-driver/rust/crates/platform-linux/tests/key_input_x11.rs@@ -404,6 +404,46 @@ fn xtest_text_fails_when_no_spare_keycode_is_left() -> Result<()> {Ok(())}+/// Text with more distinct characters missing from the keymap than there are+/// spare keycodes is typed in batches that reuse the spare keycodes.+#[test]+#[ignore = "requires an isolated X11 display with XTEST"]+fn text_types_more_distinct_unmapped_characters_than_spare_keycodes() -> Result<()> {+ let (conn, screen) = connect()?;+ let window = focused_input_window(&conn, screen)?;+ let spare = spare_keycodes(&conn)?;+ let (&free, taken) = spare+ .split_last()+ .expect("the test display has no spare keycode");+ let per = conn+ .get_keyboard_mapping(free, 1)?+ .reply()?+ .keysyms_per_keycode;+ let bind = |keysym: u32| -> Result<()> {+ for &keycode in taken {+ conn.change_keyboard_mapping(1, keycode, per, &vec![keysym; usize::from(per)])?;+ }+ conn.get_input_focus()?.reply()?;+ Ok(())+ };+ bind(0x7e1)?;+ let results = [+ send_type_text_xtest("\u{4f60}\u{597d}\u{4e16}\u{4f60}"),+ send_type_text(u64::from(window), "\u{4f60}\u{597d}\u{4e16}\u{4f60}"),+ ];+ bind(0)?;+ for result in results {+ result?;+ let events = keyboard_events(&conn, 8)?;+ for pair in events.chunks_exact(2) {+ assert!(pair[0].0 && !pair[1].0);+ assert_eq!((pair[0].1.detail, pair[1].1.detail), (free, free));+ }+ }+ assert_eq!(spare_keycodes(&conn)?, spare);+ Ok(())+}+#[test]#[ignore = "requires an isolated X11 display with XTEST"]fn xtest_text_gives_each_missing_character_its_own_keycode() -> Result<()> {