Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

fix(cua-driver): type long Linux text in spare-keycode batches (#5168)

injaneity8be7052a063a
diff --git a/libs/cua-driver/rust/crates/platform-linux/src/input/mod.rs b/libs/cua-driver/rust/crates/platform-linux/src/input/mod.rs
index 0de7e513d6..c899caddbe 100644
--- a/libs/cua-driver/rust/crates/platform-linux/src/input/mod.rs
+++ b/libs/cua-driver/rust/crates/platform-linux/src/input/mod.rs
@@ -3425,13 +3425,65 @@ fn type_text_with_preflight<G>(
Ok(guards)
}
+fn type_text_in_spare_batches<G>(
+ text: &str,
+ mapping: &GetKeyboardMappingReply,
+ mut remap: impl FnMut(char, &mut GetKeyboardMappingReply) -> Result<(u8, Option<G>)>,
+ mut send: impl FnMut(u8, bool) -> Result<()>,
+) -> Result<()> {
+ let total = text.chars().count();
+ let spares = match mapping.keysyms_per_keycode as usize {
+ 0 => 0,
+ per => mapping
+ .keysyms
+ .chunks(per)
+ .filter(|syms| syms.iter().all(|&s| s == 0))
+ .count(),
+ };
+ let mut rest = text.chars().peekable();
+ let mut typed = 0;
+ while rest.peek().is_some() {
+ let mut mapping = mapping.clone();
+ let mut guards = Vec::new();
+ let mut resolved = std::collections::HashMap::new();
+ let mut remapped = 0;
+ let mut keys = Vec::new();
+ while let Some(&ch) = rest.peek() {
+ let key = match resolved
+ .get(&ch)
+ .copied()
+ .or_else(|| char_to_keycode_shift(&mapping, mpx_keyboard::keysym_for_char(ch)))
+ {
+ Some(key) => key,
+ None if spares > 0 && remapped == spares => break,
+ None => {
+ let (keycode, guard) = remap(ch, &mut mapping).map_err(|error| {
+ anyhow::anyhow!("typed {typed} of {total} characters: {error:#}")
+ })?;
+ remapped += 1;
+ guards.extend(guard);
+ resolved.insert(ch, (keycode, false));
+ (keycode, false)
+ }
+ };
+ keys.push(key);
+ rest.next();
+ }
+ for (keycode, needs_shift) in keys {
+ send(keycode, needs_shift)?;
+ typed += 1;
+ }
+ }
+ Ok(())
+}
+
/// Type a string with an additional `inter_char_ms` delay between each character.
pub fn send_type_text_with_delay(xid: u64, text: &str, inter_char_ms: u64) -> Result<()> {
let (conn, _) = connect_x11_for_input()?;
let window = xid as u32;
let root = conn.setup().roots[0].root;
let mapping = conn.get_keyboard_mapping(8, 248)?.reply()?;
- let _guards = type_text_with_preflight(
+ type_text_in_spare_batches(
text,
&mapping,
|ch, mapping| {
@@ -3516,7 +3568,7 @@ pub fn send_type_text_xtest(text: &str) -> Result<()> {
.get(..kpm)
.and_then(|s| s.iter().copied().find(|&k| k != 0))
.unwrap_or(50);
- let _guards = type_text_with_preflight(
+ type_text_in_spare_batches(
text,
&mapping,
|ch, mapping| {
@@ -4135,8 +4187,8 @@ pub(crate) fn key_name_to_keysym(key: &str) -> Result<u32> {
#[cfg(test)]
mod key_name_alias_tests {
use super::{
- char_to_keycode_shift, key_name_to_keysym, reserve_spare_keycode, type_text_with_preflight,
- GetKeyboardMappingReply,
+ char_to_keycode_shift, key_name_to_keysym, reserve_spare_keycode,
+ type_text_in_spare_batches, type_text_with_preflight, GetKeyboardMappingReply,
};
#[test]
@@ -4214,6 +4266,86 @@ mod key_name_alias_tests {
}
}
+ #[test]
+ fn text_batches_types_more_distinct_unmapped_characters_than_spares() {
+ use std::cell::RefCell;
+ struct Restore<'a>(u8, &'a RefCell<Vec<(&'static str, u8)>>);
+ impl Drop for Restore<'_> {
+ fn drop(&mut self) {
+ self.1.borrow_mut().push(("restore", self.0));
+ }
+ }
+ let mapping = GetKeyboardMappingReply {
+ keysyms_per_keycode: 2,
+ sequence: 0,
+ keysyms: vec!['a' as u32, 'A' as u32, 0, 0, 0, 0],
+ };
+ let log = RefCell::new(Vec::new());
+ type_text_in_spare_batches(
+ "a你好你世a世",
+ &mapping,
+ |ch, mapping| {
+ reserve_spare_keycode(
+ mapping,
+ super::mpx_keyboard::keysym_for_char(ch),
+ |spare, _, _, _| {
+ log.borrow_mut().push(("remap", spare));
+ Ok(Restore(spare, &log))
+ },
+ )
+ .map(|(keycode, guard)| (keycode, Some(guard)))
+ },
+ |keycode, _| {
+ log.borrow_mut().push(("send", keycode));
+ Ok(())
+ },
+ )
+ .unwrap();
+ assert_eq!(
+ log.into_inner(),
+ [
+ ("remap", 10),
+ ("remap", 9),
+ ("send", 8),
+ ("send", 10),
+ ("send", 9),
+ ("send", 10),
+ ("restore", 10),
+ ("restore", 9),
+ ("remap", 10),
+ ("send", 10),
+ ("send", 8),
+ ("send", 10),
+ ("restore", 10),
+ ]
+ );
+ }
+
+ #[test]
+ fn text_batches_reports_progress_when_a_later_batch_fails() {
+ let mapping = GetKeyboardMappingReply {
+ keysyms_per_keycode: 2,
+ sequence: 0,
+ keysyms: vec!['a' as u32, 'A' as u32, 0, 0],
+ };
+ let mut events = Vec::new();
+ let error = type_text_in_spare_batches::<()>(
+ "你a好",
+ &mapping,
+ |ch, _| match ch {
+ '你' => Ok((9, None)),
+ _ => anyhow::bail!("keymap changed"),
+ },
+ |keycode, _| {
+ events.push(keycode);
+ Ok(())
+ },
+ )
+ .unwrap_err();
+ assert_eq!(error.to_string(), "typed 2 of 3 characters: keymap changed");
+ assert_eq!(events, [9, 8]);
+ }
+
#[test]
fn text_preflight_restores_every_remap_after_success_and_failure() {
let mapping = GetKeyboardMappingReply {
diff --git a/libs/cua-driver/rust/crates/platform-linux/tests/key_input_x11.rs b/libs/cua-driver/rust/crates/platform-linux/tests/key_input_x11.rs
index def6e5615b..918922c1b3 100644
--- a/libs/cua-driver/rust/crates/platform-linux/tests/key_input_x11.rs
+++ b/libs/cua-driver/rust/crates/platform-linux/tests/key_input_x11.rs
@@ -404,6 +404,46 @@ fn xtest_text_fails_when_no_spare_keycode_is_left() -> Result<()> {
Ok(())
}
+/// Text with more distinct characters missing from the keymap than there are
+/// spare keycodes is typed in batches that reuse the spare keycodes.
+#[test]
+#[ignore = "requires an isolated X11 display with XTEST"]
+fn text_types_more_distinct_unmapped_characters_than_spare_keycodes() -> Result<()> {
+ let (conn, screen) = connect()?;
+ let window = focused_input_window(&conn, screen)?;
+ let spare = spare_keycodes(&conn)?;
+ let (&free, taken) = spare
+ .split_last()
+ .expect("the test display has no spare keycode");
+ let per = conn
+ .get_keyboard_mapping(free, 1)?
+ .reply()?
+ .keysyms_per_keycode;
+ let bind = |keysym: u32| -> Result<()> {
+ for &keycode in taken {
+ conn.change_keyboard_mapping(1, keycode, per, &vec![keysym; usize::from(per)])?;
+ }
+ conn.get_input_focus()?.reply()?;
+ Ok(())
+ };
+ bind(0x7e1)?;
+ let results = [
+ send_type_text_xtest("\u{4f60}\u{597d}\u{4e16}\u{4f60}"),
+ send_type_text(u64::from(window), "\u{4f60}\u{597d}\u{4e16}\u{4f60}"),
+ ];
+ bind(0)?;
+ for result in results {
+ result?;
+ let events = keyboard_events(&conn, 8)?;
+ for pair in events.chunks_exact(2) {
+ assert!(pair[0].0 && !pair[1].0);
+ assert_eq!((pair[0].1.detail, pair[1].1.detail), (free, free));
+ }
+ }
+ assert_eq!(spare_keycodes(&conn)?, spare);
+ Ok(())
+}
+
#[test]
#[ignore = "requires an isolated X11 display with XTEST"]
fn xtest_text_gives_each_missing_character_its_own_keycode() -> Result<()> {