Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
fix(cua-driver): verify macOS target focus before dispatching keys (#5372)
injaneity8a66f0d8d955
diff --git a/docs/content/docs/cua-driver/reference/mcp-tools/keys-and-shortcuts.mdx b/docs/content/docs/cua-driver/reference/mcp-tools/keys-and-shortcuts.mdxindex a56a5764d5..b80a2a99a7 100644--- a/docs/content/docs/cua-driver/reference/mcp-tools/keys-and-shortcuts.mdx+++ b/docs/content/docs/cua-driver/reference/mcp-tools/keys-and-shortcuts.mdx@@ -30,6 +30,7 @@ Press and release one key. `delivery_mode` as in click/type_text.- `background` (default): posts to the pid without raising the window; with `element_token` it focuses that element first.- `foreground`: fronts the exact window (needs `window_id`), sends a real HID key transition (Chromium content, inline editors, menu equivalents), then restores the prior app.+An element target whose focus cannot be confirmed is refused with `target_focus_unconfirmed`, effect "refused", and no key dispatched.Confirmed only when a native AX value/selection read-back changes; otherwise `effect:"unverifiable"` (not a delivery failure). Keys: return, tab, escape, up/down/left/right, space, delete, home, end, pageup, pagedown, f1-f12, any letter or digit.**macOS parameters**diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/press_key.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/press_key.rsindex 1069b452f9..aec7eb7262 100644--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/press_key.rs+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/press_key.rs@@ -170,7 +170,39 @@ fn action_record(confirmed: bool, foreground: bool) -> ActionExecutionRecord {record.build().expect("press_key record is valid")}-fn delivery_failed(error: impl std::fmt::Display) -> ToolResult {+#[derive(Debug, thiserror::Error)]+#[error("the addressed element did not acquire keyboard focus; no key was dispatched")]+struct KeyFocusUnconfirmed;++fn with_confirmed_key_focus<T>(+ target: Option<usize>,+ mut focus: impl FnMut(usize) -> anyhow::Result<()>,+ mut is_focused: impl FnMut(usize) -> bool,+ dispatch: impl FnOnce() -> anyhow::Result<T>,+) -> anyhow::Result<T> {+ if let Some(target) = target {+ let deadline = std::time::Instant::now() + std::time::Duration::from_millis(200);+ loop {+ focus(target)?;+ if is_focused(target) {+ break;+ }+ if std::time::Instant::now() >= deadline {+ return Err(KeyFocusUnconfirmed.into());+ }+ std::thread::sleep(std::time::Duration::from_millis(10));+ }+ }+ dispatch()+}++fn delivery_failed(error: anyhow::Error) -> ToolResult {+ if error.is::<KeyFocusUnconfirmed>() {+ return ToolResult::error(error.to_string()).with_structured(serde_json::json!({+ "code": "target_focus_unconfirmed", "effect": "refused", "dispatched": false,+ "message": error.to_string(),+ }));+ }let message = format!("press_key delivery failed: {error}");ToolResult::error(&message).with_structured(serde_json::json!({"code": "delivery_failed",@@ -185,6 +217,7 @@ fn def() -> &'static ToolDef {- `background` (default): posts to the pid without raising the window; with `element_token` it focuses that element first.\n\- `foreground`: fronts the exact window (needs `window_id`), sends a real HID key transition (Chromium content, inline editors, menu equivalents), then restores the prior app.\n\\n\+ An element target whose focus cannot be confirmed is refused with `target_focus_unconfirmed`, effect \"refused\", and no key dispatched.\n\Confirmed only when a native AX value/selection read-back changes; otherwise `effect:\"unverifiable\"` (not a delivery failure). Keys: return, tab, escape, up/down/left/right, space, delete, home, end, pageup, pagedown, f1-f12, any letter or digit.".into(),input_schema: serde_json::json!({"type": "object",@@ -409,16 +442,6 @@ impl Tool for PressKeyTool {prior_front,"press_key.CGEvent",|| async move {- // Pre-focus the element under suppression so its- // side-effects are captured by the snapshot + lease.- if let Some(guard) = pre_focus_guard.clone() {- let _ = tokio::task::spawn_blocking(move || {- crate::input::ax_actions::focus_element(guard.as_ptr())- })- .await;- tokio::time::sleep(std::time::Duration::from_millis(30)).await;- }-tokio::task::spawn_blocking(move || {let pre_focus_ptr = pre_focus_guard.as_ref().map(|guard| guard.as_ptr());let m: Vec<&str> = modifiers.iter().map(String::as_str).collect();@@ -439,22 +462,29 @@ impl Tool for PressKeyTool {pid as libc::pid_t,wid,|| {- // Activation can change the first responder, so- // repeat the best-effort AX focus write inside- // the guarded foreground interval immediately- // before the physical key transition.- if let Some(element_ptr) = pre_focus_ptr {- let _ =- crate::input::ax_actions::focus_element(element_ptr);- }- crate::input::keyboard::press_key_bare_global(&key, &m)+ // Activation may replace the first responder; confirm inside the HID guard.+ with_confirmed_key_focus(+ pre_focus_ptr,+ crate::input::ax_actions::focus_element,+ |target| {+ crate::input::ax_actions::is_element_focused(+ pid, target,+ )+ },+ || crate::input::keyboard::press_key_bare_global(&key, &m),+ )},)});}// background (default): auth-envelope post, no raise.dispatch_with_ax_oracle(pid, window_id, pre_focus_ptr, || {- crate::input::keyboard::press_key(pid, &key, &m)+ with_confirmed_key_focus(+ pre_focus_ptr,+ crate::input::ax_actions::focus_element,+ |target| crate::input::ax_actions::is_element_focused(pid, target),+ || crate::input::keyboard::press_key(pid, &key, &m),+ )})}).await@@ -501,6 +531,108 @@ impl Tool for PressKeyTool {mod tests {use super::*;+ #[test]+ fn targeted_key_requires_focus_readback_before_dispatch() {+ use std::cell::RefCell;+ let events = RefCell::new(Vec::new());+ let result = with_confirmed_key_focus(+ Some(42),+ |target| {+ events.borrow_mut().push(("focus", target));+ Ok(())+ },+ |target| {+ events.borrow_mut().push(("verify", target));+ false+ },+ || {+ events.borrow_mut().push(("dispatch", 0));+ Ok(())+ },+ );+ let failure = delivery_failed(result.unwrap_err());+ assert_eq!(+ failure.structured_content.as_ref().unwrap()["code"],+ "target_focus_unconfirmed"+ );+ assert_eq!(+ failure.structured_content.as_ref().unwrap()["effect"],+ "refused"+ );+ let events = events.borrow();+ assert!(events.len() >= 4);+ for pair in events.chunks(2) {+ assert_eq!(pair, &[("focus", 42), ("verify", 42)]);+ }+ }++ #[test]+ fn targeted_key_retries_focus_before_dispatch() {+ use std::cell::RefCell;+ let events = RefCell::new(Vec::new());+ let mut checks = 0;+ with_confirmed_key_focus(+ Some(42),+ |target| {+ events.borrow_mut().push(("focus", target));+ Ok(())+ },+ |target| {+ events.borrow_mut().push(("verify", target));+ checks += 1;+ checks == 2+ },+ || {+ events.borrow_mut().push(("dispatch", 0));+ Ok(())+ },+ )+ .unwrap();+ assert_eq!(+ *events.borrow(),+ vec![+ ("focus", 42),+ ("verify", 42),+ ("focus", 42),+ ("verify", 42),+ ("dispatch", 0)+ ]+ );+ }++ #[test]+ fn confirmed_target_dispatches_once_and_window_keys_do_not_focus() {+ use std::cell::Cell;+ let sent = Cell::new(0);+ for target in [Some(42), None] {+ with_confirmed_key_focus(+ target,+ |_| Ok(()),+ |_| true,+ || {+ sent.set(sent.get() + 1);+ Ok(())+ },+ )+ .unwrap();+ }+ assert_eq!(sent.get(), 2);+ let failed = with_confirmed_key_focus(+ Some(42),+ |_| anyhow::bail!("focus unavailable"),+ |_| panic!("must not confirm failed focus"),+ || -> anyhow::Result<()> { panic!("must not dispatch") },+ );+ assert_eq!(failed.unwrap_err().to_string(), "focus unavailable");+ with_confirmed_key_focus(+ None,+ |_| panic!("window key has no element"),+ |_| panic!("window key has no element"),+ || Ok(()),+ )+ .unwrap();+ }+#[test]fn delivery_outcome_mapper_distinguishes_confirmed_unverifiable_and_failed() {assert!(matches!(diff --git a/scripts/docs-generators/cli-specs/cua-driver-mcp-macos.json b/scripts/docs-generators/cli-specs/cua-driver-mcp-macos.jsonindex 455f4422f8..9a27c050e0 100644--- a/scripts/docs-generators/cli-specs/cua-driver-mcp-macos.json+++ b/scripts/docs-generators/cli-specs/cua-driver-mcp-macos.json@@ -938,7 +938,7 @@"read_only": false},{- "description": "Press and release one key. `delivery_mode` as in click/type_text.\n- `background` (default): posts to the pid without raising the window; with `element_token` it focuses that element first.\n- `foreground`: fronts the exact window (needs `window_id`), sends a real HID key transition (Chromium content, inline editors, menu equivalents), then restores the prior app.\n\nConfirmed only when a native AX value/selection read-back changes; otherwise `effect:\"unverifiable\"` (not a delivery failure). Keys: return, tab, escape, up/down/left/right, space, delete, home, end, pageup, pagedown, f1-f12, any letter or digit.",+ "description": "Press and release one key. `delivery_mode` as in click/type_text.\n- `background` (default): posts to the pid without raising the window; with `element_token` it focuses that element first.\n- `foreground`: fronts the exact window (needs `window_id`), sends a real HID key transition (Chromium content, inline editors, menu equivalents), then restores the prior app.\n\nAn element target whose focus cannot be confirmed is refused with `target_focus_unconfirmed`, effect \"refused\", and no key dispatched.\nConfirmed only when a native AX value/selection read-back changes; otherwise `effect:\"unverifiable\"` (not a delivery failure). Keys: return, tab, escape, up/down/left/right, space, delete, home, end, pageup, pagedown, f1-f12, any letter or digit.","destructive": true,"idempotent": false,"input_schema": {