Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
fix(cua-driver): report macOS key delivery truthfully (#2830)
Francesco Bonacci3dc0fc0e345f
diff --git a/docs/content/docs/reference/cua-driver/mcp-tools.mdx b/docs/content/docs/reference/cua-driver/mcp-tools.mdxindex 6624757045..c862c5ac5c 100644--- a/docs/content/docs/reference/cua-driver/mcp-tools.mdx+++ b/docs/content/docs/reference/cua-driver/mcp-tools.mdx@@ -368,7 +368,7 @@ Press and release a single key. Follows the same `delivery_mode` ladder as click• `background` (default): post to the pid WITHOUT fronting/raising — the auth-message path (Chromium-safe). With element_index it focuses that AX element first. `window_id` only targets; it does not raise.• `foreground`: guard and briefly front the exact window, focus an addressed AX element when supplied, send a genuine HID key transition so Chromium content, inline editors, and native menu equivalents receive it, then restore prior frontmost. Requires window_id.-A key press is never driver-verifiable → effect:"unverifiable"; confirm via screenshot. Key names: return, tab, escape, up/down/left/right, space, delete, home, end, pageup, pagedown, f1-f12, plus any letter or digit. Modifiers array: cmd, shift, option/alt, ctrl, fn.+A key press is confirmed only when a bounded native AX value/selection read-back changes on the same control. Otherwise a successfully attempted post remains effect:"unverifiable" without implying delivery failure or recommending foreground. Key names: return, tab, escape, up/down/left/right, space, delete, home, end, pageup, pagedown, f1-f12, plus any letter or digit. Modifiers array: cmd, shift, option/alt, ctrl, fn.**Arguments:**diff --git a/libs/cua-driver/rust/crates/cua-driver/tests/harness_appkit_test.rs b/libs/cua-driver/rust/crates/cua-driver/tests/harness_appkit_test.rsindex fb46502f1a..b52a05b395 100644--- a/libs/cua-driver/rust/crates/cua-driver/tests/harness_appkit_test.rs+++ b/libs/cua-driver/rust/crates/cua-driver/tests/harness_appkit_test.rs@@ -24,7 +24,7 @@#![cfg(target_os = "macos")]-use std::path::PathBuf;+use std::path::{Path, PathBuf};use std::process::{Child, Command, Stdio};use std::time::Duration;@@ -62,6 +62,10 @@ struct Harness {impl Harness {fn launch() -> Self {+ Self::launch_with_command_oracle(None)+ }++ fn launch_with_command_oracle(command_oracle: Option<&Path>) -> Self {let exe = harness_exe();assert!(exe.exists(),@@ -70,9 +74,12 @@ impl Harness {// Launch the binary directly (not via `open`) so we control the pid// and can kill it cleanly on Drop. The app still installs an AppKit// window via NSApp.run().- let app = Command::new(&exe)- .stdout(Stdio::null())- .stderr(Stdio::null())+ let mut command = Command::new(&exe);+ command.stdout(Stdio::null()).stderr(Stdio::null());+ if let Some(path) = command_oracle {+ command.env("CUA_APPKIT_COMMAND_ORACLE", path);+ }+ let app = command.spawn().unwrap_or_else(|error| panic!("launch AppKit harness {exe:?}: {error}"));let pid = app.id();@@ -534,6 +541,116 @@ fn harness_appkit_element_foreground_press_key_commits_edit() {);}+#[test]+#[ignore]+fn harness_appkit_px_background_press_key_reports_honest_delivery_truth() {+ let case = native_background_case(+ "appkit",+ "press_key_command",+ Targeting::Px,+ DriverRoute::MacosCgEventPid,+ );+ let cell_id = case.cell_id.clone();+ execute_case(case, |evidence| {+ let mut driver = McpDriver::spawn_macos_daemon_proxy_named(&cell_id)+ .expect("start installed macOS daemon proxy");+ *evidence = recording_evidence(driver.recording_dir());+ let oracle_dir = tempfile::tempdir().expect("create command oracle directory");+ let oracle_path = oracle_dir.path().join("child-process-output.txt");+ let harness = Harness::launch_with_command_oracle(Some(&oracle_path));+ let (wid, _) = driver+ .find_window(harness.pid as i64, "CuaTestHarness AppKit")+ .expect("AppKit main window not found");++ let (_, passed) = run_with_background_oracles(+ &mut driver,+ TargetWindow {+ pid: harness.pid,+ native_id: wid,+ },+ |driver| {+ let first = snapshot_elements(driver, harness.pid, wid);+ let field = element_token_by_id(&first, "txt-input");+ let set = driver.call(+ "set_value",+ serde_json::json!({+ "pid": harness.pid as i64,+ "window_id": wid,+ "element_token": field,+ "value": "printf cua-press-key"+ }),+ );+ assert!(!set.is_error(), "set command failed: {}", set.text());++ let focused = snapshot_elements(driver, harness.pid, wid);+ let (x, y, width, height) = element_pixel_frame(&focused, "txt-input");+ let pressed = driver.call(+ "press_key",+ serde_json::json!({+ "pid": harness.pid as i64,+ "window_id": wid,+ "x": x + width / 2.0,+ "y": y + height / 2.0,+ "key": "return",+ "delivery_mode": "background"+ }),+ );+ assert!(+ !pressed.is_error(),+ "background Return failed: {}",+ pressed.text()+ );+ assert_eq!(pressed.action_route(), Some("synthetic_events"));+ assert_eq!(pressed.action_delivery_mode(), Some("background"));+ assert_eq!(pressed.action_effect(), Some("unverifiable"));+ assert!(+ pressed.structured()["escalation"].is_null(),+ "accepted post without a positive oracle must not claim delivery_failed: {}",+ pressed.raw+ );++ let deadline = std::time::Instant::now() + Duration::from_secs(3);+ loop {+ if std::fs::read_to_string(&oracle_path)+ .is_ok_and(|value| value == "cua-press-key")+ {+ break;+ }+ assert!(+ std::time::Instant::now() < deadline,+ "background Return did not execute the controlled child process"+ );+ std::thread::sleep(Duration::from_millis(25));+ }++ let mut exited = Command::new("/usr/bin/true")+ .spawn()+ .expect("spawn posting-failure fixture");+ let exited_pid = exited.id();+ exited.wait().expect("wait for posting-failure fixture");+ let failed = driver.call(+ "press_key",+ serde_json::json!({+ "pid": exited_pid,+ // An explicit target bypasses the PID-only window resolver so+ // this negative oracle reaches the posting preflight. Without+ // one, the earlier and equally truthful result is+ // window_target_not_found because /usr/bin/true owns no window.+ "window_id": wid,+ "key": "return",+ "delivery_mode": "background"+ }),+ );+ assert!(failed.is_error(), "dead-pid post unexpectedly succeeded");+ assert_eq!(failed.structured()["code"], "delivery_failed");+ },+ )+ .unwrap_or_else(|error| panic!("background desktop contract failed: {error}"));++ Observation::delivered_with_fixture_state(passed)+ });+}+#[test]#[ignore]fn harness_appkit_modified_click_preserves_selection() {diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/press_key.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/press_key.rsindex 18e70f8467..d1ad65c245 100644--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/press_key.rs+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/press_key.rs@@ -1,6 +1,11 @@use async_trait::async_trait;+use core_foundation::base::CFRelease;use cua_driver_contract::PressKeyInput;use cua_driver_core::{+ action_record::{+ ActionEffect, ActionEvidence, ActionExecutionRecord, ActionTransport, ActualDelivery,+ EvidenceKind, RequestedDelivery,+ },protocol::ToolResult,tool::{Tool, ToolDef},tool_args::parse_typed_projection,@@ -10,6 +15,9 @@ use serde_json::Value;use std::sync::Arc;use crate::apps;+use crate::ax::bindings::{+ copy_bool_attr, copy_string_attr, focused_element_of_pid, AXUIElementRef,+};use crate::focus_guard;use crate::window_change_detector::WindowChangeDetector;@@ -27,6 +35,143 @@ impl PressKeyTool {static DEF: std::sync::OnceLock<ToolDef> = std::sync::OnceLock::new();+#[derive(Clone, Debug, Eq, PartialEq)]+struct AxKeyState {+ value: Option<String>,+ selected: Option<bool>,+}++#[derive(Debug)]+enum PressKeyDeliveryOutcome {+ Confirmed,+ Unverifiable,+ Failed(anyhow::Error),+}++fn map_delivery_outcome(result: anyhow::Result<bool>) -> PressKeyDeliveryOutcome {+ match result {+ Ok(true) => PressKeyDeliveryOutcome::Confirmed,+ Ok(false) => PressKeyDeliveryOutcome::Unverifiable,+ Err(error) => PressKeyDeliveryOutcome::Failed(error),+ }+}++fn validate_post_target(pid: i32) -> anyhow::Result<()> {+ if pid <= 0 {+ anyhow::bail!("target pid {pid} is invalid");+ }+ // Both SLEventPostToPid and CGEventPostToPid are void APIs. A successful+ // call proves only that the request was accepted for posting, not that the+ // target consumed it. Reject the one positive pre-post failure oracle macOS+ // exposes: a process that no longer exists. EPERM still proves liveness.+ let status = unsafe { libc::kill(pid, 0) };+ if status == 0 {+ return Ok(());+ }+ let error = std::io::Error::last_os_error();+ if error.raw_os_error() == Some(libc::EPERM) {+ Ok(())+ } else {+ anyhow::bail!("target pid {pid} is not available for event posting: {error}")+ }+}++fn read_ax_key_state(pid: i32, element_ptr: usize) -> Option<AxKeyState> {+ if super::type_text::target_in_web_area(pid, Some((element_ptr, None))) {+ return None;+ }+ let element = element_ptr as AXUIElementRef;+ let state = AxKeyState {+ value: unsafe { copy_string_attr(element, "AXValue") },+ selected: unsafe { copy_bool_attr(element, "AXSelected") },+ };+ (state.value.is_some() || state.selected.is_some()).then_some(state)+}++fn dispatch_with_ax_oracle(+ pid: i32,+ explicit_element_ptr: Option<usize>,+ dispatch: impl FnOnce() -> anyhow::Result<()>,+) -> anyhow::Result<bool> {+ let (element_ptr, owns_element) = match explicit_element_ptr {+ Some(ptr) => (Some(ptr), false),+ None => unsafe { focused_element_of_pid(pid) }+ .map(|element| (Some(element as usize), true))+ .unwrap_or((None, false)),+ };+ let before = element_ptr.and_then(|ptr| read_ax_key_state(pid, ptr));+ let result = dispatch();+ // Native controls normally publish their new value/selection on the next+ // run-loop turn. Keep this bounded and reuse the exact retained element so+ // a focus move cannot become false confirmation from a different control.+ if before.is_some() {+ std::thread::sleep(std::time::Duration::from_millis(60));+ }+ let after = element_ptr.and_then(|ptr| read_ax_key_state(pid, ptr));+ if owns_element {+ if let Some(ptr) = element_ptr {+ unsafe { CFRelease(ptr as _) };+ }+ }+ result?;+ Ok(matches!((before, after), (Some(before), Some(after)) if ax_state_changed(&before, &after)))+}++fn ax_state_changed(before: &AxKeyState, after: &AxKeyState) -> bool {+ matches!((&before.value, &after.value), (Some(before), Some(after)) if before != after)+ || matches!((before.selected, after.selected), (Some(before), Some(after)) if before != after)+}++fn action_record(confirmed: bool, foreground: bool) -> ActionExecutionRecord {+ let effect = if confirmed {+ ActionEffect::Confirmed+ } else {+ ActionEffect::Unverifiable+ };+ let transport = if foreground {+ ActionTransport::MacosCgEventHid+ } else {+ ActionTransport::MacosCgEventPid+ };+ let requested = if foreground {+ RequestedDelivery::Foreground+ } else {+ RequestedDelivery::Background+ };+ let actual = if foreground {+ ActualDelivery::Foreground+ } else {+ ActualDelivery::Background+ };+ let mut record =+ ActionExecutionRecord::builder(effect, transport, requested).actual_delivery(actual);+ if confirmed {+ record = record.evidence(ActionEvidence {+ kind: EvidenceKind::AccessibilityReadback,+ detail: "the same native AX element changed value or selection after the key post"+ .into(),+ });+ } else {+ record = record.evidence(ActionEvidence {+ kind: EvidenceKind::NativeApiResult,+ detail: if foreground {+ "the key events were constructed and the foreground HID post was attempted".into()+ } else {+ "the key events were constructed and the PID-routed post was attempted".into()+ },+ });+ }+ record.build().expect("press_key record is valid")+}++fn delivery_failed(error: impl std::fmt::Display) -> ToolResult {+ let message = format!("press_key delivery failed: {error}");+ ToolResult::error(&message).with_structured(serde_json::json!({+ "code": "delivery_failed",+ "message": message,+ }))+}+fn def() -> &'static ToolDef {DEF.get_or_init(|| ToolDef {name: "press_key".into(),@@ -39,8 +184,10 @@ fn def() -> &'static ToolDef {element when supplied, send a genuine HID key transition so Chromium content, \inline editors, and native menu equivalents receive it, then restore prior \frontmost. Requires window_id.\n\n\- A key press is never driver-verifiable → effect:\"unverifiable\"; confirm via \- screenshot. Key names: return, tab, escape, up/down/left/right, space, delete, \+ A key press is confirmed only when a bounded native AX value/selection read-back \+ changes on the same control. Otherwise a successfully attempted post remains \+ effect:\"unverifiable\" without implying delivery failure or recommending foreground. \+ Key names: return, tab, escape, up/down/left/right, space, delete, \home, end, pageup, pagedown, f1-f12, plus any letter or digit. \Modifiers array: cmd, shift, option/alt, ctrl, fn.".into(),input_schema: serde_json::json!({@@ -141,6 +288,10 @@ impl Tool for PressKeyTool {} => (Some(idx), wid),};+ if let Err(error) = validate_post_target(pid) {+ return delivery_failed(error);+ }+// Remap "+" / "plus" → "=" + Shift (same physical key on US layout).let key = if key_raw == "+" || key_raw == "plus" {if !modifiers.iter().any(|m| m.eq_ignore_ascii_case("shift")) {@@ -247,24 +398,28 @@ impl Tool for PressKeyTool {"delivery_mode=foreground requires window_id for press_key")})?;- crate::input::skylight::with_foreground_hid_activation(- pid as libc::pid_t,- wid,- || {- // Activation can change the first responder, so- // repeat the best-effort AX focus write inside- // the guarded foreground interval immediately- // before the physical key transition.- if let Some(element_ptr) = pre_focus_ptr {- let _ = crate::input::ax_actions::focus_element(element_ptr);- }- crate::input::keyboard::press_key_bare_global(&key, &m)- },- )?;- return Ok(());+ return dispatch_with_ax_oracle(pid, pre_focus_ptr, || {+ crate::input::skylight::with_foreground_hid_activation(+ pid as libc::pid_t,+ wid,+ || {+ // Activation can change the first responder, so+ // repeat the best-effort AX focus write inside+ // the guarded foreground interval immediately+ // before the physical key transition.+ if let Some(element_ptr) = pre_focus_ptr {+ let _ =+ crate::input::ax_actions::focus_element(element_ptr);+ }+ crate::input::keyboard::press_key_bare_global(&key, &m)+ },+ )+ });}// background (default): auth-envelope post, no raise.- crate::input::keyboard::press_key(pid, &key, &m)+ dispatch_with_ax_oracle(pid, pre_focus_ptr, || {+ crate::input::keyboard::press_key(pid, &key, &m)+ })}).await},@@ -273,34 +428,92 @@ impl Tool for PressKeyTool {let changes = super::finish_window_observation(snapshot, &args).await;- match result {- Ok(Ok(())) => {+ let delivery_outcome = match result {+ Ok(result) => map_delivery_outcome(result),+ Err(error) => {+ PressKeyDeliveryOutcome::Failed(anyhow::anyhow!("posting task failed: {error}"))+ }+ };++ match delivery_outcome {+ outcome @ (PressKeyDeliveryOutcome::Confirmed+ | PressKeyDeliveryOutcome::Unverifiable) => {+ let confirmed = matches!(outcome, PressKeyDeliveryOutcome::Confirmed);let label = if fg {" (delivery_mode:foreground)"} else {""};- let mut structured = serde_json::json!({+ let structured = serde_json::json!({"path": if fg { "key_events_fg" } else { "key_events" },- "verified": false,- "effect": "unverifiable",+ "verified": confirmed,+ "effect": if confirmed { "confirmed" } else { "unverifiable" },});- if !fg && window_id.is_some() && element_index.is_none() {- structured["escalation"] = serde_json::json!({- "recommended": "foreground",- "reason": "a background menu key didn't land? re-call with \- delivery_mode:\"foreground\". (To type into a field, \- pixel-click to focus then type_text.)"- });- }ToolResult::text(format!("✅ Pressed {display_key} on pid {pid}{label}.{}",changes.result_suffix())).with_structured(structured)+ .with_action_record(action_record(confirmed, fg))}- Ok(Err(e)) => ToolResult::error(format!("press_key failed: {e}")),- Err(e) => ToolResult::error(format!("Task error: {e}")),+ PressKeyDeliveryOutcome::Failed(error) => delivery_failed(error),}}}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn delivery_outcome_mapper_distinguishes_confirmed_unverifiable_and_failed() {+ assert!(matches!(+ map_delivery_outcome(Ok(true)),+ PressKeyDeliveryOutcome::Confirmed+ ));+ assert!(matches!(+ map_delivery_outcome(Ok(false)),+ PressKeyDeliveryOutcome::Unverifiable+ ));+ let failed = map_delivery_outcome(Err(anyhow::anyhow!("post rejected")));+ assert!(matches!(failed, PressKeyDeliveryOutcome::Failed(_)));+ }++ #[test]+ fn accepted_without_oracle_has_no_escalation_but_ax_change_confirms() {+ let unverifiable = action_record(false, false).public_result().unwrap();+ assert_eq!(+ unverifiable.effect,+ cua_driver_contract::ActionEffect::Unverifiable+ );+ assert_eq!(+ unverifiable.delivery.unwrap().mode,+ cua_driver_contract::ActionDeliveryMode::Background+ );+ assert!(unverifiable.escalation.is_none());++ let confirmed = action_record(true, false).public_result().unwrap();+ assert_eq!(+ confirmed.effect,+ cua_driver_contract::ActionEffect::Confirmed+ );+ assert_eq!(confirmed.evidence.unwrap().len(), 1);+ assert!(confirmed.escalation.is_none());+ }++ #[test]+ fn definitely_dead_pid_is_a_typed_delivery_failure() {+ let child = std::process::Command::new("/usr/bin/true")+ .spawn()+ .expect("spawn short-lived child");+ let pid = child.id() as i32;+ let mut child = child;+ child.wait().expect("wait for child exit");+ let failure = delivery_failed(validate_post_target(pid).unwrap_err());+ assert_eq!(failure.is_error, Some(true));+ assert_eq!(+ failure.structured_content.unwrap()["code"],+ "delivery_failed"+ );+ }+}diff --git a/libs/cua-driver/tests/fixtures/apps/macos/appkit/main.swift b/libs/cua-driver/tests/fixtures/apps/macos/appkit/main.swiftindex fade434faf..7251cd8f49 100644--- a/libs/cua-driver/tests/fixtures/apps/macos/appkit/main.swift+++ b/libs/cua-driver/tests/fixtures/apps/macos/appkit/main.swift@@ -8,6 +8,7 @@// counter — NSButton increments NSTextField counter// text_body — NSTextField with the shared HARNESS_TEXT_MARKER_v1// text_input — NSTextField with a mirror label for type_text / set_value+// and an opt-in controlled child-process command oracle// click_target — NSButton (AX-addressable) records click/double_click/right_click// slider — NSSlider drives drag / set_value (slider_value=)// checkable_controls — NSButton checkbox (agreed=)@@ -438,6 +439,30 @@ final class HarnessWindowController: NSObject, NSTextFieldDelegate, NSTableViewDguard let field = obj.object as? NSTextField else { return }if field === textInput {textInputCommit.stringValue = "committed=\(field.stringValue)"+ runControlledCommand(field.stringValue)+ }+ }++ /// Test-only terminal-like command seam. It accepts exactly one harmless+ /// synthetic command and has a child process create the external oracle;+ /// arbitrary field contents are never executed.+ private func runControlledCommand(_ command: String) {+ guard command == "printf cua-press-key",+ let oraclePath = ProcessInfo.processInfo.environment["CUA_APPKIT_COMMAND_ORACLE"]+ else { return }++ let process = Process()+ process.executableURL = URL(fileURLWithPath: "/bin/sh")+ process.arguments = [+ "-c",+ "printf cua-press-key > \"$1\"",+ "cua-appkit-command",+ oraclePath,+ ]+ do {+ try process.run()+ } catch {+ textInputCommit.stringValue = "command_error=launch_failed"}}