Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

fix(cua-driver): report macOS key delivery truthfully (#2830)

Francesco Bonacci3dc0fc0e345f
diff --git a/docs/content/docs/reference/cua-driver/mcp-tools.mdx b/docs/content/docs/reference/cua-driver/mcp-tools.mdx
index 6624757045..c862c5ac5c 100644
--- a/docs/content/docs/reference/cua-driver/mcp-tools.mdx
+++ b/docs/content/docs/reference/cua-driver/mcp-tools.mdx
@@ -368,7 +368,7 @@ Press and release a single key. Follows the same `delivery_mode` ladder as click
• `background` (default): post to the pid WITHOUT fronting/raising — the auth-message path (Chromium-safe). With element_index it focuses that AX element first. `window_id` only targets; it does not raise.
• `foreground`: guard and briefly front the exact window, focus an addressed AX element when supplied, send a genuine HID key transition so Chromium content, inline editors, and native menu equivalents receive it, then restore prior frontmost. Requires window_id.
-A key press is never driver-verifiable → effect:"unverifiable"; confirm via screenshot. Key names: return, tab, escape, up/down/left/right, space, delete, home, end, pageup, pagedown, f1-f12, plus any letter or digit. Modifiers array: cmd, shift, option/alt, ctrl, fn.
+A key press is confirmed only when a bounded native AX value/selection read-back changes on the same control. Otherwise a successfully attempted post remains effect:"unverifiable" without implying delivery failure or recommending foreground. Key names: return, tab, escape, up/down/left/right, space, delete, home, end, pageup, pagedown, f1-f12, plus any letter or digit. Modifiers array: cmd, shift, option/alt, ctrl, fn.
**Arguments:**
diff --git a/libs/cua-driver/rust/crates/cua-driver/tests/harness_appkit_test.rs b/libs/cua-driver/rust/crates/cua-driver/tests/harness_appkit_test.rs
index fb46502f1a..b52a05b395 100644
--- a/libs/cua-driver/rust/crates/cua-driver/tests/harness_appkit_test.rs
+++ b/libs/cua-driver/rust/crates/cua-driver/tests/harness_appkit_test.rs
@@ -24,7 +24,7 @@
#![cfg(target_os = "macos")]
-use std::path::PathBuf;
+use std::path::{Path, PathBuf};
use std::process::{Child, Command, Stdio};
use std::time::Duration;
@@ -62,6 +62,10 @@ struct Harness {
impl Harness {
fn launch() -> Self {
+ Self::launch_with_command_oracle(None)
+ }
+
+ fn launch_with_command_oracle(command_oracle: Option<&Path>) -> Self {
let exe = harness_exe();
assert!(
exe.exists(),
@@ -70,9 +74,12 @@ impl Harness {
// Launch the binary directly (not via `open`) so we control the pid
// and can kill it cleanly on Drop. The app still installs an AppKit
// window via NSApp.run().
- let app = Command::new(&exe)
- .stdout(Stdio::null())
- .stderr(Stdio::null())
+ let mut command = Command::new(&exe);
+ command.stdout(Stdio::null()).stderr(Stdio::null());
+ if let Some(path) = command_oracle {
+ command.env("CUA_APPKIT_COMMAND_ORACLE", path);
+ }
+ let app = command
.spawn()
.unwrap_or_else(|error| panic!("launch AppKit harness {exe:?}: {error}"));
let pid = app.id();
@@ -534,6 +541,116 @@ fn harness_appkit_element_foreground_press_key_commits_edit() {
);
}
+#[test]
+#[ignore]
+fn harness_appkit_px_background_press_key_reports_honest_delivery_truth() {
+ let case = native_background_case(
+ "appkit",
+ "press_key_command",
+ Targeting::Px,
+ DriverRoute::MacosCgEventPid,
+ );
+ let cell_id = case.cell_id.clone();
+ execute_case(case, |evidence| {
+ let mut driver = McpDriver::spawn_macos_daemon_proxy_named(&cell_id)
+ .expect("start installed macOS daemon proxy");
+ *evidence = recording_evidence(driver.recording_dir());
+ let oracle_dir = tempfile::tempdir().expect("create command oracle directory");
+ let oracle_path = oracle_dir.path().join("child-process-output.txt");
+ let harness = Harness::launch_with_command_oracle(Some(&oracle_path));
+ let (wid, _) = driver
+ .find_window(harness.pid as i64, "CuaTestHarness AppKit")
+ .expect("AppKit main window not found");
+
+ let (_, passed) = run_with_background_oracles(
+ &mut driver,
+ TargetWindow {
+ pid: harness.pid,
+ native_id: wid,
+ },
+ |driver| {
+ let first = snapshot_elements(driver, harness.pid, wid);
+ let field = element_token_by_id(&first, "txt-input");
+ let set = driver.call(
+ "set_value",
+ serde_json::json!({
+ "pid": harness.pid as i64,
+ "window_id": wid,
+ "element_token": field,
+ "value": "printf cua-press-key"
+ }),
+ );
+ assert!(!set.is_error(), "set command failed: {}", set.text());
+
+ let focused = snapshot_elements(driver, harness.pid, wid);
+ let (x, y, width, height) = element_pixel_frame(&focused, "txt-input");
+ let pressed = driver.call(
+ "press_key",
+ serde_json::json!({
+ "pid": harness.pid as i64,
+ "window_id": wid,
+ "x": x + width / 2.0,
+ "y": y + height / 2.0,
+ "key": "return",
+ "delivery_mode": "background"
+ }),
+ );
+ assert!(
+ !pressed.is_error(),
+ "background Return failed: {}",
+ pressed.text()
+ );
+ assert_eq!(pressed.action_route(), Some("synthetic_events"));
+ assert_eq!(pressed.action_delivery_mode(), Some("background"));
+ assert_eq!(pressed.action_effect(), Some("unverifiable"));
+ assert!(
+ pressed.structured()["escalation"].is_null(),
+ "accepted post without a positive oracle must not claim delivery_failed: {}",
+ pressed.raw
+ );
+
+ let deadline = std::time::Instant::now() + Duration::from_secs(3);
+ loop {
+ if std::fs::read_to_string(&oracle_path)
+ .is_ok_and(|value| value == "cua-press-key")
+ {
+ break;
+ }
+ assert!(
+ std::time::Instant::now() < deadline,
+ "background Return did not execute the controlled child process"
+ );
+ std::thread::sleep(Duration::from_millis(25));
+ }
+
+ let mut exited = Command::new("/usr/bin/true")
+ .spawn()
+ .expect("spawn posting-failure fixture");
+ let exited_pid = exited.id();
+ exited.wait().expect("wait for posting-failure fixture");
+ let failed = driver.call(
+ "press_key",
+ serde_json::json!({
+ "pid": exited_pid,
+ // An explicit target bypasses the PID-only window resolver so
+ // this negative oracle reaches the posting preflight. Without
+ // one, the earlier and equally truthful result is
+ // window_target_not_found because /usr/bin/true owns no window.
+ "window_id": wid,
+ "key": "return",
+ "delivery_mode": "background"
+ }),
+ );
+ assert!(failed.is_error(), "dead-pid post unexpectedly succeeded");
+ assert_eq!(failed.structured()["code"], "delivery_failed");
+ },
+ )
+ .unwrap_or_else(|error| panic!("background desktop contract failed: {error}"));
+
+ Observation::delivered_with_fixture_state(passed)
+ });
+}
+
#[test]
#[ignore]
fn harness_appkit_modified_click_preserves_selection() {
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/press_key.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/press_key.rs
index 18e70f8467..d1ad65c245 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/press_key.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/press_key.rs
@@ -1,6 +1,11 @@
use async_trait::async_trait;
+use core_foundation::base::CFRelease;
use cua_driver_contract::PressKeyInput;
use cua_driver_core::{
+ action_record::{
+ ActionEffect, ActionEvidence, ActionExecutionRecord, ActionTransport, ActualDelivery,
+ EvidenceKind, RequestedDelivery,
+ },
protocol::ToolResult,
tool::{Tool, ToolDef},
tool_args::parse_typed_projection,
@@ -10,6 +15,9 @@ use serde_json::Value;
use std::sync::Arc;
use crate::apps;
+use crate::ax::bindings::{
+ copy_bool_attr, copy_string_attr, focused_element_of_pid, AXUIElementRef,
+};
use crate::focus_guard;
use crate::window_change_detector::WindowChangeDetector;
@@ -27,6 +35,143 @@ impl PressKeyTool {
static DEF: std::sync::OnceLock<ToolDef> = std::sync::OnceLock::new();
+#[derive(Clone, Debug, Eq, PartialEq)]
+struct AxKeyState {
+ value: Option<String>,
+ selected: Option<bool>,
+}
+
+#[derive(Debug)]
+enum PressKeyDeliveryOutcome {
+ Confirmed,
+ Unverifiable,
+ Failed(anyhow::Error),
+}
+
+fn map_delivery_outcome(result: anyhow::Result<bool>) -> PressKeyDeliveryOutcome {
+ match result {
+ Ok(true) => PressKeyDeliveryOutcome::Confirmed,
+ Ok(false) => PressKeyDeliveryOutcome::Unverifiable,
+ Err(error) => PressKeyDeliveryOutcome::Failed(error),
+ }
+}
+
+fn validate_post_target(pid: i32) -> anyhow::Result<()> {
+ if pid <= 0 {
+ anyhow::bail!("target pid {pid} is invalid");
+ }
+ // Both SLEventPostToPid and CGEventPostToPid are void APIs. A successful
+ // call proves only that the request was accepted for posting, not that the
+ // target consumed it. Reject the one positive pre-post failure oracle macOS
+ // exposes: a process that no longer exists. EPERM still proves liveness.
+ let status = unsafe { libc::kill(pid, 0) };
+ if status == 0 {
+ return Ok(());
+ }
+ let error = std::io::Error::last_os_error();
+ if error.raw_os_error() == Some(libc::EPERM) {
+ Ok(())
+ } else {
+ anyhow::bail!("target pid {pid} is not available for event posting: {error}")
+ }
+}
+
+fn read_ax_key_state(pid: i32, element_ptr: usize) -> Option<AxKeyState> {
+ if super::type_text::target_in_web_area(pid, Some((element_ptr, None))) {
+ return None;
+ }
+ let element = element_ptr as AXUIElementRef;
+ let state = AxKeyState {
+ value: unsafe { copy_string_attr(element, "AXValue") },
+ selected: unsafe { copy_bool_attr(element, "AXSelected") },
+ };
+ (state.value.is_some() || state.selected.is_some()).then_some(state)
+}
+
+fn dispatch_with_ax_oracle(
+ pid: i32,
+ explicit_element_ptr: Option<usize>,
+ dispatch: impl FnOnce() -> anyhow::Result<()>,
+) -> anyhow::Result<bool> {
+ let (element_ptr, owns_element) = match explicit_element_ptr {
+ Some(ptr) => (Some(ptr), false),
+ None => unsafe { focused_element_of_pid(pid) }
+ .map(|element| (Some(element as usize), true))
+ .unwrap_or((None, false)),
+ };
+ let before = element_ptr.and_then(|ptr| read_ax_key_state(pid, ptr));
+ let result = dispatch();
+ // Native controls normally publish their new value/selection on the next
+ // run-loop turn. Keep this bounded and reuse the exact retained element so
+ // a focus move cannot become false confirmation from a different control.
+ if before.is_some() {
+ std::thread::sleep(std::time::Duration::from_millis(60));
+ }
+ let after = element_ptr.and_then(|ptr| read_ax_key_state(pid, ptr));
+ if owns_element {
+ if let Some(ptr) = element_ptr {
+ unsafe { CFRelease(ptr as _) };
+ }
+ }
+ result?;
+ Ok(matches!((before, after), (Some(before), Some(after)) if ax_state_changed(&before, &after)))
+}
+
+fn ax_state_changed(before: &AxKeyState, after: &AxKeyState) -> bool {
+ matches!((&before.value, &after.value), (Some(before), Some(after)) if before != after)
+ || matches!((before.selected, after.selected), (Some(before), Some(after)) if before != after)
+}
+
+fn action_record(confirmed: bool, foreground: bool) -> ActionExecutionRecord {
+ let effect = if confirmed {
+ ActionEffect::Confirmed
+ } else {
+ ActionEffect::Unverifiable
+ };
+ let transport = if foreground {
+ ActionTransport::MacosCgEventHid
+ } else {
+ ActionTransport::MacosCgEventPid
+ };
+ let requested = if foreground {
+ RequestedDelivery::Foreground
+ } else {
+ RequestedDelivery::Background
+ };
+ let actual = if foreground {
+ ActualDelivery::Foreground
+ } else {
+ ActualDelivery::Background
+ };
+ let mut record =
+ ActionExecutionRecord::builder(effect, transport, requested).actual_delivery(actual);
+ if confirmed {
+ record = record.evidence(ActionEvidence {
+ kind: EvidenceKind::AccessibilityReadback,
+ detail: "the same native AX element changed value or selection after the key post"
+ .into(),
+ });
+ } else {
+ record = record.evidence(ActionEvidence {
+ kind: EvidenceKind::NativeApiResult,
+ detail: if foreground {
+ "the key events were constructed and the foreground HID post was attempted".into()
+ } else {
+ "the key events were constructed and the PID-routed post was attempted".into()
+ },
+ });
+ }
+ record.build().expect("press_key record is valid")
+}
+
+fn delivery_failed(error: impl std::fmt::Display) -> ToolResult {
+ let message = format!("press_key delivery failed: {error}");
+ ToolResult::error(&message).with_structured(serde_json::json!({
+ "code": "delivery_failed",
+ "message": message,
+ }))
+}
+
fn def() -> &'static ToolDef {
DEF.get_or_init(|| ToolDef {
name: "press_key".into(),
@@ -39,8 +184,10 @@ fn def() -> &'static ToolDef {
element when supplied, send a genuine HID key transition so Chromium content, \
inline editors, and native menu equivalents receive it, then restore prior \
frontmost. Requires window_id.\n\n\
- A key press is never driver-verifiable → effect:\"unverifiable\"; confirm via \
- screenshot. Key names: return, tab, escape, up/down/left/right, space, delete, \
+ A key press is confirmed only when a bounded native AX value/selection read-back \
+ changes on the same control. Otherwise a successfully attempted post remains \
+ effect:\"unverifiable\" without implying delivery failure or recommending foreground. \
+ Key names: return, tab, escape, up/down/left/right, space, delete, \
home, end, pageup, pagedown, f1-f12, plus any letter or digit. \
Modifiers array: cmd, shift, option/alt, ctrl, fn.".into(),
input_schema: serde_json::json!({
@@ -141,6 +288,10 @@ impl Tool for PressKeyTool {
} => (Some(idx), wid),
};
+ if let Err(error) = validate_post_target(pid) {
+ return delivery_failed(error);
+ }
+
// Remap "+" / "plus" → "=" + Shift (same physical key on US layout).
let key = if key_raw == "+" || key_raw == "plus" {
if !modifiers.iter().any(|m| m.eq_ignore_ascii_case("shift")) {
@@ -247,24 +398,28 @@ impl Tool for PressKeyTool {
"delivery_mode=foreground requires window_id for press_key"
)
})?;
- crate::input::skylight::with_foreground_hid_activation(
- pid as libc::pid_t,
- wid,
- || {
- // Activation can change the first responder, so
- // repeat the best-effort AX focus write inside
- // the guarded foreground interval immediately
- // before the physical key transition.
- if let Some(element_ptr) = pre_focus_ptr {
- let _ = crate::input::ax_actions::focus_element(element_ptr);
- }
- crate::input::keyboard::press_key_bare_global(&key, &m)
- },
- )?;
- return Ok(());
+ return dispatch_with_ax_oracle(pid, pre_focus_ptr, || {
+ crate::input::skylight::with_foreground_hid_activation(
+ pid as libc::pid_t,
+ wid,
+ || {
+ // Activation can change the first responder, so
+ // repeat the best-effort AX focus write inside
+ // the guarded foreground interval immediately
+ // before the physical key transition.
+ if let Some(element_ptr) = pre_focus_ptr {
+ let _ =
+ crate::input::ax_actions::focus_element(element_ptr);
+ }
+ crate::input::keyboard::press_key_bare_global(&key, &m)
+ },
+ )
+ });
}
// background (default): auth-envelope post, no raise.
- crate::input::keyboard::press_key(pid, &key, &m)
+ dispatch_with_ax_oracle(pid, pre_focus_ptr, || {
+ crate::input::keyboard::press_key(pid, &key, &m)
+ })
})
.await
},
@@ -273,34 +428,92 @@ impl Tool for PressKeyTool {
let changes = super::finish_window_observation(snapshot, &args).await;
- match result {
- Ok(Ok(())) => {
+ let delivery_outcome = match result {
+ Ok(result) => map_delivery_outcome(result),
+ Err(error) => {
+ PressKeyDeliveryOutcome::Failed(anyhow::anyhow!("posting task failed: {error}"))
+ }
+ };
+
+ match delivery_outcome {
+ outcome @ (PressKeyDeliveryOutcome::Confirmed
+ | PressKeyDeliveryOutcome::Unverifiable) => {
+ let confirmed = matches!(outcome, PressKeyDeliveryOutcome::Confirmed);
let label = if fg {
" (delivery_mode:foreground)"
} else {
""
};
- let mut structured = serde_json::json!({
+ let structured = serde_json::json!({
"path": if fg { "key_events_fg" } else { "key_events" },
- "verified": false,
- "effect": "unverifiable",
+ "verified": confirmed,
+ "effect": if confirmed { "confirmed" } else { "unverifiable" },
});
- if !fg && window_id.is_some() && element_index.is_none() {
- structured["escalation"] = serde_json::json!({
- "recommended": "foreground",
- "reason": "a background menu key didn't land? re-call with \
- delivery_mode:\"foreground\". (To type into a field, \
- pixel-click to focus then type_text.)"
- });
- }
ToolResult::text(format!(
"✅ Pressed {display_key} on pid {pid}{label}.{}",
changes.result_suffix()
))
.with_structured(structured)
+ .with_action_record(action_record(confirmed, fg))
}
- Ok(Err(e)) => ToolResult::error(format!("press_key failed: {e}")),
- Err(e) => ToolResult::error(format!("Task error: {e}")),
+ PressKeyDeliveryOutcome::Failed(error) => delivery_failed(error),
}
}
}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn delivery_outcome_mapper_distinguishes_confirmed_unverifiable_and_failed() {
+ assert!(matches!(
+ map_delivery_outcome(Ok(true)),
+ PressKeyDeliveryOutcome::Confirmed
+ ));
+ assert!(matches!(
+ map_delivery_outcome(Ok(false)),
+ PressKeyDeliveryOutcome::Unverifiable
+ ));
+ let failed = map_delivery_outcome(Err(anyhow::anyhow!("post rejected")));
+ assert!(matches!(failed, PressKeyDeliveryOutcome::Failed(_)));
+ }
+
+ #[test]
+ fn accepted_without_oracle_has_no_escalation_but_ax_change_confirms() {
+ let unverifiable = action_record(false, false).public_result().unwrap();
+ assert_eq!(
+ unverifiable.effect,
+ cua_driver_contract::ActionEffect::Unverifiable
+ );
+ assert_eq!(
+ unverifiable.delivery.unwrap().mode,
+ cua_driver_contract::ActionDeliveryMode::Background
+ );
+ assert!(unverifiable.escalation.is_none());
+
+ let confirmed = action_record(true, false).public_result().unwrap();
+ assert_eq!(
+ confirmed.effect,
+ cua_driver_contract::ActionEffect::Confirmed
+ );
+ assert_eq!(confirmed.evidence.unwrap().len(), 1);
+ assert!(confirmed.escalation.is_none());
+ }
+
+ #[test]
+ fn definitely_dead_pid_is_a_typed_delivery_failure() {
+ let child = std::process::Command::new("/usr/bin/true")
+ .spawn()
+ .expect("spawn short-lived child");
+ let pid = child.id() as i32;
+ let mut child = child;
+ child.wait().expect("wait for child exit");
+ let failure = delivery_failed(validate_post_target(pid).unwrap_err());
+ assert_eq!(failure.is_error, Some(true));
+ assert_eq!(
+ failure.structured_content.unwrap()["code"],
+ "delivery_failed"
+ );
+ }
+}
diff --git a/libs/cua-driver/tests/fixtures/apps/macos/appkit/main.swift b/libs/cua-driver/tests/fixtures/apps/macos/appkit/main.swift
index fade434faf..7251cd8f49 100644
--- a/libs/cua-driver/tests/fixtures/apps/macos/appkit/main.swift
+++ b/libs/cua-driver/tests/fixtures/apps/macos/appkit/main.swift
@@ -8,6 +8,7 @@
// counter — NSButton increments NSTextField counter
// text_body — NSTextField with the shared HARNESS_TEXT_MARKER_v1
// text_input — NSTextField with a mirror label for type_text / set_value
+// and an opt-in controlled child-process command oracle
// click_target — NSButton (AX-addressable) records click/double_click/right_click
// slider — NSSlider drives drag / set_value (slider_value=)
// checkable_controls — NSButton checkbox (agreed=)
@@ -438,6 +439,30 @@ final class HarnessWindowController: NSObject, NSTextFieldDelegate, NSTableViewD
guard let field = obj.object as? NSTextField else { return }
if field === textInput {
textInputCommit.stringValue = "committed=\(field.stringValue)"
+ runControlledCommand(field.stringValue)
+ }
+ }
+
+ /// Test-only terminal-like command seam. It accepts exactly one harmless
+ /// synthetic command and has a child process create the external oracle;
+ /// arbitrary field contents are never executed.
+ private func runControlledCommand(_ command: String) {
+ guard command == "printf cua-press-key",
+ let oraclePath = ProcessInfo.processInfo.environment["CUA_APPKIT_COMMAND_ORACLE"]
+ else { return }
+
+ let process = Process()
+ process.executableURL = URL(fileURLWithPath: "/bin/sh")
+ process.arguments = [
+ "-c",
+ "printf cua-press-key > \"$1\"",
+ "cua-appkit-command",
+ oraclePath,
+ ]
+ do {
+ try process.run()
+ } catch {
+ textInputCommit.stringValue = "command_error=launch_failed"
}
}