Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
fix(cua-driver): accept element_token without pid; say what to do after refusals (#4739)
Francesco Bonacci3931962cc76f
diff --git a/docs/content/docs/cua-driver/reference/mcp-tools/click.mdx b/docs/content/docs/cua-driver/reference/mcp-tools/click.mdxindex 2acc0c96d6..0936c963d6 100644--- a/docs/content/docs/cua-driver/reference/mcp-tools/click.mdx+++ b/docs/content/docs/cua-driver/reference/mcp-tools/click.mdx@@ -40,7 +40,7 @@ Click a target pid. Prefer `element_token` (from get_window_state): it identifie| <span id="click--macos--delivery_mode"></span>`delivery_mode` | `"background" \| "foreground"` | | Default "background" (no fronting). "foreground" briefly fronts the window, acts, then restores the prior app; needs window_id and is required for modified clicks. Escalate in order: AX element_token, background pixel, foreground, checking state between rungs; a click has no read-back except list-row selection. || <span id="click--macos--from_zoom"></span>`from_zoom` | `boolean` | | x and y are in the last zoom image for this pid. || <span id="click--macos--modifier"></span>`modifier` | `string[]` | | Modifier keys: cmd, shift, option/alt, ctrl. |-| <span id="click--macos--pid"></span>`pid` | `integer` | | Target process ID. |+| <span id="click--macos--pid"></span>`pid` | `integer` | | Target process ID. Optional with element_token (it carries the pid). || <span id="click--macos--scope"></span>`scope` | `"window" \| "desktop"` | | Default "window". "desktop": x,y with no pid/window_id are true screen pixels from get_desktop_state. || <span id="click--macos--session"></span>`session` | `string` | | For multi-call work, prefer a short public session label and repeat it on every call that accepts it. || <span id="click--macos--window_id"></span>`window_id` | `integer` | | Target window ID. Omit with element_token. |diff --git a/docs/content/docs/cua-driver/reference/mcp-tools/pointer.mdx b/docs/content/docs/cua-driver/reference/mcp-tools/pointer.mdxindex e94ffa8885..66e973b49e 100644--- a/docs/content/docs/cua-driver/reference/mcp-tools/pointer.mdx+++ b/docs/content/docs/cua-driver/reference/mcp-tools/pointer.mdx@@ -136,6 +136,7 @@ Scroll the target pid.- Targeted wheel: pass `element_token` (preferred) or window-local `x, y`. Sends a real wheel event at that point, so it scrolls whatever is under it. The only way to scroll nested `overflow:auto` regions in web views.- Keystroke: no target, just pid + direction. PageDown/PageUp (by='page') or arrows (by='line') on the focused scroller.+Electron/Chromium windows refuse background delivery; pass `delivery_mode:"foreground"`.`amount` is wheel notches (targeted) or key repetitions (keystroke).**macOS parameters**@@ -144,7 +145,7 @@ Scroll the target pid.| --- | --- | --- | --- || <span id="scroll--macos--amount"></span>`amount` | `integer` | | Wheel notches or key repetitions, clamped to the maximum. Default 3. Range: 1 to 50. || <span id="scroll--macos--delivery_mode"></span>`delivery_mode` | `"background" \| "foreground"` | | Default "background": no fronting or focus steal. "foreground": briefly front the window, act, restore the prior app; last resort when background did not land. |-| <span id="scroll--macos--pid"></span>`pid` | `integer` | | Target process ID. Required unless scope is "desktop". |+| <span id="scroll--macos--pid"></span>`pid` | `integer` | | Target process ID. Required unless scope is "desktop" or element_token is supplied. || <span id="scroll--macos--scope"></span>`scope` | `"window" \| "desktop"` | `"window"` | "desktop" with x,y and no pid/window_id uses get_desktop_state coordinates. || <span id="scroll--macos--session"></span>`session` | `string` | | For multi-call work, prefer a short public session label and repeat it on every call that accepts it. || <span id="scroll--macos--window_id"></span>`window_id` | `integer` | | Window ID. Required with x/y; optional with element_token. |diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/snapshot_store.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/snapshot_store.rsindex e61290c894..ad613a7e2d 100644--- a/libs/cua-driver/rust/crates/cua-driver-core/src/snapshot_store.rs+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/snapshot_store.rs@@ -371,6 +371,22 @@ impl<S: SnapshotPayload> SnapshotStore<S> {})}+ /// The pid an `element_token` was minted for, found without the caller+ /// naming it. Snapshot ids are unique per runtime, so a token alone names+ /// its snapshot, window and process. `None` when the token is absent,+ /// malformed, or no longer current; callers then fall back to the+ /// ordinary missing-`pid` error.+ pub fn pid_for_token(&self, args: &serde_json::Value) -> Option<i32> {+ let token = args.get("element_token")?.as_str()?;+ let (snapshot_id, _) = parse_token(token)?;+ self.inner+ .lock()+ .unwrap()+ .iter()+ .find(|(_, lane)| lane.iter().any(|snapshot| snapshot.id == snapshot_id))+ .map(|(pid, _)| *pid)+ }+pub fn resolve(&self,pid: i32,@@ -510,6 +526,18 @@ mod tests {use crate::snapshot_test_support::Payload;use std::sync::atomic::{AtomicUsize, Ordering};+ #[test]+ fn pid_for_token_finds_the_owning_process() {+ let cache = SnapshotStore::new();+ let id = cache.publish(42, 7, Payload(vec![10, 20]));+ cache.publish(43, 8, Payload(vec![1]));+ let args = serde_json::json!({ "element_token": token_for(id, 1) });+ assert_eq!(cache.pid_for_token(&args), Some(42));+ assert_eq!(cache.pid_for_token(&serde_json::json!({})), None);+ let unknown = serde_json::json!({ "element_token": token_for(id + 1000, 0) });+ assert_eq!(cache.pid_for_token(&unknown), None);+ }+#[test]fn publish_then_resolve_returns_projection() {let cache = SnapshotStore::new();diff --git a/libs/cua-driver/rust/crates/platform-macos/src/input/keyboard.rs b/libs/cua-driver/rust/crates/platform-macos/src/input/keyboard.rsindex 5b8128e42e..7e7ac71cc9 100644--- a/libs/cua-driver/rust/crates/platform-macos/src/input/keyboard.rs+++ b/libs/cua-driver/rust/crates/platform-macos/src/input/keyboard.rs@@ -769,16 +769,53 @@ pub(super) fn key_name_to_code(key: &str) -> anyhow::Result<u16> {"m" => 46,"." => 47,"`" => 50,- _ => anyhow::bail!("Unknown key name: {key}"),+ _ => anyhow::bail!("{}", unknown_key_message(key)),};Ok(code)}+/// The error for a name `key_name_to_code` does not know, saying what to send+/// instead: a combination such as "shift+Right" is a key plus modifiers, and+/// a printable symbol is text.+fn unknown_key_message(key: &str) -> String {+ if key.chars().count() > 1 && key.contains('+') && !key.ends_with('+') {+ let parts: Vec<&str> = key.split('+').collect();+ let (modifiers, name) = parts.split_at(parts.len() - 1);+ return format!(+ "Unknown key name: {key}. A key name is one key, not a combination. Send the last \+ part as the key and the rest as modifiers, e.g. key:\"{}\" with modifiers:{:?}, \+ or use the hotkey tool with keys:{:?}.",+ name[0],+ modifiers+ .iter()+ .map(|m| m.to_lowercase())+ .collect::<Vec<_>>(),+ parts+ );+ }+ format!(+ "Unknown key name: {key}. Use a key name such as return, tab, escape, space, delete, \+ left, right, up, down, pageup, pagedown, home, end, f1..f12, or a single letter or \+ digit; to enter other characters (for example * or $) use type_text."+ )+}+#[cfg(test)]mod tests {use super::*;use core_graphics::event::CGEventType;+ #[test]+ fn unknown_key_errors_say_what_to_send_instead() {+ let combo = key_name_to_code("shift+Right").unwrap_err().to_string();+ assert!(combo.contains("modifiers"), "{combo}");+ assert!(combo.contains("hotkey"), "{combo}");+ assert!(combo.contains("key:\"Right\""), "{combo}");+ let symbol = key_name_to_code("asterisk").unwrap_err().to_string();+ assert!(symbol.contains("type_text"), "{symbol}");+ assert!(key_name_to_code("shift").is_ok());+ }+/// cua-spacesd names the Meta key `super` (the cross-platform name), and/// the macOS driver dropped it: `cua do hotkey cmd+w` pressed a bare `w`.#[test]diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/click.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/click.rsindex daefb4b914..28696c3ae0 100644--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/click.rs+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/click.rs@@ -241,7 +241,7 @@ fn def() -> &'static ToolDef {"required": [],"properties": {"session": { "type": "string", "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it." },- "pid": { "type": "integer", "description": "Target process ID." },+ "pid": { "type": "integer", "description": "Target process ID. Optional with element_token (it carries the pid)." },"window_id": { "type": "integer", "description": "Target window ID. Omit with element_token." },"element_token": cua_driver_core::tool_schema::element_token_schema(),"capture_id": { "type": "string", "description": "Optional capture ID from get_window_state or get_desktop_state. With x,y it is consumed atomically; stale or mismatched captures are refused." },@@ -433,7 +433,7 @@ impl Tool for ClickTool {};}- let pid = match args.require_i32("pid") {+ let pid = match super::target_pid(&self.state, &args) {Ok(v) => v,Err(e) => return e,};@@ -1327,7 +1327,7 @@ impl Tool for ClickTool {Err(e) => ToolResult::error(format!("Task error: {e}")),}} else {- ToolResult::error("Provide either element_token or (x + y). pid is always required.")+ ToolResult::error("Provide either element_token or (x + y). pid is required for x,y clicks; an element_token carries its own pid.")}}}diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/double_click.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/double_click.rsindex 206c8705c9..90a4e89d9a 100644--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/double_click.rs+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/double_click.rs@@ -73,7 +73,7 @@ impl Tool for DoubleClickTool {async fn invoke(&self, args: Value) -> ToolResult {use cua_driver_core::tool_args::ArgsExt;- let pid = match args.require_i32("pid") {+ let pid = match super::target_pid(&self.state, &args) {Ok(v) => v,Err(e) => return e,};diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/mod.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/mod.rsindex e0bcc43db8..fb884da3d0 100644--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/mod.rs+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/mod.rs@@ -686,6 +686,42 @@ impl ToolState {}}+/// The target pid of an element-addressable call. An explicit `pid` wins; a+/// call that carries only an `element_token` takes the pid the token was+/// minted for, as the schemas promise ("the token carries it").+pub(super) fn target_pid(+ state: &ToolState,+ args: &serde_json::Value,+) -> Result<i32, cua_driver_core::protocol::ToolResult> {+ use cua_driver_core::tool_args::ArgsExt;+ if args.get("pid").is_some_and(|pid| !pid.is_null()) {+ return args.require_i32("pid");+ }+ if let Some(pid) = state.snapshots.pid_for_token(args) {+ return Ok(pid);+ }+ if args+ .get("element_token")+ .is_some_and(|token| !token.is_null())+ {+ return Err(cua_driver_core::protocol::ToolResult::error(+ "element_token is stale or unknown; call get_window_state again to refresh \+ (a current token names its own pid).",+ )+ .with_structured(serde_json::json!({+ "status": "refused",+ "refusal": {+ "code": "stale_element_token",+ "message": "element_token is stale or unknown; call get_window_state again to refresh",+ },+ })));+ }+ Err(cua_driver_core::protocol::ToolResult::error(+ "Missing required integer field: pid. Pass pid, or pass an element_token from the \+ current get_window_state (a token names its own pid).",+ ))+}+pub(super) fn screenshot_scale(state: &ToolState,args: &serde_json::Value,diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/press_key.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/press_key.rsindex 74a2206549..9ea393dc2f 100644--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/press_key.rs+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/press_key.rs@@ -249,7 +249,7 @@ impl Tool for PressKeyTool {Err(error) => ToolResult::error(format!("desktop press_key task failed: {error}")),};}- let pid = match args.require_i32("pid") {+ let pid = match super::target_pid(&self.state, &args) {Ok(v) => v,Err(e) => return e,};diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/right_click.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/right_click.rsindex 9af941d8be..a45e01c3a5 100644--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/right_click.rs+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/right_click.rs@@ -82,7 +82,7 @@ impl Tool for RightClickTool {async fn invoke(&self, args: Value) -> ToolResult {use cua_driver_core::tool_args::ArgsExt;- let pid = match args.require_i32("pid") {+ let pid = match super::target_pid(&self.state, &args) {Ok(v) => v,Err(e) => return e,};diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/scroll.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/scroll.rsindex 7b8d94ed28..7a5efaed92 100644--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/scroll.rs+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/scroll.rs@@ -58,6 +58,12 @@ fn clamp_amount(requested: u64) -> usize {requested.clamp(AMOUNT_MIN, AMOUNT_MAX) as usize}+const ELECTRON_BACKGROUND_SCROLL_UNAVAILABLE: &str = "Background scroll is unavailable for \+ Electron/Chromium windows on macOS: they drop background wheel events and keystrokes, so \+ nothing was sent. Retry with delivery_mode:\"foreground\" (and the same window_id, \+ element_token or x,y); the driver briefly fronts that window, scrolls, and restores the \+ prior frontmost app.";+fn def() -> &'static ToolDef {DEF.get_or_init(|| ToolDef {name: "scroll".into(),@@ -65,6 +71,7 @@ fn def() -> &'static ToolDef {- Targeted wheel: pass `element_token` (preferred) or window-local `x, y`. Sends a real wheel event at that point, so it scrolls whatever is under it. The only way to scroll nested `overflow:auto` regions in web views.\n\- Keystroke: no target, just pid + direction. PageDown/PageUp (by='page') or arrows (by='line') on the focused scroller.\n\\n\+ Electron/Chromium windows refuse background delivery; pass `delivery_mode:\"foreground\"`.\n\`amount` is wheel notches (targeted) or key repetitions (keystroke).".into(),input_schema: serde_json::json!({"type": "object",@@ -73,7 +80,7 @@ fn def() -> &'static ToolDef {"required": ["direction"],"properties": {"session": { "type": "string", "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it." },- "pid": { "type": "integer", "description": "Target process ID. Required unless scope is \"desktop\"." },+ "pid": { "type": "integer", "description": "Target process ID. Required unless scope is \"desktop\" or element_token is supplied." },"direction": {"type": "string","enum": ["up", "down", "left", "right"],@@ -155,7 +162,7 @@ impl Tool for ScrollTool {Err(error) => ToolResult::error(format!("desktop scroll task failed: {error}")),};}- let pid = match args.require_i32("pid") {+ let pid = match super::target_pid(&self.state, &args) {Ok(v) => v,Err(e) => return e,};@@ -165,11 +172,12 @@ impl Tool for ScrollTool {// keystroke path is background-by-design and untouched.let delivery_mode = super::DeliveryMode::parse(args.opt_str("delivery_mode").as_deref());if !delivery_mode.is_foreground() && crate::browser::ElectronJs::is_electron(pid) {- return ToolResult::error(- "Background scroll is unavailable for Electron/Chromium windows on macOS."- .to_owned(),- )- .with_structured(serde_json::json!({ "code": "background_unavailable" }));+ return cua_driver_core::delivery::background_unavailable_result(+ ELECTRON_BACKGROUND_SCROLL_UNAVAILABLE,+ "background_unavailable",+ "Electron/Chromium windows drop background wheel events and keystrokes on macOS",+ serde_json::json!({ "effect": "refused" }),+ );}let direction = match args.require_str("direction") {Ok(v) => v,diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/set_value.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/set_value.rsindex ded77100d1..ac0a6d969f 100644--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/set_value.rs+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/set_value.rs@@ -81,7 +81,7 @@ impl Tool for SetValueTool {async fn invoke(&self, args: Value) -> ToolResult {use cua_driver_core::tool_args::ArgsExt;- let pid = match args.require_i32("pid") {+ let pid = match super::target_pid(&self.state, &args) {Ok(v) => v,Err(e) => return e,};diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/type_text.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/type_text.rsindex 663ebc9f45..9040f1c094 100644--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/type_text.rs+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/type_text.rs@@ -166,7 +166,7 @@ impl Tool for TypeTextTool {Err(error) => ToolResult::error(format!("desktop type_text task failed: {error}")),};}- let pid = match args.require_i32("pid") {+ let pid = match super::target_pid(&self.state, &args) {Ok(v) => v,Err(e) => return e,};diff --git a/scripts/docs-generators/cli-specs/cua-driver-mcp-macos.json b/scripts/docs-generators/cli-specs/cua-driver-mcp-macos.jsonindex 4b7b270507..22174142d4 100644--- a/scripts/docs-generators/cli-specs/cua-driver-mcp-macos.json+++ b/scripts/docs-generators/cli-specs/cua-driver-mcp-macos.json@@ -536,7 +536,7 @@"type": "array"},"pid": {- "description": "Target process ID.",+ "description": "Target process ID. Optional with element_token (it carries the pid).","type": "integer"},"scope": {@@ -1270,7 +1270,7 @@"read_only": false},{- "description": "Scroll the target pid.\n- Targeted wheel: pass `element_token` (preferred) or window-local `x, y`. Sends a real wheel event at that point, so it scrolls whatever is under it. The only way to scroll nested `overflow:auto` regions in web views.\n- Keystroke: no target, just pid + direction. PageDown/PageUp (by='page') or arrows (by='line') on the focused scroller.\n\n`amount` is wheel notches (targeted) or key repetitions (keystroke).",+ "description": "Scroll the target pid.\n- Targeted wheel: pass `element_token` (preferred) or window-local `x, y`. Sends a real wheel event at that point, so it scrolls whatever is under it. The only way to scroll nested `overflow:auto` regions in web views.\n- Keystroke: no target, just pid + direction. PageDown/PageUp (by='page') or arrows (by='line') on the focused scroller.\n\nElectron/Chromium windows refuse background delivery; pass `delivery_mode:\"foreground\"`.\n`amount` is wheel notches (targeted) or key repetitions (keystroke).","destructive": false,"idempotent": false,"input_schema": {@@ -1314,7 +1314,7 @@"type": "string"},"pid": {- "description": "Target process ID. Required unless scope is \"desktop\".",+ "description": "Target process ID. Required unless scope is \"desktop\" or element_token is supplied.","type": "integer"},"scope": {