Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

fix(cua-driver): accept element_token without pid; say what to do after refusals (#4739)

Francesco Bonacci3931962cc76f
diff --git a/docs/content/docs/cua-driver/reference/mcp-tools/click.mdx b/docs/content/docs/cua-driver/reference/mcp-tools/click.mdx
index 2acc0c96d6..0936c963d6 100644
--- a/docs/content/docs/cua-driver/reference/mcp-tools/click.mdx
+++ b/docs/content/docs/cua-driver/reference/mcp-tools/click.mdx
@@ -40,7 +40,7 @@ Click a target pid. Prefer `element_token` (from get_window_state): it identifie
| <span id="click--macos--delivery_mode"></span>`delivery_mode` | `"background" \| "foreground"` | | Default "background" (no fronting). "foreground" briefly fronts the window, acts, then restores the prior app; needs window_id and is required for modified clicks. Escalate in order: AX element_token, background pixel, foreground, checking state between rungs; a click has no read-back except list-row selection. |
| <span id="click--macos--from_zoom"></span>`from_zoom` | `boolean` | | x and y are in the last zoom image for this pid. |
| <span id="click--macos--modifier"></span>`modifier` | `string[]` | | Modifier keys: cmd, shift, option/alt, ctrl. |
-| <span id="click--macos--pid"></span>`pid` | `integer` | | Target process ID. |
+| <span id="click--macos--pid"></span>`pid` | `integer` | | Target process ID. Optional with element_token (it carries the pid). |
| <span id="click--macos--scope"></span>`scope` | `"window" \| "desktop"` | | Default "window". "desktop": x,y with no pid/window_id are true screen pixels from get_desktop_state. |
| <span id="click--macos--session"></span>`session` | `string` | | For multi-call work, prefer a short public session label and repeat it on every call that accepts it. |
| <span id="click--macos--window_id"></span>`window_id` | `integer` | | Target window ID. Omit with element_token. |
diff --git a/docs/content/docs/cua-driver/reference/mcp-tools/pointer.mdx b/docs/content/docs/cua-driver/reference/mcp-tools/pointer.mdx
index e94ffa8885..66e973b49e 100644
--- a/docs/content/docs/cua-driver/reference/mcp-tools/pointer.mdx
+++ b/docs/content/docs/cua-driver/reference/mcp-tools/pointer.mdx
@@ -136,6 +136,7 @@ Scroll the target pid.
- Targeted wheel: pass `element_token` (preferred) or window-local `x, y`. Sends a real wheel event at that point, so it scrolls whatever is under it. The only way to scroll nested `overflow:auto` regions in web views.
- Keystroke: no target, just pid + direction. PageDown/PageUp (by='page') or arrows (by='line') on the focused scroller.
+Electron/Chromium windows refuse background delivery; pass `delivery_mode:"foreground"`.
`amount` is wheel notches (targeted) or key repetitions (keystroke).
**macOS parameters**
@@ -144,7 +145,7 @@ Scroll the target pid.
| --- | --- | --- | --- |
| <span id="scroll--macos--amount"></span>`amount` | `integer` | | Wheel notches or key repetitions, clamped to the maximum. Default 3. Range: 1 to 50. |
| <span id="scroll--macos--delivery_mode"></span>`delivery_mode` | `"background" \| "foreground"` | | Default "background": no fronting or focus steal. "foreground": briefly front the window, act, restore the prior app; last resort when background did not land. |
-| <span id="scroll--macos--pid"></span>`pid` | `integer` | | Target process ID. Required unless scope is "desktop". |
+| <span id="scroll--macos--pid"></span>`pid` | `integer` | | Target process ID. Required unless scope is "desktop" or element_token is supplied. |
| <span id="scroll--macos--scope"></span>`scope` | `"window" \| "desktop"` | `"window"` | "desktop" with x,y and no pid/window_id uses get_desktop_state coordinates. |
| <span id="scroll--macos--session"></span>`session` | `string` | | For multi-call work, prefer a short public session label and repeat it on every call that accepts it. |
| <span id="scroll--macos--window_id"></span>`window_id` | `integer` | | Window ID. Required with x/y; optional with element_token. |
diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/snapshot_store.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/snapshot_store.rs
index e61290c894..ad613a7e2d 100644
--- a/libs/cua-driver/rust/crates/cua-driver-core/src/snapshot_store.rs
+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/snapshot_store.rs
@@ -371,6 +371,22 @@ impl<S: SnapshotPayload> SnapshotStore<S> {
})
}
+ /// The pid an `element_token` was minted for, found without the caller
+ /// naming it. Snapshot ids are unique per runtime, so a token alone names
+ /// its snapshot, window and process. `None` when the token is absent,
+ /// malformed, or no longer current; callers then fall back to the
+ /// ordinary missing-`pid` error.
+ pub fn pid_for_token(&self, args: &serde_json::Value) -> Option<i32> {
+ let token = args.get("element_token")?.as_str()?;
+ let (snapshot_id, _) = parse_token(token)?;
+ self.inner
+ .lock()
+ .unwrap()
+ .iter()
+ .find(|(_, lane)| lane.iter().any(|snapshot| snapshot.id == snapshot_id))
+ .map(|(pid, _)| *pid)
+ }
+
pub fn resolve(
&self,
pid: i32,
@@ -510,6 +526,18 @@ mod tests {
use crate::snapshot_test_support::Payload;
use std::sync::atomic::{AtomicUsize, Ordering};
+ #[test]
+ fn pid_for_token_finds_the_owning_process() {
+ let cache = SnapshotStore::new();
+ let id = cache.publish(42, 7, Payload(vec![10, 20]));
+ cache.publish(43, 8, Payload(vec![1]));
+ let args = serde_json::json!({ "element_token": token_for(id, 1) });
+ assert_eq!(cache.pid_for_token(&args), Some(42));
+ assert_eq!(cache.pid_for_token(&serde_json::json!({})), None);
+ let unknown = serde_json::json!({ "element_token": token_for(id + 1000, 0) });
+ assert_eq!(cache.pid_for_token(&unknown), None);
+ }
+
#[test]
fn publish_then_resolve_returns_projection() {
let cache = SnapshotStore::new();
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/input/keyboard.rs b/libs/cua-driver/rust/crates/platform-macos/src/input/keyboard.rs
index 5b8128e42e..7e7ac71cc9 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/input/keyboard.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/input/keyboard.rs
@@ -769,16 +769,53 @@ pub(super) fn key_name_to_code(key: &str) -> anyhow::Result<u16> {
"m" => 46,
"." => 47,
"`" => 50,
- _ => anyhow::bail!("Unknown key name: {key}"),
+ _ => anyhow::bail!("{}", unknown_key_message(key)),
};
Ok(code)
}
+/// The error for a name `key_name_to_code` does not know, saying what to send
+/// instead: a combination such as "shift+Right" is a key plus modifiers, and
+/// a printable symbol is text.
+fn unknown_key_message(key: &str) -> String {
+ if key.chars().count() > 1 && key.contains('+') && !key.ends_with('+') {
+ let parts: Vec<&str> = key.split('+').collect();
+ let (modifiers, name) = parts.split_at(parts.len() - 1);
+ return format!(
+ "Unknown key name: {key}. A key name is one key, not a combination. Send the last \
+ part as the key and the rest as modifiers, e.g. key:\"{}\" with modifiers:{:?}, \
+ or use the hotkey tool with keys:{:?}.",
+ name[0],
+ modifiers
+ .iter()
+ .map(|m| m.to_lowercase())
+ .collect::<Vec<_>>(),
+ parts
+ );
+ }
+ format!(
+ "Unknown key name: {key}. Use a key name such as return, tab, escape, space, delete, \
+ left, right, up, down, pageup, pagedown, home, end, f1..f12, or a single letter or \
+ digit; to enter other characters (for example * or $) use type_text."
+ )
+}
+
#[cfg(test)]
mod tests {
use super::*;
use core_graphics::event::CGEventType;
+ #[test]
+ fn unknown_key_errors_say_what_to_send_instead() {
+ let combo = key_name_to_code("shift+Right").unwrap_err().to_string();
+ assert!(combo.contains("modifiers"), "{combo}");
+ assert!(combo.contains("hotkey"), "{combo}");
+ assert!(combo.contains("key:\"Right\""), "{combo}");
+ let symbol = key_name_to_code("asterisk").unwrap_err().to_string();
+ assert!(symbol.contains("type_text"), "{symbol}");
+ assert!(key_name_to_code("shift").is_ok());
+ }
+
/// cua-spacesd names the Meta key `super` (the cross-platform name), and
/// the macOS driver dropped it: `cua do hotkey cmd+w` pressed a bare `w`.
#[test]
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/click.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/click.rs
index daefb4b914..28696c3ae0 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/click.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/click.rs
@@ -241,7 +241,7 @@ fn def() -> &'static ToolDef {
"required": [],
"properties": {
"session": { "type": "string", "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it." },
- "pid": { "type": "integer", "description": "Target process ID." },
+ "pid": { "type": "integer", "description": "Target process ID. Optional with element_token (it carries the pid)." },
"window_id": { "type": "integer", "description": "Target window ID. Omit with element_token." },
"element_token": cua_driver_core::tool_schema::element_token_schema(),
"capture_id": { "type": "string", "description": "Optional capture ID from get_window_state or get_desktop_state. With x,y it is consumed atomically; stale or mismatched captures are refused." },
@@ -433,7 +433,7 @@ impl Tool for ClickTool {
};
}
- let pid = match args.require_i32("pid") {
+ let pid = match super::target_pid(&self.state, &args) {
Ok(v) => v,
Err(e) => return e,
};
@@ -1327,7 +1327,7 @@ impl Tool for ClickTool {
Err(e) => ToolResult::error(format!("Task error: {e}")),
}
} else {
- ToolResult::error("Provide either element_token or (x + y). pid is always required.")
+ ToolResult::error("Provide either element_token or (x + y). pid is required for x,y clicks; an element_token carries its own pid.")
}
}
}
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/double_click.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/double_click.rs
index 206c8705c9..90a4e89d9a 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/double_click.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/double_click.rs
@@ -73,7 +73,7 @@ impl Tool for DoubleClickTool {
async fn invoke(&self, args: Value) -> ToolResult {
use cua_driver_core::tool_args::ArgsExt;
- let pid = match args.require_i32("pid") {
+ let pid = match super::target_pid(&self.state, &args) {
Ok(v) => v,
Err(e) => return e,
};
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/mod.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/mod.rs
index e0bcc43db8..fb884da3d0 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/mod.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/mod.rs
@@ -686,6 +686,42 @@ impl ToolState {
}
}
+/// The target pid of an element-addressable call. An explicit `pid` wins; a
+/// call that carries only an `element_token` takes the pid the token was
+/// minted for, as the schemas promise ("the token carries it").
+pub(super) fn target_pid(
+ state: &ToolState,
+ args: &serde_json::Value,
+) -> Result<i32, cua_driver_core::protocol::ToolResult> {
+ use cua_driver_core::tool_args::ArgsExt;
+ if args.get("pid").is_some_and(|pid| !pid.is_null()) {
+ return args.require_i32("pid");
+ }
+ if let Some(pid) = state.snapshots.pid_for_token(args) {
+ return Ok(pid);
+ }
+ if args
+ .get("element_token")
+ .is_some_and(|token| !token.is_null())
+ {
+ return Err(cua_driver_core::protocol::ToolResult::error(
+ "element_token is stale or unknown; call get_window_state again to refresh \
+ (a current token names its own pid).",
+ )
+ .with_structured(serde_json::json!({
+ "status": "refused",
+ "refusal": {
+ "code": "stale_element_token",
+ "message": "element_token is stale or unknown; call get_window_state again to refresh",
+ },
+ })));
+ }
+ Err(cua_driver_core::protocol::ToolResult::error(
+ "Missing required integer field: pid. Pass pid, or pass an element_token from the \
+ current get_window_state (a token names its own pid).",
+ ))
+}
+
pub(super) fn screenshot_scale(
state: &ToolState,
args: &serde_json::Value,
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/press_key.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/press_key.rs
index 74a2206549..9ea393dc2f 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/press_key.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/press_key.rs
@@ -249,7 +249,7 @@ impl Tool for PressKeyTool {
Err(error) => ToolResult::error(format!("desktop press_key task failed: {error}")),
};
}
- let pid = match args.require_i32("pid") {
+ let pid = match super::target_pid(&self.state, &args) {
Ok(v) => v,
Err(e) => return e,
};
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/right_click.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/right_click.rs
index 9af941d8be..a45e01c3a5 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/right_click.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/right_click.rs
@@ -82,7 +82,7 @@ impl Tool for RightClickTool {
async fn invoke(&self, args: Value) -> ToolResult {
use cua_driver_core::tool_args::ArgsExt;
- let pid = match args.require_i32("pid") {
+ let pid = match super::target_pid(&self.state, &args) {
Ok(v) => v,
Err(e) => return e,
};
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/scroll.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/scroll.rs
index 7b8d94ed28..7a5efaed92 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/scroll.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/scroll.rs
@@ -58,6 +58,12 @@ fn clamp_amount(requested: u64) -> usize {
requested.clamp(AMOUNT_MIN, AMOUNT_MAX) as usize
}
+const ELECTRON_BACKGROUND_SCROLL_UNAVAILABLE: &str = "Background scroll is unavailable for \
+ Electron/Chromium windows on macOS: they drop background wheel events and keystrokes, so \
+ nothing was sent. Retry with delivery_mode:\"foreground\" (and the same window_id, \
+ element_token or x,y); the driver briefly fronts that window, scrolls, and restores the \
+ prior frontmost app.";
+
fn def() -> &'static ToolDef {
DEF.get_or_init(|| ToolDef {
name: "scroll".into(),
@@ -65,6 +71,7 @@ fn def() -> &'static ToolDef {
- Targeted wheel: pass `element_token` (preferred) or window-local `x, y`. Sends a real wheel event at that point, so it scrolls whatever is under it. The only way to scroll nested `overflow:auto` regions in web views.\n\
- Keystroke: no target, just pid + direction. PageDown/PageUp (by='page') or arrows (by='line') on the focused scroller.\n\
\n\
+ Electron/Chromium windows refuse background delivery; pass `delivery_mode:\"foreground\"`.\n\
`amount` is wheel notches (targeted) or key repetitions (keystroke).".into(),
input_schema: serde_json::json!({
"type": "object",
@@ -73,7 +80,7 @@ fn def() -> &'static ToolDef {
"required": ["direction"],
"properties": {
"session": { "type": "string", "description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it." },
- "pid": { "type": "integer", "description": "Target process ID. Required unless scope is \"desktop\"." },
+ "pid": { "type": "integer", "description": "Target process ID. Required unless scope is \"desktop\" or element_token is supplied." },
"direction": {
"type": "string",
"enum": ["up", "down", "left", "right"],
@@ -155,7 +162,7 @@ impl Tool for ScrollTool {
Err(error) => ToolResult::error(format!("desktop scroll task failed: {error}")),
};
}
- let pid = match args.require_i32("pid") {
+ let pid = match super::target_pid(&self.state, &args) {
Ok(v) => v,
Err(e) => return e,
};
@@ -165,11 +172,12 @@ impl Tool for ScrollTool {
// keystroke path is background-by-design and untouched.
let delivery_mode = super::DeliveryMode::parse(args.opt_str("delivery_mode").as_deref());
if !delivery_mode.is_foreground() && crate::browser::ElectronJs::is_electron(pid) {
- return ToolResult::error(
- "Background scroll is unavailable for Electron/Chromium windows on macOS."
- .to_owned(),
- )
- .with_structured(serde_json::json!({ "code": "background_unavailable" }));
+ return cua_driver_core::delivery::background_unavailable_result(
+ ELECTRON_BACKGROUND_SCROLL_UNAVAILABLE,
+ "background_unavailable",
+ "Electron/Chromium windows drop background wheel events and keystrokes on macOS",
+ serde_json::json!({ "effect": "refused" }),
+ );
}
let direction = match args.require_str("direction") {
Ok(v) => v,
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/set_value.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/set_value.rs
index ded77100d1..ac0a6d969f 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/set_value.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/set_value.rs
@@ -81,7 +81,7 @@ impl Tool for SetValueTool {
async fn invoke(&self, args: Value) -> ToolResult {
use cua_driver_core::tool_args::ArgsExt;
- let pid = match args.require_i32("pid") {
+ let pid = match super::target_pid(&self.state, &args) {
Ok(v) => v,
Err(e) => return e,
};
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/type_text.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/type_text.rs
index 663ebc9f45..9040f1c094 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/type_text.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/type_text.rs
@@ -166,7 +166,7 @@ impl Tool for TypeTextTool {
Err(error) => ToolResult::error(format!("desktop type_text task failed: {error}")),
};
}
- let pid = match args.require_i32("pid") {
+ let pid = match super::target_pid(&self.state, &args) {
Ok(v) => v,
Err(e) => return e,
};
diff --git a/scripts/docs-generators/cli-specs/cua-driver-mcp-macos.json b/scripts/docs-generators/cli-specs/cua-driver-mcp-macos.json
index 4b7b270507..22174142d4 100644
--- a/scripts/docs-generators/cli-specs/cua-driver-mcp-macos.json
+++ b/scripts/docs-generators/cli-specs/cua-driver-mcp-macos.json
@@ -536,7 +536,7 @@
"type": "array"
},
"pid": {
- "description": "Target process ID.",
+ "description": "Target process ID. Optional with element_token (it carries the pid).",
"type": "integer"
},
"scope": {
@@ -1270,7 +1270,7 @@
"read_only": false
},
{
- "description": "Scroll the target pid.\n- Targeted wheel: pass `element_token` (preferred) or window-local `x, y`. Sends a real wheel event at that point, so it scrolls whatever is under it. The only way to scroll nested `overflow:auto` regions in web views.\n- Keystroke: no target, just pid + direction. PageDown/PageUp (by='page') or arrows (by='line') on the focused scroller.\n\n`amount` is wheel notches (targeted) or key repetitions (keystroke).",
+ "description": "Scroll the target pid.\n- Targeted wheel: pass `element_token` (preferred) or window-local `x, y`. Sends a real wheel event at that point, so it scrolls whatever is under it. The only way to scroll nested `overflow:auto` regions in web views.\n- Keystroke: no target, just pid + direction. PageDown/PageUp (by='page') or arrows (by='line') on the focused scroller.\n\nElectron/Chromium windows refuse background delivery; pass `delivery_mode:\"foreground\"`.\n`amount` is wheel notches (targeted) or key repetitions (keystroke).",
"destructive": false,
"idempotent": false,
"input_schema": {
@@ -1314,7 +1314,7 @@
"type": "string"
},
"pid": {
- "description": "Target process ID. Required unless scope is \"desktop\".",
+ "description": "Target process ID. Required unless scope is \"desktop\" or element_token is supplied.",
"type": "integer"
},
"scope": {