Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

fix(cua-driver): accept invoke_menu and zoom shapes models send, point named direct actions to run_actions (#4862)

Francesco Bonacci7b3a8ee656a6
diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/tool.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/tool.rs
index 35fa8dd7a7..7a6f179f9f 100644
--- a/libs/cua-driver/rust/crates/cua-driver-core/src/tool.rs
+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/tool.rs
@@ -214,6 +214,10 @@ fn unknown_argument(def: &ToolDef, args: &Value) -> Option<String> {
if unknown.is_empty() {
return None;
}
+ // A direct action named by role/name: that targeting lives in run_actions.
+ const NAME_TARGET: &[&str] = &["role", "name", "label", "app", "window", "nth"];
+ let name_targeted = crate::batch_tools::BATCHABLE_TOOLS.contains(&def.name.as_str())
+ && unknown.iter().all(|name| NAME_TARGET.contains(name));
let accepted: Vec<&str> = properties
.map(|properties| {
properties
@@ -228,8 +232,17 @@ fn unknown_argument(def: &ToolDef, args: &Value) -> Option<String> {
} else {
"arguments"
};
+ let hint = if name_targeted {
+ format!(
+ ". To target by role/name/app/window, send the same arguments as one run_actions \
+ step: run_actions {{\"steps\":[{{\"{}\": {{...}}}}]}}",
+ def.name
+ )
+ } else {
+ String::new()
+ };
Some(format!(
- "{noun} {}; accepted: {}",
+ "{noun} {}; accepted: {}{hint}",
unknown.join(", "),
accepted.join(", ")
))
@@ -242,9 +255,49 @@ fn unknown_argument(def: &ToolDef, args: &Value) -> Option<String> {
pub(crate) fn normalize_argument_aliases(tool_name: &str, args: &mut Value) -> Result<(), String> {
normalize_zoom_args(tool_name, args);
normalize_key_args(tool_name, args);
+ normalize_menu_args(tool_name, args);
normalize_scroll_args(tool_name, args)
}
+/// `invoke_menu` takes `path` as a list of labels. Models also send
+/// `menu_path`, a JSON-encoded list, or one "File > Save As..." string.
+fn normalize_menu_args(tool_name: &str, args: &mut Value) {
+ if tool_name != "invoke_menu" {
+ return;
+ }
+ let Some(arguments) = args.as_object_mut() else {
+ return;
+ };
+ if !arguments.contains_key("path") {
+ if let Some(alias) = ["menu_path", "menu", "items"]
+ .iter()
+ .find(|alias| arguments.contains_key(**alias))
+ {
+ let value = arguments.remove(*alias).expect("present");
+ arguments.insert("path".to_owned(), value);
+ }
+ }
+ let Some(Value::String(text)) = arguments.get("path") else {
+ return;
+ };
+ let text = text.trim();
+ let labels: Vec<String> = match serde_json::from_str::<Vec<String>>(text) {
+ Ok(labels) => labels,
+ Err(_) => text
+ .split(['>', '→'])
+ .map(str::trim)
+ .filter(|label| !label.is_empty())
+ .map(str::to_owned)
+ .collect(),
+ };
+ if !labels.is_empty() {
+ arguments.insert(
+ "path".to_owned(),
+ Value::Array(labels.into_iter().map(Value::String).collect()),
+ );
+ }
+}
+
/// The JSON types a property schema admits, from `type` (a string or a list)
/// and the `type` of each `anyOf`/`oneOf` branch. Empty when the schema
/// declares none.
@@ -6850,6 +6903,51 @@ mod argument_shape_tests {
);
}
+ #[test]
+ fn menu_paths_in_the_shapes_models_send() {
+ let menu = |args: serde_json::Value| {
+ let mut args = args;
+ normalize_argument_aliases("invoke_menu", &mut args).unwrap();
+ args
+ };
+ // v036 shapes.
+ assert_eq!(
+ menu(json!({"pid": 1, "menu_path": "[\"Sheet\",\"Navigate\",\"Next Sheet\"]"})),
+ json!({"pid": 1, "path": ["Sheet", "Navigate", "Next Sheet"]})
+ );
+ assert_eq!(
+ menu(json!({"path": "File > Reload"})),
+ json!({"path": ["File", "Reload"]})
+ );
+ assert_eq!(
+ menu(json!({"menu": ["Edit", "Copy"]})),
+ json!({"path": ["Edit", "Copy"]})
+ );
+ // A real path and an explicit path win.
+ assert_eq!(
+ menu(json!({"path": ["File", "Save As..."], "menu_path": "x"})),
+ json!({"path": ["File", "Save As..."], "menu_path": "x"})
+ );
+ }
+
+ #[test]
+ fn a_direct_action_named_by_role_points_to_run_actions() {
+ let click = ToolDef {
+ name: "click".into(),
+ description: String::new(),
+ input_schema: json!({"type": "object", "properties": {"pid": {"type": "integer"}}, "additionalProperties": false}),
+ read_only: false,
+ destructive: false,
+ idempotent: false,
+ open_world: false,
+ };
+ let detail =
+ unknown_argument(&click, &json!({"pid": 1, "role": "button", "name": "OK"})).unwrap();
+ assert!(detail.contains("one run_actions step"), "{detail}");
+ let other = unknown_argument(&click, &json!({"pid": 1, "bogus": 1})).unwrap();
+ assert!(!other.contains("run_actions"), "{other}");
+ }
+
#[test]
fn key_combinations_and_spellings_are_normalized() {
let press = |args: serde_json::Value| {
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/invoke_menu.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/invoke_menu.rs
index e809ce289f..1b18cd8580 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/invoke_menu.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/invoke_menu.rs
@@ -548,6 +548,10 @@ impl Tool for InvokeMenuTool {
}
async fn invoke(&self, args: Value) -> ToolResult {
+ // The menu bar belongs to the app: with only a pid, use its focused
+ // window.
+ let mut args = args;
+ super::default_window_for_pid(&mut args);
let input: InvokeMenuInput =
match cua_driver_core::tool_args::parse_typed_input("invoke_menu", args) {
Ok(input) => input,
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/mod.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/mod.rs
index 01c7268f70..f7bac73cad 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/mod.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/mod.rs
@@ -303,6 +303,39 @@ pub(crate) fn background_refusal_result(
/// Exclusive per-process ownership of one background mutation. Callers must
/// keep this value alive through actuator dispatch, focus restoration, and
/// target-bound verification.
+/// The window a pid-only call means: the app's focused window, else its
+/// frontmost on-screen window. Fills `window_id` for tools that need one
+/// when the caller named only the app's pid.
+pub(crate) fn default_window_for_pid(args: &mut serde_json::Value) {
+ let Some(object) = args.as_object_mut() else {
+ return;
+ };
+ if object
+ .get("window_id")
+ .is_some_and(|value| !value.is_null())
+ {
+ return;
+ }
+ let Some(pid) = object
+ .get("pid")
+ .and_then(serde_json::Value::as_i64)
+ .and_then(|pid| i32::try_from(pid).ok())
+ else {
+ return;
+ };
+ let window_id = crate::ax::bindings::focused_window_id_of_pid(pid).or_else(|| {
+ crate::windows::all_windows()
+ .into_iter()
+ .filter(|window| window.pid == pid && window.is_on_screen && window.layer == 0)
+ .filter(|window| window.bounds.width >= 2.0 && window.bounds.height >= 2.0)
+ .max_by_key(|window| window.z_index)
+ .map(|window| window.window_id)
+ });
+ if let Some(window_id) = window_id {
+ object.insert("window_id".into(), serde_json::json!(window_id));
+ }
+}
+
pub(crate) struct BackgroundMutationLease {
pid: i32,
_guard: tokio::sync::OwnedMutexGuard<()>,
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/zoom.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/zoom.rs
index 777c0f22cf..976ff7fc74 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/zoom.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/zoom.rs
@@ -54,6 +54,18 @@ impl Tool for ZoomTool {
async fn invoke(&self, args: Value) -> ToolResult {
use cua_driver_core::tool_args::ArgsExt;
+ let mut args = args;
+ super::default_window_for_pid(&mut args);
+ if ["x1", "y1", "x2", "y2"]
+ .iter()
+ .all(|key| args.get(*key).is_none())
+ {
+ return ToolResult::error(
+ "zoom needs a region of the window's screenshot: x1,y1 (top-left) and x2,y2 \
+ (bottom-right) in screenshot pixels, or x,y,width,height. To see the whole \
+ window, call get_window_state (it returns the screenshot).",
+ );
+ }
let window_id = match args.require_u32("window_id") {
Ok(v) => v,
Err(e) => return e,