Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
fix(cua-driver): accept invoke_menu and zoom shapes models send, point named direct actions to run_actions (#4862)
Francesco Bonacci7b3a8ee656a6
diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/tool.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/tool.rsindex 35fa8dd7a7..7a6f179f9f 100644--- a/libs/cua-driver/rust/crates/cua-driver-core/src/tool.rs+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/tool.rs@@ -214,6 +214,10 @@ fn unknown_argument(def: &ToolDef, args: &Value) -> Option<String> {if unknown.is_empty() {return None;}+ // A direct action named by role/name: that targeting lives in run_actions.+ const NAME_TARGET: &[&str] = &["role", "name", "label", "app", "window", "nth"];+ let name_targeted = crate::batch_tools::BATCHABLE_TOOLS.contains(&def.name.as_str())+ && unknown.iter().all(|name| NAME_TARGET.contains(name));let accepted: Vec<&str> = properties.map(|properties| {properties@@ -228,8 +232,17 @@ fn unknown_argument(def: &ToolDef, args: &Value) -> Option<String> {} else {"arguments"};+ let hint = if name_targeted {+ format!(+ ". To target by role/name/app/window, send the same arguments as one run_actions \+ step: run_actions {{\"steps\":[{{\"{}\": {{...}}}}]}}",+ def.name+ )+ } else {+ String::new()+ };Some(format!(- "{noun} {}; accepted: {}",+ "{noun} {}; accepted: {}{hint}",unknown.join(", "),accepted.join(", ")))@@ -242,9 +255,49 @@ fn unknown_argument(def: &ToolDef, args: &Value) -> Option<String> {pub(crate) fn normalize_argument_aliases(tool_name: &str, args: &mut Value) -> Result<(), String> {normalize_zoom_args(tool_name, args);normalize_key_args(tool_name, args);+ normalize_menu_args(tool_name, args);normalize_scroll_args(tool_name, args)}+/// `invoke_menu` takes `path` as a list of labels. Models also send+/// `menu_path`, a JSON-encoded list, or one "File > Save As..." string.+fn normalize_menu_args(tool_name: &str, args: &mut Value) {+ if tool_name != "invoke_menu" {+ return;+ }+ let Some(arguments) = args.as_object_mut() else {+ return;+ };+ if !arguments.contains_key("path") {+ if let Some(alias) = ["menu_path", "menu", "items"]+ .iter()+ .find(|alias| arguments.contains_key(**alias))+ {+ let value = arguments.remove(*alias).expect("present");+ arguments.insert("path".to_owned(), value);+ }+ }+ let Some(Value::String(text)) = arguments.get("path") else {+ return;+ };+ let text = text.trim();+ let labels: Vec<String> = match serde_json::from_str::<Vec<String>>(text) {+ Ok(labels) => labels,+ Err(_) => text+ .split(['>', '→'])+ .map(str::trim)+ .filter(|label| !label.is_empty())+ .map(str::to_owned)+ .collect(),+ };+ if !labels.is_empty() {+ arguments.insert(+ "path".to_owned(),+ Value::Array(labels.into_iter().map(Value::String).collect()),+ );+ }+}+/// The JSON types a property schema admits, from `type` (a string or a list)/// and the `type` of each `anyOf`/`oneOf` branch. Empty when the schema/// declares none.@@ -6850,6 +6903,51 @@ mod argument_shape_tests {);}+ #[test]+ fn menu_paths_in_the_shapes_models_send() {+ let menu = |args: serde_json::Value| {+ let mut args = args;+ normalize_argument_aliases("invoke_menu", &mut args).unwrap();+ args+ };+ // v036 shapes.+ assert_eq!(+ menu(json!({"pid": 1, "menu_path": "[\"Sheet\",\"Navigate\",\"Next Sheet\"]"})),+ json!({"pid": 1, "path": ["Sheet", "Navigate", "Next Sheet"]})+ );+ assert_eq!(+ menu(json!({"path": "File > Reload"})),+ json!({"path": ["File", "Reload"]})+ );+ assert_eq!(+ menu(json!({"menu": ["Edit", "Copy"]})),+ json!({"path": ["Edit", "Copy"]})+ );+ // A real path and an explicit path win.+ assert_eq!(+ menu(json!({"path": ["File", "Save As..."], "menu_path": "x"})),+ json!({"path": ["File", "Save As..."], "menu_path": "x"})+ );+ }++ #[test]+ fn a_direct_action_named_by_role_points_to_run_actions() {+ let click = ToolDef {+ name: "click".into(),+ description: String::new(),+ input_schema: json!({"type": "object", "properties": {"pid": {"type": "integer"}}, "additionalProperties": false}),+ read_only: false,+ destructive: false,+ idempotent: false,+ open_world: false,+ };+ let detail =+ unknown_argument(&click, &json!({"pid": 1, "role": "button", "name": "OK"})).unwrap();+ assert!(detail.contains("one run_actions step"), "{detail}");+ let other = unknown_argument(&click, &json!({"pid": 1, "bogus": 1})).unwrap();+ assert!(!other.contains("run_actions"), "{other}");+ }+#[test]fn key_combinations_and_spellings_are_normalized() {let press = |args: serde_json::Value| {diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/invoke_menu.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/invoke_menu.rsindex e809ce289f..1b18cd8580 100644--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/invoke_menu.rs+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/invoke_menu.rs@@ -548,6 +548,10 @@ impl Tool for InvokeMenuTool {}async fn invoke(&self, args: Value) -> ToolResult {+ // The menu bar belongs to the app: with only a pid, use its focused+ // window.+ let mut args = args;+ super::default_window_for_pid(&mut args);let input: InvokeMenuInput =match cua_driver_core::tool_args::parse_typed_input("invoke_menu", args) {Ok(input) => input,diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/mod.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/mod.rsindex 01c7268f70..f7bac73cad 100644--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/mod.rs+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/mod.rs@@ -303,6 +303,39 @@ pub(crate) fn background_refusal_result(/// Exclusive per-process ownership of one background mutation. Callers must/// keep this value alive through actuator dispatch, focus restoration, and/// target-bound verification.+/// The window a pid-only call means: the app's focused window, else its+/// frontmost on-screen window. Fills `window_id` for tools that need one+/// when the caller named only the app's pid.+pub(crate) fn default_window_for_pid(args: &mut serde_json::Value) {+ let Some(object) = args.as_object_mut() else {+ return;+ };+ if object+ .get("window_id")+ .is_some_and(|value| !value.is_null())+ {+ return;+ }+ let Some(pid) = object+ .get("pid")+ .and_then(serde_json::Value::as_i64)+ .and_then(|pid| i32::try_from(pid).ok())+ else {+ return;+ };+ let window_id = crate::ax::bindings::focused_window_id_of_pid(pid).or_else(|| {+ crate::windows::all_windows()+ .into_iter()+ .filter(|window| window.pid == pid && window.is_on_screen && window.layer == 0)+ .filter(|window| window.bounds.width >= 2.0 && window.bounds.height >= 2.0)+ .max_by_key(|window| window.z_index)+ .map(|window| window.window_id)+ });+ if let Some(window_id) = window_id {+ object.insert("window_id".into(), serde_json::json!(window_id));+ }+}+pub(crate) struct BackgroundMutationLease {pid: i32,_guard: tokio::sync::OwnedMutexGuard<()>,diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/zoom.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/zoom.rsindex 777c0f22cf..976ff7fc74 100644--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/zoom.rs+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/zoom.rs@@ -54,6 +54,18 @@ impl Tool for ZoomTool {async fn invoke(&self, args: Value) -> ToolResult {use cua_driver_core::tool_args::ArgsExt;+ let mut args = args;+ super::default_window_for_pid(&mut args);+ if ["x1", "y1", "x2", "y2"]+ .iter()+ .all(|key| args.get(*key).is_none())+ {+ return ToolResult::error(+ "zoom needs a region of the window's screenshot: x1,y1 (top-left) and x2,y2 \+ (bottom-right) in screenshot pixels, or x,y,width,height. To see the whole \+ window, call get_window_state (it returns the screenshot).",+ );+ }let window_id = match args.require_u32("window_id") {Ok(v) => v,Err(e) => return e,