Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
fix(cua-driver-core): derive desktop coordination from the action contract (#6638)
injaneity13fdf9cb6a80
diff --git a/libs/cua-driver/rust/crates/cua-driver-contract/src/lib.rs b/libs/cua-driver/rust/crates/cua-driver-contract/src/lib.rsindex c24a241d20..26369994fd 100644--- a/libs/cua-driver/rust/crates/cua-driver-contract/src/lib.rs+++ b/libs/cua-driver/rust/crates/cua-driver-contract/src/lib.rs@@ -131,6 +131,14 @@ pub fn is_action_result_tool(name: &str) -> bool {ACTION_RESULT_TOOLS.contains(&name)}+/// Action-result tools that act on native desktop state.+///+/// Browser actions are scheduled per CDP tab inside the browser engine; every+/// other action-result tool drives the shared native input and window state.+pub fn is_desktop_action_result_tool(name: &str) -> bool {+ is_action_result_tool(name) && !name.starts_with("browser_")+}+#[derive(Debug,Clone,diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/tool.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/tool.rsindex aad1f13481..6459fb775b 100644--- a/libs/cua-driver/rust/crates/cua-driver-core/src/tool.rs+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/tool.rs@@ -3576,26 +3576,12 @@ fn canonical_proposed_path(raw: &str) -> Result<String, ToolResult> {Ok(canonical.to_string_lossy().into_owned())}+/// Tools that drive shared native input or window state. Derived from the+/// contract's action-result vocabulary so a new native action tool cannot+/// skip the desktop lane. `bring_to_front` returns its own result shape but+/// still activates a window.fn is_physical_desktop_action(tool: &str) -> bool {- matches!(- tool,- "click"- | "double_click"- | "right_click"- | "scroll"- | "drag"- | "mouse_drag"- | "parallel_mouse_drag"- | "move_cursor"- | "mouse_button_down"- | "mouse_button_up"- | "type_text"- | "press_key"- | "hotkey"- | "set_value"- | "bring_to_front"- | "set_window_frame"- )+ cua_driver_contract::is_desktop_action_result_tool(tool) || tool == "bring_to_front"}fn requires_desktop_coordination(tool: &str, args: &Value, independent_lane: bool) -> bool {@@ -6551,6 +6537,27 @@ resources:assert!(next.await.unwrap());}+ #[test]+ fn every_native_action_result_tool_takes_the_desktop_lane() {+ use super::is_physical_desktop_action;+ for name in cua_driver_contract::ACTION_RESULT_TOOLS {+ assert_eq!(+ is_physical_desktop_action(name),+ !name.starts_with("browser_"),+ "{name}"+ );+ }+ assert!(is_physical_desktop_action("bring_to_front"));+ for name in [+ "get_window_state",+ "screenshot",+ "launch_app",+ "browser_click",+ ] {+ assert!(!is_physical_desktop_action(name), "{name}");+ }+ }+#[test]fn independent_input_requires_adapter_attestation_and_an_exact_background_window() {use super::requires_desktop_coordination;