Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

fix(cua-driver-core): derive desktop coordination from the action contract (#6638)

injaneity13fdf9cb6a80
diff --git a/libs/cua-driver/rust/crates/cua-driver-contract/src/lib.rs b/libs/cua-driver/rust/crates/cua-driver-contract/src/lib.rs
index c24a241d20..26369994fd 100644
--- a/libs/cua-driver/rust/crates/cua-driver-contract/src/lib.rs
+++ b/libs/cua-driver/rust/crates/cua-driver-contract/src/lib.rs
@@ -131,6 +131,14 @@ pub fn is_action_result_tool(name: &str) -> bool {
ACTION_RESULT_TOOLS.contains(&name)
}
+/// Action-result tools that act on native desktop state.
+///
+/// Browser actions are scheduled per CDP tab inside the browser engine; every
+/// other action-result tool drives the shared native input and window state.
+pub fn is_desktop_action_result_tool(name: &str) -> bool {
+ is_action_result_tool(name) && !name.starts_with("browser_")
+}
+
#[derive(
Debug,
Clone,
diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/tool.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/tool.rs
index aad1f13481..6459fb775b 100644
--- a/libs/cua-driver/rust/crates/cua-driver-core/src/tool.rs
+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/tool.rs
@@ -3576,26 +3576,12 @@ fn canonical_proposed_path(raw: &str) -> Result<String, ToolResult> {
Ok(canonical.to_string_lossy().into_owned())
}
+/// Tools that drive shared native input or window state. Derived from the
+/// contract's action-result vocabulary so a new native action tool cannot
+/// skip the desktop lane. `bring_to_front` returns its own result shape but
+/// still activates a window.
fn is_physical_desktop_action(tool: &str) -> bool {
- matches!(
- tool,
- "click"
- | "double_click"
- | "right_click"
- | "scroll"
- | "drag"
- | "mouse_drag"
- | "parallel_mouse_drag"
- | "move_cursor"
- | "mouse_button_down"
- | "mouse_button_up"
- | "type_text"
- | "press_key"
- | "hotkey"
- | "set_value"
- | "bring_to_front"
- | "set_window_frame"
- )
+ cua_driver_contract::is_desktop_action_result_tool(tool) || tool == "bring_to_front"
}
fn requires_desktop_coordination(tool: &str, args: &Value, independent_lane: bool) -> bool {
@@ -6551,6 +6537,27 @@ resources:
assert!(next.await.unwrap());
}
+ #[test]
+ fn every_native_action_result_tool_takes_the_desktop_lane() {
+ use super::is_physical_desktop_action;
+ for name in cua_driver_contract::ACTION_RESULT_TOOLS {
+ assert_eq!(
+ is_physical_desktop_action(name),
+ !name.starts_with("browser_"),
+ "{name}"
+ );
+ }
+ assert!(is_physical_desktop_action("bring_to_front"));
+ for name in [
+ "get_window_state",
+ "screenshot",
+ "launch_app",
+ "browser_click",
+ ] {
+ assert!(!is_physical_desktop_action(name), "{name}");
+ }
+ }
+
#[test]
fn independent_input_requires_adapter_attestation_and_an_exact_background_window() {
use super::requires_desktop_coordination;