Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
test(cua-driver): restore cross-platform schema coverage (#2189)
Francesco Bonacci2bbbb11ad5d8
diff --git a/.github/workflows/ci-rust-windows.yml b/.github/workflows/ci-rust-windows.ymlindex cbd7af1138..2d3cafeaf0 100644--- a/.github/workflows/ci-rust-windows.yml+++ b/.github/workflows/ci-rust-windows.yml@@ -43,3 +43,6 @@ jobs:# Compile every Rust target without executing desktop-dependent integration# tests; interactive behavior belongs in e2e-rust-windows.yml.run: cargo test -p cua-driver -p cua-driver-core -p cua-driver-testkit -p platform-windows -p cua-driver-uia --all-targets --no-run --locked+ - name: Run Windows protocol schema contract+ working-directory: libs/cua-driver/rust+ run: cargo test -p cua-driver --test protocol_schema_test --lockeddiff --git a/libs/cua-driver/docs/pr-2161-landing-plan.md b/libs/cua-driver/docs/pr-2161-landing-plan.mdindex 507135badc..fadf569b2c 100644--- a/libs/cua-driver/docs/pr-2161-landing-plan.md+++ b/libs/cua-driver/docs/pr-2161-landing-plan.md@@ -27,6 +27,17 @@ small shared-fixture correction landed separately to keep the foundation PRreviewable. Those adaptations preserve the ownership and dependency intent ofthe plan without replaying intermediate branch history.+### Post-merge audit correction++An independent frozen-tree review found one shared-file omission after the+replacement stack landed: `protocol_schema_test.rs` retained only the Windows+delivery/scope assertions, excluded Linux, and dropped Linux's unique+`set_agent_cursor_motion` applied-value check. The follow-up restores the frozen+cross-platform contract and makes Windows unit CI execute the schema test rather+than compile it only. The original parity comment's claim that this file was an+intentional consolidation was incorrect; no driver implementation or accepted+E2E matrix behavior was lost.+---## 1. Decision and rationalediff --git a/libs/cua-driver/rust/crates/cua-driver/tests/protocol_schema_test.rs b/libs/cua-driver/rust/crates/cua-driver/tests/protocol_schema_test.rsindex 08fe9e59d0..84a54a3ab8 100644--- a/libs/cua-driver/rust/crates/cua-driver/tests/protocol_schema_test.rs+++ b/libs/cua-driver/rust/crates/cua-driver/tests/protocol_schema_test.rs@@ -2,10 +2,10 @@//!//! These never invoke a tool — they only inspect the advertised inputSchemas://! that `type_text_chars` is hidden, the `list_windows.on_screen_only` knob, the-//! `set_agent_cursor_motion` Bezier knobs, and the `set_config.capture_mode`-//! enum.+//! `set_agent_cursor_motion` Bezier knobs, delivery and scope enums, and the+//! `set_config.capture_mode` enum.-#![cfg(any(target_os = "macos", target_os = "windows"))]+#![cfg(any(target_os = "linux", target_os = "macos", target_os = "windows"))]use cua_driver_testkit::RawDriver;@@ -16,14 +16,16 @@ fn tools_list_schema_shape() {//! no `type_text_chars` either — the old Windows mirror asserted it was//! exposed, but that had never been run). The advertised schemas must still//! carry their expected knobs.- let Some(mut d) = RawDriver::spawn() else { return; };+ let mut d = RawDriver::spawn().expect("spawn source-built driver for schema test");d.send(&serde_json::json!({"jsonrpc":"2.0","id":1,"method":"initialize","params":{}}));d.recv();d.send(&serde_json::json!({"jsonrpc":"2.0","id":2,"method":"tools/list","params":{}}));let list_resp = d.recv();- let tools = list_resp["result"]["tools"].as_array().expect("tools array");+ let tools = list_resp["result"]["tools"]+ .as_array()+ .expect("tools array");let properties = |name: &str| {&tools@@ -45,24 +47,33 @@ fn tools_list_schema_shape() {"type_text_chars should be hidden from tools/list");- #[cfg(target_os = "windows")]+ for tool in [+ "click",+ "double_click",+ "right_click",+ "type_text",+ "press_key",+ "hotkey",+ "scroll",+ ] {+ let delivery = &properties(tool)["delivery_mode"];+ assert!(+ enum_contains(delivery, "background") && enum_contains(delivery, "foreground"),+ "{tool}.delivery_mode should advertise background and foreground: {delivery:?}"+ );+ }+ // macOS selects desktop coordinates per click. Linux and Windows retain+ // the process-level capture_scope gate used by their desktop-state tools.+ #[cfg(target_os = "macos")]+ {+ let scope = &properties("click")["scope"];+ assert!(+ enum_contains(scope, "window") && enum_contains(scope, "desktop"),+ "click.scope should advertise window and desktop: {scope:?}"+ );+ }+ #[cfg(any(target_os = "linux", target_os = "windows"))]{- for tool in [- "click",- "double_click",- "right_click",- "type_text",- "press_key",- "hotkey",- "scroll",- ] {- let delivery = &properties(tool)["delivery_mode"];- assert!(- enum_contains(delivery, "background") && enum_contains(delivery, "foreground"),- "{tool}.delivery_mode should advertise background and foreground: {delivery:?}"- );- }-let capture_scope = &properties("set_config")["capture_scope"];assert!(enum_contains(capture_scope, "window") && enum_contains(capture_scope, "desktop"),@@ -71,18 +82,33 @@ fn tools_list_schema_shape() {}// list_windows schema has on_screen_only.- let lw = tools.iter().find(|t| t["name"] == "list_windows")+ let lw = tools+ .iter()+ .find(|t| t["name"] == "list_windows").expect("list_windows not found in tools/list");- assert!(lw["inputSchema"]["properties"]["on_screen_only"].is_object(),- "list_windows schema missing on_screen_only: {:?}", lw["inputSchema"]["properties"]);+ assert!(+ lw["inputSchema"]["properties"]["on_screen_only"].is_object(),+ "list_windows schema missing on_screen_only: {:?}",+ lw["inputSchema"]["properties"]+ );// set_agent_cursor_motion has the Bezier motion knobs.- let sacm = tools.iter().find(|t| t["name"] == "set_agent_cursor_motion")+ let sacm = tools+ .iter()+ .find(|t| t["name"] == "set_agent_cursor_motion").expect("set_agent_cursor_motion not found in tools/list");let sacm_props = &sacm["inputSchema"]["properties"];- for knob in &["arc_size", "spring", "glide_duration_ms", "dwell_after_click_ms", "idle_hide_ms"] {- assert!(sacm_props[knob].is_object(),- "set_agent_cursor_motion schema missing {knob}: {sacm_props:?}");+ for knob in &[+ "arc_size",+ "spring",+ "glide_duration_ms",+ "dwell_after_click_ms",+ "idle_hide_ms",+ ] {+ assert!(+ sacm_props[knob].is_object(),+ "set_agent_cursor_motion schema missing {knob}: {sacm_props:?}"+ );}// capture_mode enum restriction. On macOS, capture_mode is a per-call param@@ -90,16 +116,65 @@ fn tools_list_schema_shape() {// now, not settings). On Windows it is still also a set_config field.#[cfg(target_os = "macos")]{- let gws = tools.iter().find(|t| t["name"] == "get_window_state")+ let gws = tools+ .iter()+ .find(|t| t["name"] == "get_window_state").expect("get_window_state not found in tools/list");- assert!(gws["inputSchema"]["properties"]["capture_mode"]["enum"].is_array(),- "get_window_state capture_mode should have enum: {:?}", gws["inputSchema"]["properties"]);+ assert!(+ gws["inputSchema"]["properties"]["capture_mode"]["enum"].is_array(),+ "get_window_state capture_mode should have enum: {:?}",+ gws["inputSchema"]["properties"]+ );}#[cfg(target_os = "windows")]{- let sc = tools.iter().find(|t| t["name"] == "set_config")+ let sc = tools+ .iter()+ .find(|t| t["name"] == "set_config").expect("set_config not found in tools/list");- assert!(sc["inputSchema"]["properties"]["capture_mode"]["enum"].is_array(),- "set_config capture_mode should have enum: {:?}", sc["inputSchema"]["properties"]);+ assert!(+ sc["inputSchema"]["properties"]["capture_mode"]["enum"].is_array(),+ "set_config capture_mode should have enum: {:?}",+ sc["inputSchema"]["properties"]+ );}}++#[test]+#[cfg(target_os = "linux")]+fn linux_cursor_motion_knobs_are_applied() {+ let mut driver = RawDriver::spawn().expect("spawn source-built Linux driver");+ driver.send(&serde_json::json!({+ "jsonrpc": "2.0",+ "id": 1,+ "method": "initialize",+ "params": {}+ }));+ driver.recv();++ driver.send(&serde_json::json!({+ "jsonrpc": "2.0",+ "id": 2,+ "method": "tools/call",+ "params": {+ "name": "set_agent_cursor_motion",+ "arguments": {+ "session": "schema-linux",+ "arc_size": 0.4,+ "spring": 0.85,+ "glide_duration_ms": 500,+ "dwell_after_click_ms": 200,+ "idle_hide_ms": 5000+ }+ }+ }));+ let response = driver.recv();+ assert!(+ !response["result"]["isError"].as_bool().unwrap_or(false),+ "Linux cursor motion update failed: {response:?}"+ );+ let structured = &response["result"]["structuredContent"];+ assert_eq!(structured["arc_size"].as_f64(), Some(0.4));+ assert_eq!(structured["glide_duration_ms"].as_f64(), Some(500.0));+ assert_eq!(structured["idle_hide_ms"].as_f64(), Some(5000.0));+}