Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
fix(cua-driver): advertise delivery mode capability
Francesco Bonacci7f6657c440ce
diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/tool.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/tool.rsindex 2a05cfe30c..e5834b3a94 100644--- a/libs/cua-driver/rust/crates/cua-driver-core/src/tool.rs+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/tool.rs@@ -59,7 +59,7 @@ impl ToolDef {//// Published SDK tools resolve capabilities from their typed Rust// contract. The legacy map remains only for runtime-only tools.- let caps = default_capabilities_for(&self.name);+ let caps = advertised_capabilities_for(&self.name, &self.input_schema);let risk = crate::authorization::risk_metadata_json(&self.name);serde_json::json!({"name": self.name,@@ -93,6 +93,8 @@ impl ToolDef {/// `input.pointer.move`, `input.pointer.button` (raw down/up)/// - `input.keyboard.type`, `input.keyboard.hotkey`,/// `input.keyboard.press`+/// - `input.delivery_mode` (the live tool schema accepts the shared+/// `background` / `foreground` delivery ladder)/// - `screen.capture`, `screen.capture.window`,/// `screen.capture.region`, `screen.dimensions`,/// `screen.cursor.position`@@ -265,6 +267,31 @@ pub fn default_capabilities_for(tool_name: &str) -> Vec<String> {caps.iter().map(|s| (*s).to_owned()).collect()}+/// Capabilities advertised by a concrete runtime tool definition.+///+/// Most capabilities are stable properties of a tool name and come from the+/// typed portable contract (or the legacy runtime-only map). Delivery mode is+/// different: the typed desktop SDK intentionally exposes a narrower,+/// desktop-only input while the live platform schemas additionally accept+/// window-targeted `delivery_mode`. Deriving this one token from the concrete+/// schema keeps `tools/list` truthful on every platform and prevents either+/// overclaiming a tool that cannot accept the field or omitting support from a+/// richer live schema.+pub fn advertised_capabilities_for(tool_name: &str, input_schema: &Value) -> Vec<String> {+ let mut capabilities = default_capabilities_for(tool_name);+ let accepts_delivery_mode = input_schema+ .pointer("/properties/delivery_mode")+ .is_some_and(Value::is_object);+ if accepts_delivery_mode+ && !capabilities+ .iter()+ .any(|capability| capability == "input.delivery_mode")+ {+ capabilities.push("input.delivery_mode".into());+ }+ capabilities+}+/// A callable tool handler. Object-safe — uses `Box<dyn Tool>`.#[async_trait]pub trait Tool: Send + Sync {@@ -784,6 +811,7 @@ mod capability_tests {"input.keyboard.type.terminal_safe","input.keyboard.hotkey","input.keyboard.press",+ "input.delivery_mode",// screen"screen.capture","screen.capture.window",@@ -895,6 +923,28 @@ mod capability_tests {assert_eq!(CAPABILITY_VERSION, "1");}+ #[test]+ fn delivery_mode_capability_is_derived_from_the_runtime_schema() {+ let with_delivery_mode = serde_json::json!({+ "type": "object",+ "properties": {+ "delivery_mode": crate::tool_schema::delivery_mode_schema()+ }+ });+ let without_delivery_mode = serde_json::json!({"type": "object", "properties": {}});++ assert!(+ advertised_capabilities_for("press_key", &with_delivery_mode)+ .iter()+ .any(|capability| capability == "input.delivery_mode")+ );+ assert!(+ !advertised_capabilities_for("press_key", &without_delivery_mode)+ .iter()+ .any(|capability| capability == "input.delivery_mode")+ );+ }+#[test]fn unknown_tools_get_empty_capabilities() {// Tools without a mapping (typically internal/stub tools like@@ -975,6 +1025,30 @@ mod capability_tests {);}+ #[test]+ fn to_list_entry_advertises_delivery_mode_only_when_accepted() {+ let mut accepting = dummy_def("press_key");+ accepting.input_schema = serde_json::json!({+ "type": "object",+ "properties": {+ "delivery_mode": crate::tool_schema::delivery_mode_schema()+ }+ });+ let accepting_entry = accepting.to_list_entry();+ assert!(accepting_entry["capabilities"]+ .as_array()+ .expect("capabilities array")+ .iter()+ .any(|capability| capability == "input.delivery_mode"));++ let rejecting_entry = dummy_def("press_key").to_list_entry();+ assert!(!rejecting_entry["capabilities"]+ .as_array()+ .expect("capabilities array")+ .iter()+ .any(|capability| capability == "input.delivery_mode"));+ }+#[test]fn to_list_entry_includes_versioned_risk_metadata() {let entry = dummy_def("browser_prepare").to_list_entry();diff --git a/libs/cua-driver/rust/crates/cua-driver-core/tests/contract_parity.rs b/libs/cua-driver/rust/crates/cua-driver-core/tests/contract_parity.rsindex 18291a7e97..ac8c9f9b3c 100644--- a/libs/cua-driver/rust/crates/cua-driver-core/tests/contract_parity.rs+++ b/libs/cua-driver/rust/crates/cua-driver-core/tests/contract_parity.rs@@ -63,5 +63,17 @@ fn every_generated_contract_uses_live_capability_tokens() {"{} capabilities",contract.name);+ assert_eq!(+ contract+ .input_schema+ .pointer("/properties/delivery_mode")+ .is_some(),+ contract+ .capabilities+ .iter()+ .any(|capability| capability == "input.delivery_mode"),+ "{} typed contract delivery_mode schema/capability mismatch",+ contract.name+ );}}diff --git a/libs/cua-driver/rust/crates/cua-driver/src/proxy.rs b/libs/cua-driver/rust/crates/cua-driver/src/proxy.rsindex f046ad1e17..4b402a3fd3 100644--- a/libs/cua-driver/rust/crates/cua-driver/src/proxy.rs+++ b/libs/cua-driver/rust/crates/cua-driver/src/proxy.rs@@ -434,11 +434,13 @@ fn fetch_tools_list_from_daemon(.and_then(|v| v.as_array()).cloned().unwrap_or_else(|| {- // Fallback: derive from the centralised map by- // name. Keeps the proxy compatible with daemon+ // Fallback: derive from the centralised name + schema+ // resolver. Keeps the proxy compatible with daemon// builds that pre-date the capabilities field.name.as_str()- .map(cua_driver_core::tool::default_capabilities_for)+ .map(|name| {+ cua_driver_core::tool::advertised_capabilities_for(name, &input_schema)+ }).unwrap_or_default().into_iter().map(serde_json::Value::String)diff --git a/libs/cua-driver/rust/crates/cua-driver/src/serve.rs b/libs/cua-driver/rust/crates/cua-driver/src/serve.rsindex 3c4db9dbf8..407b6e4664 100644--- a/libs/cua-driver/rust/crates/cua-driver/src/serve.rs+++ b/libs/cua-driver/rust/crates/cua-driver/src/serve.rs@@ -675,12 +675,14 @@ pub async fn run_serve(// fields are ignored.//// `capabilities` is sourced from the centralised- // `cua_driver_core::tool::default_capabilities_for`- // name → tokens map so daemon responses match the- // core MCP capability contract.+ // name + concrete-schema resolver so daemon+ // responses match the core MCP capability contract.let tools: Vec<serde_json::Value> = reg.iter_defs().map(|(name, def)| {- let caps = cua_driver_core::tool::default_capabilities_for(name);+ let caps = cua_driver_core::tool::advertised_capabilities_for(+ name,+ &def.input_schema,+ );let risk = cua_driver_core::authorization::risk_metadata_json(name);serde_json::json!({"name": name,@@ -1303,7 +1305,10 @@ pub async fn run_serve(// above for rationale (capabilities map, etc.).let tools: Vec<serde_json::Value> = reg.iter_defs().map(|(name, def)| {- let caps = cua_driver_core::tool::default_capabilities_for(name);+ let caps = cua_driver_core::tool::advertised_capabilities_for(+ name,+ &def.input_schema,+ );let risk = cua_driver_core::authorization::risk_metadata_json(name);serde_json::json!({"name": name,diff --git a/libs/cua-driver/rust/crates/cua-driver/tests/protocol_schema_test.rs b/libs/cua-driver/rust/crates/cua-driver/tests/protocol_schema_test.rsindex 769fe15ecb..9a738c65ca 100644--- a/libs/cua-driver/rust/crates/cua-driver/tests/protocol_schema_test.rs+++ b/libs/cua-driver/rust/crates/cua-driver/tests/protocol_schema_test.rs@@ -8,6 +8,7 @@#![cfg(any(target_os = "linux", target_os = "macos", target_os = "windows"))]use cua_driver_testkit::{Driver, McpDriver, RawDriver};+use std::collections::BTreeSet;#[test]fn tools_list_schema_shape() {@@ -109,21 +110,53 @@ fn tools_list_schema_shape() {}}- for tool in [+ const DELIVERY_MODE_TOOLS: &[&str] = &["click","double_click","right_click",+ "drag","type_text","press_key","hotkey","scroll",- ] {+ "browser_dialog",+ ];+ for tool in DELIVERY_MODE_TOOLS {let delivery = &properties(tool)["delivery_mode"];assert!(enum_contains(delivery, "background") && enum_contains(delivery, "foreground"),"{tool}.delivery_mode should advertise background and foreground: {delivery:?}");}+ let schema_tools: BTreeSet<&str> = tools+ .iter()+ .filter(|tool| tool["inputSchema"]["properties"]["delivery_mode"].is_object())+ .filter_map(|tool| tool["name"].as_str())+ .collect();+ let capability_tools: BTreeSet<&str> = tools+ .iter()+ .filter(|tool| {+ tool["capabilities"].as_array().is_some_and(|capabilities| {+ capabilities+ .iter()+ .any(|capability| capability == "input.delivery_mode")+ })+ })+ .filter_map(|tool| tool["name"].as_str())+ .collect();+ let expected_tools: BTreeSet<&str> = DELIVERY_MODE_TOOLS.iter().copied().collect();+ assert_eq!(+ schema_tools, expected_tools,+ "unexpected delivery_mode schema set"+ );+ assert_eq!(+ capability_tools, expected_tools,+ "input.delivery_mode must match the exact runtime schema support set"+ );+ assert_eq!(+ list_resp["result"]["capability_version"], "1",+ "adding one capability token is additive and must not bump the vocabulary version"+ );// Capture scope belongs to the session lifecycle on every platform. The// action-level `scope` selects a coordinate/transport form but cannot// override the session policy enforced by the registry.diff --git a/libs/cua-driver/rust/crates/cua-driver/tests/schema_consistency_test.rs b/libs/cua-driver/rust/crates/cua-driver/tests/schema_consistency_test.rsindex f7a6aa9e85..a7874dea40 100644--- a/libs/cua-driver/rust/crates/cua-driver/tests/schema_consistency_test.rs+++ b/libs/cua-driver/rust/crates/cua-driver/tests/schema_consistency_test.rs@@ -16,6 +16,7 @@use cua_driver_contract::{compatibility::schema_subset_violations, manifest, Platform, SchemaMode,};+use cua_driver_core::tool::advertised_capabilities_for;use cua_driver_core::tool_schema::shared_schema_violations;use cua_driver_testkit::RawDriver;use serde_json::{json, Value};@@ -64,6 +65,22 @@ fn registered_tool_contracts_match_on_active_backend() {.cloned().unwrap_or_else(|| json!({}));violations.extend(shared_schema_violations(name, &schema));++ let schema_accepts_delivery_mode = schema+ .pointer("/properties/delivery_mode")+ .is_some_and(Value::is_object);+ let advertises_delivery_mode =+ tool["capabilities"].as_array().is_some_and(|capabilities| {+ capabilities+ .iter()+ .any(|capability| capability == "input.delivery_mode")+ });+ if schema_accepts_delivery_mode != advertises_delivery_mode {+ violations.push(format!(+ "{name}: delivery_mode schema={schema_accepts_delivery_mode} but \+ input.delivery_mode capability={advertises_delivery_mode}"+ ));+ }}assert!(@@ -163,12 +180,13 @@ fn portable_desktop_contracts_are_accepted_by_active_backend() {}}- if live["capabilities"] != json!(contract.capabilities) {+ let expected_capabilities = advertised_capabilities_for(&contract.name, live_schema);+ if live["capabilities"] != json!(expected_capabilities) {violations.push(format!(- "{}: live capabilities {} differ from portable capabilities {}",+ "{}: live capabilities {} differ from schema-aware capabilities {}",contract.name,live["capabilities"],- json!(contract.capabilities)+ json!(expected_capabilities)));}}