Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
fix(cua-driver): advertise coordinate scrolling support (#5149)
injaneityf53daa6c2a6f
diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/tool.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/tool.rsindex 7bf3a58ad7..7ff2e42c9d 100644--- a/libs/cua-driver/rust/crates/cua-driver-core/src/tool.rs+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/tool.rs@@ -668,6 +668,8 @@ fn deprecated_name_note(old: &str, new: &str) -> String {/// `input.pointer.click.right`, `input.pointer.click.double`,/// `input.pointer.drag`, `input.pointer.scroll`,/// `input.pointer.move`, `input.pointer.button` (raw down/up)+/// - `input.scroll.coordinates` (the live scroll schema accepts numeric x/y;+/// does not imply focus-free background delivery)/// - `input.keyboard.type`, `input.keyboard.hotkey`,/// `input.keyboard.press`/// - `input.delivery_mode` (the live tool schema accepts the shared@@ -857,10 +859,10 @@ pub fn default_capabilities_for(tool_name: &str) -> Vec<String> {/// Capabilities advertised by a concrete runtime tool definition.////// Most capabilities are stable properties of a tool name and come from the-/// typed portable contract (or the legacy runtime-only map). Delivery mode is-/// different: the typed desktop SDK intentionally exposes a narrower,-/// desktop-only input while the live platform schemas additionally accept-/// window-targeted `delivery_mode`. Deriving this one token from the concrete+/// typed portable contract (or the legacy runtime-only map). Delivery mode and+/// scroll coordinates are derived from the live schema. The typed desktop SDK+/// exposes a narrower, desktop-only input while the live schemas additionally accept+/// window-targeted `delivery_mode`. Deriving these tokens from the concrete/// schema keeps `tools/list` truthful on every platform and prevents either/// overclaiming a tool that cannot accept the field or omitting support from a/// richer live schema.@@ -874,6 +876,16 @@ pub fn advertised_capabilities_for(tool_name: &str, input_schema: &Value) -> Vec{capabilities.push("input.delivery_mode".into());}+ if tool_name == "scroll"+ && ["x", "y"].iter().all(|field| {+ matches!(+ input_schema["properties"][field]["type"].as_str(),+ Some("number" | "integer")+ )+ })+ {+ capabilities.push("input.scroll.coordinates".into());+ }capabilities}@@ -6401,6 +6413,7 @@ mod capability_tests {"input.pointer.click.double","input.pointer.drag","input.pointer.scroll",+ "input.scroll.coordinates","input.pointer.move","input.pointer.button",// keyboard@@ -6687,6 +6700,61 @@ mod capability_tests {.any(|capability| capability == "input.delivery_mode"));}+ #[test]+ fn to_list_entry_advertises_scroll_coordinates_only_when_accepted() {+ use serde_json::json;++ for (name, properties, expected) in [+ (+ "scroll",+ json!({"x": {"type": "number"}, "y": {"type": "number"}}),+ true,+ ),+ (+ "scroll",+ json!({"x": {"type": "integer"}, "y": {"type": "integer"}}),+ true,+ ),+ ("scroll", json!({}), false),+ ("scroll", json!({"x": {"type": "number"}}), false),+ ("scroll", json!({"y": {"type": "number"}}), false),+ (+ "scroll",+ json!({"x": {"type": "string"}, "y": {"type": "number"}}),+ false,+ ),+ (+ "scroll",+ json!({"x": {"type": "number"}, "y": {"type": "string"}}),+ false,+ ),+ (+ "click",+ json!({"x": {"type": "number"}, "y": {"type": "number"}}),+ false,+ ),+ (+ "unknown",+ json!({"x": {"type": "number"}, "y": {"type": "number"}}),+ false,+ ),+ ] {+ let mut def = dummy_def(name);+ def.input_schema = json!({"type": "object", "properties": properties});+ let entry = def.to_list_entry();+ let capabilities = entry["capabilities"].as_array().unwrap();+ assert_eq!(+ capabilities+ .iter()+ .filter(|capability| *capability == "input.scroll.coordinates")+ .count(),+ usize::from(expected),+ "{name}: {}",+ def.input_schema+ );+ }+ }+#[test]fn to_list_entry_includes_versioned_risk_metadata() {let entry = dummy_def("browser_prepare").to_list_entry();diff --git a/libs/cua-driver/rust/crates/platform-linux/src/input/delivery.rs b/libs/cua-driver/rust/crates/platform-linux/src/input/delivery.rsindex 381b4996c8..d5270cd471 100644--- a/libs/cua-driver/rust/crates/platform-linux/src/input/delivery.rs+++ b/libs/cua-driver/rust/crates/platform-linux/src/input/delivery.rs@@ -97,7 +97,7 @@ impl BackgroundUnavailable {background injection"}Self::FocusedInputOnly => {- "the requested target has no focus-free input backend; the remaining XTest/X11 route can only deliver to the globally focused widget"+ "the requested target has no focus-free input backend; the remaining input route can only deliver to the globally focused widget"}Self::WebKitSyntheticInput => {"WebKitGTK rejects synthetic XSendEvent input and this session has no real target-addressed pointer backend"@@ -130,6 +130,35 @@ pub fn background_unavailable_error(mod tests {use super::*;+ #[test]+ fn focused_input_refusal_does_not_claim_an_x11_backend() {+ let result = background_unavailable_error(BackgroundUnavailable::FocusedInputOnly);+ let structured = result.structured_content.as_ref().unwrap();+ assert_eq!(structured["code"], "background_unavailable");+ assert_eq!(result.is_error, Some(true));+ assert_eq!(structured["escalation"]["recommended"], "foreground");+ assert_eq!(+ structured["suggestion"],+ cua_driver_core::delivery::FOREGROUND_RETRY_SUGGESTION+ );+ let detail = structured["detail"].as_str().unwrap();+ assert!(detail.contains("globally focused widget"));+ assert!(!detail.contains("XTest"));+ assert!(!detail.contains("X11"));+ let text = result+ .content+ .iter()+ .filter_map(|content| match content {+ cua_driver_core::protocol::Content::Text { text, .. } => Some(text.as_str()),+ _ => None,+ })+ .collect::<Vec<_>>()+ .join("\n");+ assert!(!text.contains("XTest"));+ assert!(!text.contains("X11"));+ assert!(text.contains("delivery_mode:\"foreground\""));+ }+#[test]fn delivery_mode_schema_advertises_two_modes() {let s = delivery_mode_schema();