Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

fix(cua-driver): advertise coordinate scrolling support (#5149)

injaneityf53daa6c2a6f
diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/tool.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/tool.rs
index 7bf3a58ad7..7ff2e42c9d 100644
--- a/libs/cua-driver/rust/crates/cua-driver-core/src/tool.rs
+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/tool.rs
@@ -668,6 +668,8 @@ fn deprecated_name_note(old: &str, new: &str) -> String {
/// `input.pointer.click.right`, `input.pointer.click.double`,
/// `input.pointer.drag`, `input.pointer.scroll`,
/// `input.pointer.move`, `input.pointer.button` (raw down/up)
+/// - `input.scroll.coordinates` (the live scroll schema accepts numeric x/y;
+/// does not imply focus-free background delivery)
/// - `input.keyboard.type`, `input.keyboard.hotkey`,
/// `input.keyboard.press`
/// - `input.delivery_mode` (the live tool schema accepts the shared
@@ -857,10 +859,10 @@ pub fn default_capabilities_for(tool_name: &str) -> Vec<String> {
/// Capabilities advertised by a concrete runtime tool definition.
///
/// Most capabilities are stable properties of a tool name and come from the
-/// typed portable contract (or the legacy runtime-only map). Delivery mode is
-/// different: the typed desktop SDK intentionally exposes a narrower,
-/// desktop-only input while the live platform schemas additionally accept
-/// window-targeted `delivery_mode`. Deriving this one token from the concrete
+/// typed portable contract (or the legacy runtime-only map). Delivery mode and
+/// scroll coordinates are derived from the live schema. The typed desktop SDK
+/// exposes a narrower, desktop-only input while the live schemas additionally accept
+/// window-targeted `delivery_mode`. Deriving these tokens from the concrete
/// schema keeps `tools/list` truthful on every platform and prevents either
/// overclaiming a tool that cannot accept the field or omitting support from a
/// richer live schema.
@@ -874,6 +876,16 @@ pub fn advertised_capabilities_for(tool_name: &str, input_schema: &Value) -> Vec
{
capabilities.push("input.delivery_mode".into());
}
+ if tool_name == "scroll"
+ && ["x", "y"].iter().all(|field| {
+ matches!(
+ input_schema["properties"][field]["type"].as_str(),
+ Some("number" | "integer")
+ )
+ })
+ {
+ capabilities.push("input.scroll.coordinates".into());
+ }
capabilities
}
@@ -6401,6 +6413,7 @@ mod capability_tests {
"input.pointer.click.double",
"input.pointer.drag",
"input.pointer.scroll",
+ "input.scroll.coordinates",
"input.pointer.move",
"input.pointer.button",
// keyboard
@@ -6687,6 +6700,61 @@ mod capability_tests {
.any(|capability| capability == "input.delivery_mode"));
}
+ #[test]
+ fn to_list_entry_advertises_scroll_coordinates_only_when_accepted() {
+ use serde_json::json;
+
+ for (name, properties, expected) in [
+ (
+ "scroll",
+ json!({"x": {"type": "number"}, "y": {"type": "number"}}),
+ true,
+ ),
+ (
+ "scroll",
+ json!({"x": {"type": "integer"}, "y": {"type": "integer"}}),
+ true,
+ ),
+ ("scroll", json!({}), false),
+ ("scroll", json!({"x": {"type": "number"}}), false),
+ ("scroll", json!({"y": {"type": "number"}}), false),
+ (
+ "scroll",
+ json!({"x": {"type": "string"}, "y": {"type": "number"}}),
+ false,
+ ),
+ (
+ "scroll",
+ json!({"x": {"type": "number"}, "y": {"type": "string"}}),
+ false,
+ ),
+ (
+ "click",
+ json!({"x": {"type": "number"}, "y": {"type": "number"}}),
+ false,
+ ),
+ (
+ "unknown",
+ json!({"x": {"type": "number"}, "y": {"type": "number"}}),
+ false,
+ ),
+ ] {
+ let mut def = dummy_def(name);
+ def.input_schema = json!({"type": "object", "properties": properties});
+ let entry = def.to_list_entry();
+ let capabilities = entry["capabilities"].as_array().unwrap();
+ assert_eq!(
+ capabilities
+ .iter()
+ .filter(|capability| *capability == "input.scroll.coordinates")
+ .count(),
+ usize::from(expected),
+ "{name}: {}",
+ def.input_schema
+ );
+ }
+ }
+
#[test]
fn to_list_entry_includes_versioned_risk_metadata() {
let entry = dummy_def("browser_prepare").to_list_entry();
diff --git a/libs/cua-driver/rust/crates/platform-linux/src/input/delivery.rs b/libs/cua-driver/rust/crates/platform-linux/src/input/delivery.rs
index 381b4996c8..d5270cd471 100644
--- a/libs/cua-driver/rust/crates/platform-linux/src/input/delivery.rs
+++ b/libs/cua-driver/rust/crates/platform-linux/src/input/delivery.rs
@@ -97,7 +97,7 @@ impl BackgroundUnavailable {
background injection"
}
Self::FocusedInputOnly => {
- "the requested target has no focus-free input backend; the remaining XTest/X11 route can only deliver to the globally focused widget"
+ "the requested target has no focus-free input backend; the remaining input route can only deliver to the globally focused widget"
}
Self::WebKitSyntheticInput => {
"WebKitGTK rejects synthetic XSendEvent input and this session has no real target-addressed pointer backend"
@@ -130,6 +130,35 @@ pub fn background_unavailable_error(
mod tests {
use super::*;
+ #[test]
+ fn focused_input_refusal_does_not_claim_an_x11_backend() {
+ let result = background_unavailable_error(BackgroundUnavailable::FocusedInputOnly);
+ let structured = result.structured_content.as_ref().unwrap();
+ assert_eq!(structured["code"], "background_unavailable");
+ assert_eq!(result.is_error, Some(true));
+ assert_eq!(structured["escalation"]["recommended"], "foreground");
+ assert_eq!(
+ structured["suggestion"],
+ cua_driver_core::delivery::FOREGROUND_RETRY_SUGGESTION
+ );
+ let detail = structured["detail"].as_str().unwrap();
+ assert!(detail.contains("globally focused widget"));
+ assert!(!detail.contains("XTest"));
+ assert!(!detail.contains("X11"));
+ let text = result
+ .content
+ .iter()
+ .filter_map(|content| match content {
+ cua_driver_core::protocol::Content::Text { text, .. } => Some(text.as_str()),
+ _ => None,
+ })
+ .collect::<Vec<_>>()
+ .join("\n");
+ assert!(!text.contains("XTest"));
+ assert!(!text.contains("X11"));
+ assert!(text.contains("delivery_mode:\"foreground\""));
+ }
+
#[test]
fn delivery_mode_schema_advertises_two_modes() {
let s = delivery_mode_schema();