Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
fix(cua-driver): report agent cursor state before the cursor first moves (#4123)
Francesco Bonacci3dbe331f12e8
diff --git a/libs/cua-driver/contract/manifest.json b/libs/cua-driver/contract/manifest.jsonindex 9bfe1d4aa6..f448282fb4 100644--- a/libs/cua-driver/contract/manifest.json+++ b/libs/cua-driver/contract/manifest.json@@ -1007,21 +1007,28 @@"type": "object"},"position": {- "properties": {- "x": {- "format": "double",- "type": "number"+ "anyOf": [+ {+ "properties": {+ "x": {+ "format": "double",+ "type": "number"+ },+ "y": {+ "format": "double",+ "type": "number"+ }+ },+ "required": [+ "x",+ "y"+ ],+ "type": "object"},- "y": {- "format": "double",- "type": "number"+ {+ "type": "null"}- },- "required": [- "x",- "y"- ],- "type": "object"+ ]},"session": {"type": "string"diff --git a/libs/cua-driver/docs/cursor-themes.md b/libs/cua-driver/docs/cursor-themes.mdindex 4ab01093a8..4a554beb1a 100644--- a/libs/cua-driver/docs/cursor-themes.md+++ b/libs/cua-driver/docs/cursor-themes.md@@ -69,6 +69,10 @@ cua-driver set_agent_cursor_theme \cua-driver get_agent_cursor_state '{"session":"demo"}'```+`get_agent_cursor_state` always includes `position`. It is `null` until the+session cursor first moves, then `{"x": ..., "y": ...}` where the platform+tracks session cursor placement.+`set_agent_cursor_motion` changes only movement physics and visibility timing.It does not change artwork. The removed `set_agent_cursor_style` operation andits `cursor_id`, shape, color, label, size, opacity, image-path, gradient, anddiff --git a/libs/cua-driver/rust/crates/cua-driver-contract/src/outputs.rs b/libs/cua-driver/rust/crates/cua-driver-contract/src/outputs.rsindex eb517eb601..5ce0bdbfcf 100644--- a/libs/cua-driver/rust/crates/cua-driver-contract/src/outputs.rs+++ b/libs/cua-driver/rust/crates/cua-driver-contract/src/outputs.rs@@ -330,7 +330,9 @@ impl ToolOutput for SetAgentCursorThemeOutput {}pub struct GetAgentCursorStateOutput {pub session: String,pub enabled: bool,- #[schemars(required)]+ // Wire contract: the key is always present, and its value is `null` until+ // the session cursor first moves.+ #[schemars(required, schema_with = "nullable_cursor_point_schema")]pub position: Option<CursorPointOutput>,pub theme: CursorThemeOutput,pub visual_state: CursorVisualOutput,@@ -668,6 +670,11 @@ fn nullable_string_schema(_: &mut schemars::SchemaGenerator) -> schemars::Schemaschemars::json_schema!({ "anyOf": [{ "type": "string" }, { "type": "null" }] })}+fn nullable_cursor_point_schema(generator: &mut schemars::SchemaGenerator) -> schemars::Schema {+ let point = generator.subschema_for::<CursorPointOutput>();+ schemars::json_schema!({ "anyOf": [point, { "type": "null" }] })+}+fn nullable_escalation_reason_schema(_: &mut schemars::SchemaGenerator) -> schemars::Schema {schemars::json_schema!({"anyOf": [diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/mcp_result.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/mcp_result.rsindex a62d4e5a18..91406588e5 100644--- a/libs/cua-driver/rust/crates/cua-driver-core/src/mcp_result.rs+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/mcp_result.rs@@ -368,6 +368,80 @@ mod tests {assert_eq!(result["content"][1]["text"], "action completed");}+ fn agent_cursor_state(position: Value) -> Value {+ json!({+ "session": "cursor-state-test",+ "enabled": true,+ "position": position,+ "theme": {+ "id": "cua-default",+ "version": "1",+ "profile": "default",+ "reduced_motion": "auto",+ "fallback": null,+ },+ "visual_state": {+ "requested_action": "idle",+ "resolved_action": "idle",+ "modifiers": [],+ "phase": "idle",+ "frame": 0,+ "preempted_count": 0,+ },+ "motion": {+ "start_handle": 0.3,+ "end_handle": 0.3,+ "arc_size": 0.25,+ "arc_flow": 0.0,+ "spring": 0.72,+ "glide_duration_ms": 750.0,+ "dwell_after_click_ms": 400.0,+ "idle_hide_ms": 3000.0,+ "turn_radius": 12.0,+ },+ })+ }++ /// A session cursor that has never moved has no position yet. Every+ /// platform reports that as `"position": null`, and the advertised schema+ /// must accept it rather than turning the read into `tool_output_invalid`.+ #[test]+ fn agent_cursor_state_accepts_a_cursor_that_has_not_moved() {+ const TOOL: &str = "get_agent_cursor_state";+ for position in [Value::Null, json!({"x": 10.0, "y": 20.0})] {+ let success = json!({+ "content": [{"type": "text", "text": "Agent cursor state."}],+ "isError": false,+ "structuredContent": agent_cursor_state(position),+ });++ let result = conforming_tool_result(TOOL, success.clone());++ assert_conforms(TOOL, &result);+ assert_eq!(result, success);+ }+ }++ /// `position` stays required: an unmoved cursor reports `null`, it does+ /// not drop the key.+ #[test]+ fn agent_cursor_state_without_a_position_key_is_rejected() {+ const TOOL: &str = "get_agent_cursor_state";+ let mut structured = agent_cursor_state(Value::Null);+ structured.as_object_mut().unwrap().remove("position");++ let result = conforming_tool_result(+ TOOL,+ json!({"content": [], "isError": false, "structuredContent": structured}),+ );++ assert_eq!(result["isError"], true);+ assert_eq!(+ result["structuredContent"]["code"],+ TOOL_OUTPUT_INVALID_CODE+ );+ }+/// A tool with no advertised schema has no contract to violate, so its/// successful payload must not be second-guessed here.#[test]diff --git a/libs/cua-driver/rust/crates/cua-driver/tests/protocol_media_test.rs b/libs/cua-driver/rust/crates/cua-driver/tests/protocol_media_test.rsindex bf149f5bc1..d41cde33ad 100644--- a/libs/cua-driver/rust/crates/cua-driver/tests/protocol_media_test.rs+++ b/libs/cua-driver/rust/crates/cua-driver/tests/protocol_media_test.rs@@ -25,8 +25,11 @@ fn spawn_unrestricted() -> Option<RawDriver> {#[test]#[cfg(any(target_os = "macos", target_os = "windows"))]fn zoom_tool_returns_jpeg() {- //! Call zoom on a visible window and verify the result contains a JPEG image.- //! Skips gracefully if no windows are visible (headless environment).+ //! Snapshot a visible window, zoom into it, and verify the result contains+ //! a JPEG image. `zoom` crops the screenshot this session last captured, so+ //! each attempt first takes a `get_window_state` screenshot on the same+ //! connection. Skips gracefully if no windows are visible (headless+ //! environment).let Some(mut d) = RawDriver::spawn() else {return;};@@ -50,28 +53,51 @@ fn zoom_tool_returns_jpeg() {let mut tried = 0usize;let mut zoom_errors: Vec<String> = Vec::new();let mut successful_without_jpeg = false;+ let error_text = |r: &serde_json::Value| {+ r["result"]["content"]+ .as_array()+ .and_then(|items| items.iter().find_map(|c| c["text"].as_str()))+ .unwrap_or("<no error text>")+ .to_owned()+ };for win in windows.iter().take(5) {- let Some(wid) = win["window_id"].as_u64() else {+ let (Some(wid), Some(pid)) = (win["window_id"].as_u64(), win["pid"].as_u64()) else {continue;};tried += 1;+ let request_id = 2 + 2 * tried as u64;++ // zoom crops this session's latest screenshot of the window.+ d.send(&serde_json::json!({+ "jsonrpc":"2.0","id": request_id,"method":"tools/call",+ "params":{"name":"get_window_state","arguments":{+ "pid": pid,+ "window_id": wid,+ "include_screenshot": true,+ "include_accessibility_tree": false+ }}+ }));+ let snapshot = d.recv();+ if snapshot["result"]["isError"].as_bool().unwrap_or(false) {+ zoom_errors.push(format!(+ "window_id={wid}: get_window_state: {}",+ error_text(&snapshot)+ ));+ continue;+ }d.send(&serde_json::json!({- "jsonrpc":"2.0","id": 2 + tried as u64,"method":"tools/call",+ "jsonrpc":"2.0","id": request_id + 1,"method":"tools/call","params":{"name":"zoom","arguments":{"window_id": wid,+ "pid": pid,"x1": 0, "y1": 0, "x2": 100, "y2": 100}}}));let r = d.recv();if r["result"]["isError"].as_bool().unwrap_or(false) {// This window might be off-screen or not capturable — try the next.- let text = r["result"]["content"]- .as_array()- .and_then(|items| items.iter().find_map(|c| c["text"].as_str()))- .unwrap_or("<no error text>")- .to_owned();- zoom_errors.push(format!("window_id={wid}: {text}"));+ zoom_errors.push(format!("window_id={wid}: zoom: {}", error_text(&r)));continue;}let content = r["result"]["content"].as_array().expect("content array");@@ -107,13 +133,17 @@ fn zoom_tool_returns_jpeg() {eprintln!("No on-screen windows found — skipping zoom test");return;}+ // A screenshot-only get_window_state that produces no content means the+ // window capture itself failed, the same condition zoom reports as+ // `screencapture failed`.if !found_jpeg&& !successful_without_jpeg&& cfg!(target_os = "macos")&& !zoom_errors.is_empty()- && zoom_errors- .iter()- .all(|e| e.contains("screencapture failed"))+ && zoom_errors.iter().all(|e| {+ e.contains("screencapture failed")+ || e.contains("neither AX tree nor screenshot succeeded")+ }){eprintln!("No visible windows were capturable by the raw unbundled test process — skipping zoom test. \