Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

fix(cua-driver): report agent cursor state before the cursor first moves (#4123)

Francesco Bonacci3dbe331f12e8
diff --git a/libs/cua-driver/contract/manifest.json b/libs/cua-driver/contract/manifest.json
index 9bfe1d4aa6..f448282fb4 100644
--- a/libs/cua-driver/contract/manifest.json
+++ b/libs/cua-driver/contract/manifest.json
@@ -1007,21 +1007,28 @@
"type": "object"
},
"position": {
- "properties": {
- "x": {
- "format": "double",
- "type": "number"
+ "anyOf": [
+ {
+ "properties": {
+ "x": {
+ "format": "double",
+ "type": "number"
+ },
+ "y": {
+ "format": "double",
+ "type": "number"
+ }
+ },
+ "required": [
+ "x",
+ "y"
+ ],
+ "type": "object"
},
- "y": {
- "format": "double",
- "type": "number"
+ {
+ "type": "null"
}
- },
- "required": [
- "x",
- "y"
- ],
- "type": "object"
+ ]
},
"session": {
"type": "string"
diff --git a/libs/cua-driver/docs/cursor-themes.md b/libs/cua-driver/docs/cursor-themes.md
index 4ab01093a8..4a554beb1a 100644
--- a/libs/cua-driver/docs/cursor-themes.md
+++ b/libs/cua-driver/docs/cursor-themes.md
@@ -69,6 +69,10 @@ cua-driver set_agent_cursor_theme \
cua-driver get_agent_cursor_state '{"session":"demo"}'
```
+`get_agent_cursor_state` always includes `position`. It is `null` until the
+session cursor first moves, then `{"x": ..., "y": ...}` where the platform
+tracks session cursor placement.
+
`set_agent_cursor_motion` changes only movement physics and visibility timing.
It does not change artwork. The removed `set_agent_cursor_style` operation and
its `cursor_id`, shape, color, label, size, opacity, image-path, gradient, and
diff --git a/libs/cua-driver/rust/crates/cua-driver-contract/src/outputs.rs b/libs/cua-driver/rust/crates/cua-driver-contract/src/outputs.rs
index eb517eb601..5ce0bdbfcf 100644
--- a/libs/cua-driver/rust/crates/cua-driver-contract/src/outputs.rs
+++ b/libs/cua-driver/rust/crates/cua-driver-contract/src/outputs.rs
@@ -330,7 +330,9 @@ impl ToolOutput for SetAgentCursorThemeOutput {}
pub struct GetAgentCursorStateOutput {
pub session: String,
pub enabled: bool,
- #[schemars(required)]
+ // Wire contract: the key is always present, and its value is `null` until
+ // the session cursor first moves.
+ #[schemars(required, schema_with = "nullable_cursor_point_schema")]
pub position: Option<CursorPointOutput>,
pub theme: CursorThemeOutput,
pub visual_state: CursorVisualOutput,
@@ -668,6 +670,11 @@ fn nullable_string_schema(_: &mut schemars::SchemaGenerator) -> schemars::Schema
schemars::json_schema!({ "anyOf": [{ "type": "string" }, { "type": "null" }] })
}
+fn nullable_cursor_point_schema(generator: &mut schemars::SchemaGenerator) -> schemars::Schema {
+ let point = generator.subschema_for::<CursorPointOutput>();
+ schemars::json_schema!({ "anyOf": [point, { "type": "null" }] })
+}
+
fn nullable_escalation_reason_schema(_: &mut schemars::SchemaGenerator) -> schemars::Schema {
schemars::json_schema!({
"anyOf": [
diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/mcp_result.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/mcp_result.rs
index a62d4e5a18..91406588e5 100644
--- a/libs/cua-driver/rust/crates/cua-driver-core/src/mcp_result.rs
+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/mcp_result.rs
@@ -368,6 +368,80 @@ mod tests {
assert_eq!(result["content"][1]["text"], "action completed");
}
+ fn agent_cursor_state(position: Value) -> Value {
+ json!({
+ "session": "cursor-state-test",
+ "enabled": true,
+ "position": position,
+ "theme": {
+ "id": "cua-default",
+ "version": "1",
+ "profile": "default",
+ "reduced_motion": "auto",
+ "fallback": null,
+ },
+ "visual_state": {
+ "requested_action": "idle",
+ "resolved_action": "idle",
+ "modifiers": [],
+ "phase": "idle",
+ "frame": 0,
+ "preempted_count": 0,
+ },
+ "motion": {
+ "start_handle": 0.3,
+ "end_handle": 0.3,
+ "arc_size": 0.25,
+ "arc_flow": 0.0,
+ "spring": 0.72,
+ "glide_duration_ms": 750.0,
+ "dwell_after_click_ms": 400.0,
+ "idle_hide_ms": 3000.0,
+ "turn_radius": 12.0,
+ },
+ })
+ }
+
+ /// A session cursor that has never moved has no position yet. Every
+ /// platform reports that as `"position": null`, and the advertised schema
+ /// must accept it rather than turning the read into `tool_output_invalid`.
+ #[test]
+ fn agent_cursor_state_accepts_a_cursor_that_has_not_moved() {
+ const TOOL: &str = "get_agent_cursor_state";
+ for position in [Value::Null, json!({"x": 10.0, "y": 20.0})] {
+ let success = json!({
+ "content": [{"type": "text", "text": "Agent cursor state."}],
+ "isError": false,
+ "structuredContent": agent_cursor_state(position),
+ });
+
+ let result = conforming_tool_result(TOOL, success.clone());
+
+ assert_conforms(TOOL, &result);
+ assert_eq!(result, success);
+ }
+ }
+
+ /// `position` stays required: an unmoved cursor reports `null`, it does
+ /// not drop the key.
+ #[test]
+ fn agent_cursor_state_without_a_position_key_is_rejected() {
+ const TOOL: &str = "get_agent_cursor_state";
+ let mut structured = agent_cursor_state(Value::Null);
+ structured.as_object_mut().unwrap().remove("position");
+
+ let result = conforming_tool_result(
+ TOOL,
+ json!({"content": [], "isError": false, "structuredContent": structured}),
+ );
+
+ assert_eq!(result["isError"], true);
+ assert_eq!(
+ result["structuredContent"]["code"],
+ TOOL_OUTPUT_INVALID_CODE
+ );
+ }
+
/// A tool with no advertised schema has no contract to violate, so its
/// successful payload must not be second-guessed here.
#[test]
diff --git a/libs/cua-driver/rust/crates/cua-driver/tests/protocol_media_test.rs b/libs/cua-driver/rust/crates/cua-driver/tests/protocol_media_test.rs
index bf149f5bc1..d41cde33ad 100644
--- a/libs/cua-driver/rust/crates/cua-driver/tests/protocol_media_test.rs
+++ b/libs/cua-driver/rust/crates/cua-driver/tests/protocol_media_test.rs
@@ -25,8 +25,11 @@ fn spawn_unrestricted() -> Option<RawDriver> {
#[test]
#[cfg(any(target_os = "macos", target_os = "windows"))]
fn zoom_tool_returns_jpeg() {
- //! Call zoom on a visible window and verify the result contains a JPEG image.
- //! Skips gracefully if no windows are visible (headless environment).
+ //! Snapshot a visible window, zoom into it, and verify the result contains
+ //! a JPEG image. `zoom` crops the screenshot this session last captured, so
+ //! each attempt first takes a `get_window_state` screenshot on the same
+ //! connection. Skips gracefully if no windows are visible (headless
+ //! environment).
let Some(mut d) = RawDriver::spawn() else {
return;
};
@@ -50,28 +53,51 @@ fn zoom_tool_returns_jpeg() {
let mut tried = 0usize;
let mut zoom_errors: Vec<String> = Vec::new();
let mut successful_without_jpeg = false;
+ let error_text = |r: &serde_json::Value| {
+ r["result"]["content"]
+ .as_array()
+ .and_then(|items| items.iter().find_map(|c| c["text"].as_str()))
+ .unwrap_or("<no error text>")
+ .to_owned()
+ };
for win in windows.iter().take(5) {
- let Some(wid) = win["window_id"].as_u64() else {
+ let (Some(wid), Some(pid)) = (win["window_id"].as_u64(), win["pid"].as_u64()) else {
continue;
};
tried += 1;
+ let request_id = 2 + 2 * tried as u64;
+
+ // zoom crops this session's latest screenshot of the window.
+ d.send(&serde_json::json!({
+ "jsonrpc":"2.0","id": request_id,"method":"tools/call",
+ "params":{"name":"get_window_state","arguments":{
+ "pid": pid,
+ "window_id": wid,
+ "include_screenshot": true,
+ "include_accessibility_tree": false
+ }}
+ }));
+ let snapshot = d.recv();
+ if snapshot["result"]["isError"].as_bool().unwrap_or(false) {
+ zoom_errors.push(format!(
+ "window_id={wid}: get_window_state: {}",
+ error_text(&snapshot)
+ ));
+ continue;
+ }
d.send(&serde_json::json!({
- "jsonrpc":"2.0","id": 2 + tried as u64,"method":"tools/call",
+ "jsonrpc":"2.0","id": request_id + 1,"method":"tools/call",
"params":{"name":"zoom","arguments":{
"window_id": wid,
+ "pid": pid,
"x1": 0, "y1": 0, "x2": 100, "y2": 100
}}
}));
let r = d.recv();
if r["result"]["isError"].as_bool().unwrap_or(false) {
// This window might be off-screen or not capturable — try the next.
- let text = r["result"]["content"]
- .as_array()
- .and_then(|items| items.iter().find_map(|c| c["text"].as_str()))
- .unwrap_or("<no error text>")
- .to_owned();
- zoom_errors.push(format!("window_id={wid}: {text}"));
+ zoom_errors.push(format!("window_id={wid}: zoom: {}", error_text(&r)));
continue;
}
let content = r["result"]["content"].as_array().expect("content array");
@@ -107,13 +133,17 @@ fn zoom_tool_returns_jpeg() {
eprintln!("No on-screen windows found — skipping zoom test");
return;
}
+ // A screenshot-only get_window_state that produces no content means the
+ // window capture itself failed, the same condition zoom reports as
+ // `screencapture failed`.
if !found_jpeg
&& !successful_without_jpeg
&& cfg!(target_os = "macos")
&& !zoom_errors.is_empty()
- && zoom_errors
- .iter()
- .all(|e| e.contains("screencapture failed"))
+ && zoom_errors.iter().all(|e| {
+ e.contains("screencapture failed")
+ || e.contains("neither AX tree nor screenshot succeeded")
+ })
{
eprintln!(
"No visible windows were capturable by the raw unbundled test process — skipping zoom test. \