Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
fix(cua-driver): advertise refusals in the MCP outputSchema (#2968)
Francesco Bonaccidc6f32cd4d32
diff --git a/docs/content/docs/reference/cua-driver/contracts.mdx b/docs/content/docs/reference/cua-driver/contracts.mdxindex 6b19100028..bf11d01219 100644--- a/docs/content/docs/reference/cua-driver/contracts.mdx+++ b/docs/content/docs/reference/cua-driver/contracts.mdx@@ -121,6 +121,42 @@ The rejected combinations are enforced. Desktop scope is foreground and pixel-on---+## What a tool returns++An action tool answers with one of two payloads, and MCP holds **both** to the tool's advertised `outputSchema` — the schema covers every `structuredContent` a tool emits, refusals included. So `outputSchema` is an `anyOf` of two variants.++**Success variant.** Closed: exactly these keys, nothing else.++| Field | Required | Values |+|---|---|---|+| `effect` | ✅ | `confirmed`, `partial`, `unverifiable`, `suspected_noop`, `refused` |+| `route` | ✅ | `accessibility`, `synthetic_events`, `global_input`, `system_api`, `dom`, `trusted_input` |+| `delivery` | — | `{mode: background \| foreground \| not_applicable \| unknown}` |+| `evidence` | — | what was read back to justify `effect` |+| `escalation` | — | which rung to try next, and why this one fell short |++**Refusal variant.** Accompanies `isError: true` and carries diagnostics instead of an effect. Two shapes are in service:++```json+{"status": "refused",+ "refusal": {"code": "stale_element_token",+ "message": "element_token is stale; call get_window_state again to refresh"}}+```++```json+{"code": "window_target_not_found", "effect": "refused", "candidates": [], "pid": 4711}+```++This variant is deliberately open — refusals carry tool-specific diagnostic keys (`candidates`, `detail`, `pid`) that help a caller recover. The marker keys `refusal`, `status`, and `code` are what identify a payload as a refusal rather than a malformed success.++<Callout type="warn">+Branch on the refusal `code`, and surface the `content` text — it states the recovery step in plain language. `stale_element_token` means re-run `get_window_state` and retry with fresh indices; it does **not** mean the accessibility route is unavailable. Agents that treat a refusal as a dead route tend to abandon the element path and fall back to blind pixel clicking, which is both slower and unverifiable.+</Callout>++Success payloads stay strictly validated: an unknown key on the success shape is still a contract violation and will not be quietly accepted through the refusal variant.++---+## Platform support| Capability | Windows | macOS | Linux |diff --git a/libs/cua-driver/rust/Cargo.lock b/libs/cua-driver/rust/Cargo.lockindex 449a4a2ec8..1b91afcff8 100644--- a/libs/cua-driver/rust/Cargo.lock+++ b/libs/cua-driver/rust/Cargo.lock@@ -1043,6 +1043,7 @@ dependencies = ["cua-driver-contract","futures-util","image",+ "jsonschema","libc","regex","regorus",diff --git a/libs/cua-driver/rust/crates/cua-driver-contract/src/lib.rs b/libs/cua-driver/rust/crates/cua-driver-contract/src/lib.rsindex 2e723bb95f..60558739aa 100644--- a/libs/cua-driver/rust/crates/cua-driver-contract/src/lib.rs+++ b/libs/cua-driver/rust/crates/cua-driver-contract/src/lib.rs@@ -35,13 +35,14 @@ pub use inputs::{SetWindowFrameInput, StartSessionInput, ToolInput, TypeTextInput,};pub use outputs::{- ActionDelivery, ActionDeliveryMode, ActionEffect, ActionEscalation, ActionEscalationReason,- ActionEscalationTarget, ActionEvidence, ActionEvidenceKind, ActionResult,- ActionResultValidationError, ActionRoute, ClipboardReadOutput, ClipboardWriteOutput,- CursorMotionOutput, CursorPointOutput, CursorPositionOutput, CursorThemeOutput,- CursorVisualOutput, DesktopStateOutput, EffectiveScope, EndSessionOutput,- GetAgentCursorStateOutput, ScreenSizeOutput, SessionStateOutput, SetAgentCursorEnabledOutput,- SetAgentCursorMotionOutput, SetAgentCursorThemeOutput, StartSessionOutput, ToolOutput,+ advertised_output_schema, refusal_envelope_schema, ActionDelivery, ActionDeliveryMode,+ ActionEffect, ActionEscalation, ActionEscalationReason, ActionEscalationTarget, ActionEvidence,+ ActionEvidenceKind, ActionResult, ActionResultValidationError, ActionRoute,+ ClipboardReadOutput, ClipboardWriteOutput, CursorMotionOutput, CursorPointOutput,+ CursorPositionOutput, CursorThemeOutput, CursorVisualOutput, DesktopStateOutput,+ EffectiveScope, EndSessionOutput, GetAgentCursorStateOutput, ScreenSizeOutput,+ SessionStateOutput, SetAgentCursorEnabledOutput, SetAgentCursorMotionOutput,+ SetAgentCursorThemeOutput, StartSessionOutput, ToolOutput,};pub use verification::{BoundsExpectation, ElementPredicate, ElementSelector, PredicateOutcome, StatePredicate,diff --git a/libs/cua-driver/rust/crates/cua-driver-contract/src/outputs.rs b/libs/cua-driver/rust/crates/cua-driver-contract/src/outputs.rsindex eaea4cea46..34430f292d 100644--- a/libs/cua-driver/rust/crates/cua-driver-contract/src/outputs.rs+++ b/libs/cua-driver/rust/crates/cua-driver-contract/src/outputs.rs@@ -18,6 +18,43 @@ pub trait ToolOutput: Serialize + DeserializeOwned + JsonSchema {}}+/// Schema for the refusal payload a tool emits alongside `isError: true`.+///+/// Refusals answer with a diagnostic shape rather than the success shape, and+/// two are in service: `{"status":"refused","refusal":{code,message,detail}}`+/// on the element-token and daemon paths, and `{"code":…,"effect":"refused",…}`+/// on the window-target paths. Both carry tool-specific diagnostic keys, so the+/// envelope stays open — the marker keys are what make it recognisably a+/// refusal rather than a malformed success.+pub fn refusal_envelope_schema() -> Value {+ serde_json::json!({+ "type": "object",+ "additionalProperties": true,+ "anyOf": [+ {"required": ["refusal"]},+ {"required": ["status"]},+ {"required": ["code"]},+ ],+ })+}++/// Wrap a success schema into the shape advertised as the MCP `outputSchema`.+///+/// MCP requires every `structuredContent` a tool emits to validate against its+/// advertised `outputSchema` — including payloads that accompany+/// `isError: true`. Advertising the success shape alone made strict clients+/// reject refusals outright, replacing the actionable message the driver had+/// already placed in `content` ("element_token is stale; call get_window_state+/// again to refresh") with an opaque schema-validation error. Agents then had+/// no signal to re-snapshot and fell back to blind pixel clicking.+///+/// The success variant is kept exactly as generated — still closed, so success+/// payloads stay strictly checked — and the refusal envelope joins it as a+/// sibling variant.+pub fn advertised_output_schema(success: Value) -> Value {+ serde_json::json!({ "anyOf": [success, refusal_envelope_schema()] })+}+fn output_schema_with_additional_properties<T: JsonSchema>(additional_properties: bool) -> Value {let mut settings = SchemaSettings::draft2020_12();settings.inline_subschemas = true;diff --git a/libs/cua-driver/rust/crates/cua-driver-core/Cargo.toml b/libs/cua-driver/rust/crates/cua-driver-core/Cargo.tomlindex 4fddd8f175..508d1c251d 100644--- a/libs/cua-driver/rust/crates/cua-driver-core/Cargo.toml+++ b/libs/cua-driver/rust/crates/cua-driver-core/Cargo.toml@@ -44,6 +44,13 @@ libc = "0.2"# a duplicate crate version.[dev-dependencies]tempfile = "3"+# Test-only. Asserts the advertised `outputSchema` the way a strict MCP client+# does — by validating real payloads against it. The runtime itself validates+# success output by deserialising into the Rust type, so this stays out of the+# shipped dependency graph.+# Version matches the one regorus already pulls into this crate's graph, so+# the test dependency reuses that build instead of adding a second copy.+jsonschema = { version = "0.46", default-features = false }[target.'cfg(windows)'.dependencies]windows = { version = "0.58", features = ["Win32_Foundation", "Win32_UI_WindowsAndMessaging"] }diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/tool.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/tool.rsindex ceab188de1..ed0fc5b3a4 100644--- a/libs/cua-driver/rust/crates/cua-driver-core/src/tool.rs+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/tool.rs@@ -187,10 +187,17 @@ impl ToolDef {cua_driver_contract::tool_success_output_schema(&self.name)};if let Some(output_schema) = output_schema {+ // Advertise the refusal envelope alongside the success shape. MCP+ // holds every `structuredContent` we emit — refusals included — to+ // the advertised schema, and a success-only schema made strict+ // clients discard our refusal message in favour of a schema error.entry.as_object_mut().expect("tool list entry is an object")- .insert("outputSchema".into(), output_schema);+ .insert(+ "outputSchema".into(),+ cua_driver_contract::advertised_output_schema(output_schema),+ );}entry}@@ -4603,27 +4610,81 @@ mod capability_tests {assert!(entry["capabilities"].is_array());}+ fn action_tool_entry(name: &str) -> serde_json::Value {+ ToolDef {+ name: name.into(),+ description: "Action.".into(),+ input_schema: serde_json::json!({"type":"object","properties":{}}),+ read_only: false,+ destructive: false,+ idempotent: false,+ open_world: true,+ }+ .to_list_entry()+ }+#[test]fn action_tools_advertise_the_same_closed_output_schema() {let expected =<cua_driver_contract::ActionResult as cua_driver_contract::ToolOutput>::output_schema();for name in ["click", "browser_click", "browser_pointer", "browser_type"] {- let def = ToolDef {- name: name.into(),- description: "Action.".into(),- input_schema: serde_json::json!({"type":"object","properties":{}}),- read_only: false,- destructive: false,- idempotent: false,- open_world: true,- };- let entry = def.to_list_entry();- assert_eq!(entry["outputSchema"], expected, "{name}");- assert_eq!(- entry["outputSchema"]["additionalProperties"], false,- "{name}"+ let entry = action_tool_entry(name);+ // The success variant is unchanged and still closed; it now sits+ // beside the refusal envelope instead of standing alone.+ let success = &entry["outputSchema"]["anyOf"][0];+ assert_eq!(success, &expected, "{name}");+ assert_eq!(success["additionalProperties"], false, "{name}");+ }+ }++ /// Regression: refusals must validate against the advertised schema.+ ///+ /// Both payloads below are verbatim captures from a live `cua-driver mcp`+ /// stdio session. Advertising only the success shape made strict MCP+ /// clients reject them with `-32602`, which discarded the refusal message+ /// the driver had put in `content` and left agents with no signal to+ /// re-snapshot.+ #[test]+ fn advertised_output_schema_accepts_live_refusal_payloads() {+ let schema = &action_tool_entry("click")["outputSchema"];+ let compiled = jsonschema::validator_for(schema).expect("advertised schema compiles");++ let stale_element_token = serde_json::json!({+ "refusal": {+ "code": "stale_element_token",+ "message": "element_token is stale; call get_window_state again to refresh"+ },+ "status": "refused"+ });+ let window_target_not_found = serde_json::json!({+ "candidates": [],+ "code": "window_target_not_found",+ "effect": "refused",+ "pid": 999_999+ });+ let success = serde_json::json!({+ "delivery": {"mode": "background"},+ "effect": "unverifiable",+ "route": "accessibility"+ });++ for payload in [&stale_element_token, &window_target_not_found, &success] {+ assert!(+ compiled.is_valid(payload),+ "advertised schema rejected {payload}");}++ // The success variant stays strict: an unknown key is still a bug, and+ // must not be laundered through the permissive refusal variant.+ assert!(+ !compiled.is_valid(&serde_json::json!({+ "effect": "confirmed",+ "route": "accessibility",+ "bogus_key": true+ })),+ "a malformed success payload must not validate"+ );}#[test]@@ -4665,8 +4726,9 @@ mod capability_tests {open_world: false,}.to_list_entry();- let z_index =- &entry["outputSchema"]["properties"]["windows"]["items"]["properties"]["z_index"];+ // anyOf[0] is the success variant; anyOf[1] is the refusal envelope.+ let z_index = &entry["outputSchema"]["anyOf"][0]["properties"]["windows"]["items"]+ ["properties"]["z_index"];assert_eq!(z_index["type"], serde_json::json!(["integer", "null"]));assert!(z_index["description"].as_str()diff --git a/libs/cua-driver/rust/crates/cua-driver/tests/schema_consistency_test.rs b/libs/cua-driver/rust/crates/cua-driver/tests/schema_consistency_test.rsindex 41ea9e7f10..f971947a8e 100644--- a/libs/cua-driver/rust/crates/cua-driver/tests/schema_consistency_test.rs+++ b/libs/cua-driver/rust/crates/cua-driver/tests/schema_consistency_test.rs@@ -84,12 +84,17 @@ fn registered_tool_contracts_match_on_active_backend() {}if is_action_result_tool(name) {- let expected = ActionResult::output_schema();+ // The live surface advertises the success shape beside the refusal+ // envelope, because MCP holds every `structuredContent` — refusals+ // included — to the advertised schema. The success variant must+ // still be the shared ActionResult schema, byte for byte.+ let expected =+ cua_driver_contract::advertised_output_schema(ActionResult::output_schema());let actual = tool.get("outputSchema");if actual != Some(&expected) {violations.push(format!(- "{name}: live outputSchema does not equal the shared ActionResult schema; \- actual={} expected={expected}",+ "{name}: live outputSchema does not equal the advertised ActionResult schema \+ (success variant + refusal envelope); actual={} expected={expected}",actual.unwrap_or(&Value::Null)));}