Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

fix(cua-driver): advertise refusals in the MCP outputSchema (#2968)

Francesco Bonaccidc6f32cd4d32
diff --git a/docs/content/docs/reference/cua-driver/contracts.mdx b/docs/content/docs/reference/cua-driver/contracts.mdx
index 6b19100028..bf11d01219 100644
--- a/docs/content/docs/reference/cua-driver/contracts.mdx
+++ b/docs/content/docs/reference/cua-driver/contracts.mdx
@@ -121,6 +121,42 @@ The rejected combinations are enforced. Desktop scope is foreground and pixel-on
---
+## What a tool returns
+
+An action tool answers with one of two payloads, and MCP holds **both** to the tool's advertised `outputSchema` — the schema covers every `structuredContent` a tool emits, refusals included. So `outputSchema` is an `anyOf` of two variants.
+
+**Success variant.** Closed: exactly these keys, nothing else.
+
+| Field | Required | Values |
+|---|---|---|
+| `effect` | ✅ | `confirmed`, `partial`, `unverifiable`, `suspected_noop`, `refused` |
+| `route` | ✅ | `accessibility`, `synthetic_events`, `global_input`, `system_api`, `dom`, `trusted_input` |
+| `delivery` | — | `{mode: background \| foreground \| not_applicable \| unknown}` |
+| `evidence` | — | what was read back to justify `effect` |
+| `escalation` | — | which rung to try next, and why this one fell short |
+
+**Refusal variant.** Accompanies `isError: true` and carries diagnostics instead of an effect. Two shapes are in service:
+
+```json
+{"status": "refused",
+ "refusal": {"code": "stale_element_token",
+ "message": "element_token is stale; call get_window_state again to refresh"}}
+```
+
+```json
+{"code": "window_target_not_found", "effect": "refused", "candidates": [], "pid": 4711}
+```
+
+This variant is deliberately open — refusals carry tool-specific diagnostic keys (`candidates`, `detail`, `pid`) that help a caller recover. The marker keys `refusal`, `status`, and `code` are what identify a payload as a refusal rather than a malformed success.
+
+<Callout type="warn">
+Branch on the refusal `code`, and surface the `content` text — it states the recovery step in plain language. `stale_element_token` means re-run `get_window_state` and retry with fresh indices; it does **not** mean the accessibility route is unavailable. Agents that treat a refusal as a dead route tend to abandon the element path and fall back to blind pixel clicking, which is both slower and unverifiable.
+</Callout>
+
+Success payloads stay strictly validated: an unknown key on the success shape is still a contract violation and will not be quietly accepted through the refusal variant.
+
+---
+
## Platform support
| Capability | Windows | macOS | Linux |
diff --git a/libs/cua-driver/rust/Cargo.lock b/libs/cua-driver/rust/Cargo.lock
index 449a4a2ec8..1b91afcff8 100644
--- a/libs/cua-driver/rust/Cargo.lock
+++ b/libs/cua-driver/rust/Cargo.lock
@@ -1043,6 +1043,7 @@ dependencies = [
"cua-driver-contract",
"futures-util",
"image",
+ "jsonschema",
"libc",
"regex",
"regorus",
diff --git a/libs/cua-driver/rust/crates/cua-driver-contract/src/lib.rs b/libs/cua-driver/rust/crates/cua-driver-contract/src/lib.rs
index 2e723bb95f..60558739aa 100644
--- a/libs/cua-driver/rust/crates/cua-driver-contract/src/lib.rs
+++ b/libs/cua-driver/rust/crates/cua-driver-contract/src/lib.rs
@@ -35,13 +35,14 @@ pub use inputs::{
SetWindowFrameInput, StartSessionInput, ToolInput, TypeTextInput,
};
pub use outputs::{
- ActionDelivery, ActionDeliveryMode, ActionEffect, ActionEscalation, ActionEscalationReason,
- ActionEscalationTarget, ActionEvidence, ActionEvidenceKind, ActionResult,
- ActionResultValidationError, ActionRoute, ClipboardReadOutput, ClipboardWriteOutput,
- CursorMotionOutput, CursorPointOutput, CursorPositionOutput, CursorThemeOutput,
- CursorVisualOutput, DesktopStateOutput, EffectiveScope, EndSessionOutput,
- GetAgentCursorStateOutput, ScreenSizeOutput, SessionStateOutput, SetAgentCursorEnabledOutput,
- SetAgentCursorMotionOutput, SetAgentCursorThemeOutput, StartSessionOutput, ToolOutput,
+ advertised_output_schema, refusal_envelope_schema, ActionDelivery, ActionDeliveryMode,
+ ActionEffect, ActionEscalation, ActionEscalationReason, ActionEscalationTarget, ActionEvidence,
+ ActionEvidenceKind, ActionResult, ActionResultValidationError, ActionRoute,
+ ClipboardReadOutput, ClipboardWriteOutput, CursorMotionOutput, CursorPointOutput,
+ CursorPositionOutput, CursorThemeOutput, CursorVisualOutput, DesktopStateOutput,
+ EffectiveScope, EndSessionOutput, GetAgentCursorStateOutput, ScreenSizeOutput,
+ SessionStateOutput, SetAgentCursorEnabledOutput, SetAgentCursorMotionOutput,
+ SetAgentCursorThemeOutput, StartSessionOutput, ToolOutput,
};
pub use verification::{
BoundsExpectation, ElementPredicate, ElementSelector, PredicateOutcome, StatePredicate,
diff --git a/libs/cua-driver/rust/crates/cua-driver-contract/src/outputs.rs b/libs/cua-driver/rust/crates/cua-driver-contract/src/outputs.rs
index eaea4cea46..34430f292d 100644
--- a/libs/cua-driver/rust/crates/cua-driver-contract/src/outputs.rs
+++ b/libs/cua-driver/rust/crates/cua-driver-contract/src/outputs.rs
@@ -18,6 +18,43 @@ pub trait ToolOutput: Serialize + DeserializeOwned + JsonSchema {
}
}
+/// Schema for the refusal payload a tool emits alongside `isError: true`.
+///
+/// Refusals answer with a diagnostic shape rather than the success shape, and
+/// two are in service: `{"status":"refused","refusal":{code,message,detail}}`
+/// on the element-token and daemon paths, and `{"code":…,"effect":"refused",…}`
+/// on the window-target paths. Both carry tool-specific diagnostic keys, so the
+/// envelope stays open — the marker keys are what make it recognisably a
+/// refusal rather than a malformed success.
+pub fn refusal_envelope_schema() -> Value {
+ serde_json::json!({
+ "type": "object",
+ "additionalProperties": true,
+ "anyOf": [
+ {"required": ["refusal"]},
+ {"required": ["status"]},
+ {"required": ["code"]},
+ ],
+ })
+}
+
+/// Wrap a success schema into the shape advertised as the MCP `outputSchema`.
+///
+/// MCP requires every `structuredContent` a tool emits to validate against its
+/// advertised `outputSchema` — including payloads that accompany
+/// `isError: true`. Advertising the success shape alone made strict clients
+/// reject refusals outright, replacing the actionable message the driver had
+/// already placed in `content` ("element_token is stale; call get_window_state
+/// again to refresh") with an opaque schema-validation error. Agents then had
+/// no signal to re-snapshot and fell back to blind pixel clicking.
+///
+/// The success variant is kept exactly as generated — still closed, so success
+/// payloads stay strictly checked — and the refusal envelope joins it as a
+/// sibling variant.
+pub fn advertised_output_schema(success: Value) -> Value {
+ serde_json::json!({ "anyOf": [success, refusal_envelope_schema()] })
+}
+
fn output_schema_with_additional_properties<T: JsonSchema>(additional_properties: bool) -> Value {
let mut settings = SchemaSettings::draft2020_12();
settings.inline_subschemas = true;
diff --git a/libs/cua-driver/rust/crates/cua-driver-core/Cargo.toml b/libs/cua-driver/rust/crates/cua-driver-core/Cargo.toml
index 4fddd8f175..508d1c251d 100644
--- a/libs/cua-driver/rust/crates/cua-driver-core/Cargo.toml
+++ b/libs/cua-driver/rust/crates/cua-driver-core/Cargo.toml
@@ -44,6 +44,13 @@ libc = "0.2"
# a duplicate crate version.
[dev-dependencies]
tempfile = "3"
+# Test-only. Asserts the advertised `outputSchema` the way a strict MCP client
+# does — by validating real payloads against it. The runtime itself validates
+# success output by deserialising into the Rust type, so this stays out of the
+# shipped dependency graph.
+# Version matches the one regorus already pulls into this crate's graph, so
+# the test dependency reuses that build instead of adding a second copy.
+jsonschema = { version = "0.46", default-features = false }
[target.'cfg(windows)'.dependencies]
windows = { version = "0.58", features = ["Win32_Foundation", "Win32_UI_WindowsAndMessaging"] }
diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/tool.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/tool.rs
index ceab188de1..ed0fc5b3a4 100644
--- a/libs/cua-driver/rust/crates/cua-driver-core/src/tool.rs
+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/tool.rs
@@ -187,10 +187,17 @@ impl ToolDef {
cua_driver_contract::tool_success_output_schema(&self.name)
};
if let Some(output_schema) = output_schema {
+ // Advertise the refusal envelope alongside the success shape. MCP
+ // holds every `structuredContent` we emit — refusals included — to
+ // the advertised schema, and a success-only schema made strict
+ // clients discard our refusal message in favour of a schema error.
entry
.as_object_mut()
.expect("tool list entry is an object")
- .insert("outputSchema".into(), output_schema);
+ .insert(
+ "outputSchema".into(),
+ cua_driver_contract::advertised_output_schema(output_schema),
+ );
}
entry
}
@@ -4603,27 +4610,81 @@ mod capability_tests {
assert!(entry["capabilities"].is_array());
}
+ fn action_tool_entry(name: &str) -> serde_json::Value {
+ ToolDef {
+ name: name.into(),
+ description: "Action.".into(),
+ input_schema: serde_json::json!({"type":"object","properties":{}}),
+ read_only: false,
+ destructive: false,
+ idempotent: false,
+ open_world: true,
+ }
+ .to_list_entry()
+ }
+
#[test]
fn action_tools_advertise_the_same_closed_output_schema() {
let expected =
<cua_driver_contract::ActionResult as cua_driver_contract::ToolOutput>::output_schema();
for name in ["click", "browser_click", "browser_pointer", "browser_type"] {
- let def = ToolDef {
- name: name.into(),
- description: "Action.".into(),
- input_schema: serde_json::json!({"type":"object","properties":{}}),
- read_only: false,
- destructive: false,
- idempotent: false,
- open_world: true,
- };
- let entry = def.to_list_entry();
- assert_eq!(entry["outputSchema"], expected, "{name}");
- assert_eq!(
- entry["outputSchema"]["additionalProperties"], false,
- "{name}"
+ let entry = action_tool_entry(name);
+ // The success variant is unchanged and still closed; it now sits
+ // beside the refusal envelope instead of standing alone.
+ let success = &entry["outputSchema"]["anyOf"][0];
+ assert_eq!(success, &expected, "{name}");
+ assert_eq!(success["additionalProperties"], false, "{name}");
+ }
+ }
+
+ /// Regression: refusals must validate against the advertised schema.
+ ///
+ /// Both payloads below are verbatim captures from a live `cua-driver mcp`
+ /// stdio session. Advertising only the success shape made strict MCP
+ /// clients reject them with `-32602`, which discarded the refusal message
+ /// the driver had put in `content` and left agents with no signal to
+ /// re-snapshot.
+ #[test]
+ fn advertised_output_schema_accepts_live_refusal_payloads() {
+ let schema = &action_tool_entry("click")["outputSchema"];
+ let compiled = jsonschema::validator_for(schema).expect("advertised schema compiles");
+
+ let stale_element_token = serde_json::json!({
+ "refusal": {
+ "code": "stale_element_token",
+ "message": "element_token is stale; call get_window_state again to refresh"
+ },
+ "status": "refused"
+ });
+ let window_target_not_found = serde_json::json!({
+ "candidates": [],
+ "code": "window_target_not_found",
+ "effect": "refused",
+ "pid": 999_999
+ });
+ let success = serde_json::json!({
+ "delivery": {"mode": "background"},
+ "effect": "unverifiable",
+ "route": "accessibility"
+ });
+
+ for payload in [&stale_element_token, &window_target_not_found, &success] {
+ assert!(
+ compiled.is_valid(payload),
+ "advertised schema rejected {payload}"
);
}
+
+ // The success variant stays strict: an unknown key is still a bug, and
+ // must not be laundered through the permissive refusal variant.
+ assert!(
+ !compiled.is_valid(&serde_json::json!({
+ "effect": "confirmed",
+ "route": "accessibility",
+ "bogus_key": true
+ })),
+ "a malformed success payload must not validate"
+ );
}
#[test]
@@ -4665,8 +4726,9 @@ mod capability_tests {
open_world: false,
}
.to_list_entry();
- let z_index =
- &entry["outputSchema"]["properties"]["windows"]["items"]["properties"]["z_index"];
+ // anyOf[0] is the success variant; anyOf[1] is the refusal envelope.
+ let z_index = &entry["outputSchema"]["anyOf"][0]["properties"]["windows"]["items"]
+ ["properties"]["z_index"];
assert_eq!(z_index["type"], serde_json::json!(["integer", "null"]));
assert!(z_index["description"]
.as_str()
diff --git a/libs/cua-driver/rust/crates/cua-driver/tests/schema_consistency_test.rs b/libs/cua-driver/rust/crates/cua-driver/tests/schema_consistency_test.rs
index 41ea9e7f10..f971947a8e 100644
--- a/libs/cua-driver/rust/crates/cua-driver/tests/schema_consistency_test.rs
+++ b/libs/cua-driver/rust/crates/cua-driver/tests/schema_consistency_test.rs
@@ -84,12 +84,17 @@ fn registered_tool_contracts_match_on_active_backend() {
}
if is_action_result_tool(name) {
- let expected = ActionResult::output_schema();
+ // The live surface advertises the success shape beside the refusal
+ // envelope, because MCP holds every `structuredContent` — refusals
+ // included — to the advertised schema. The success variant must
+ // still be the shared ActionResult schema, byte for byte.
+ let expected =
+ cua_driver_contract::advertised_output_schema(ActionResult::output_schema());
let actual = tool.get("outputSchema");
if actual != Some(&expected) {
violations.push(format!(
- "{name}: live outputSchema does not equal the shared ActionResult schema; \
- actual={} expected={expected}",
+ "{name}: live outputSchema does not equal the advertised ActionResult schema \
+ (success variant + refusal envelope); actual={} expected={expected}",
actual.unwrap_or(&Value::Null)
));
}