Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

fix(cua-driver): constrain element_token tool schemas (#4318)

Kevin Rajandb5573914647
diff --git a/libs/cua-driver/contract/manifest.json b/libs/cua-driver/contract/manifest.json
index 1d73ecacd8..8a8486b1e6 100644
--- a/libs/cua-driver/contract/manifest.json
+++ b/libs/cua-driver/contract/manifest.json
@@ -100,6 +100,7 @@
},
"element_token": {
"default": null,
+ "pattern": "^s[0-9a-f]{8}:[0-9]+$",
"type": "string"
},
"session": {
diff --git a/libs/cua-driver/rust/crates/cua-driver-contract/src/compatibility.rs b/libs/cua-driver/rust/crates/cua-driver-contract/src/compatibility.rs
index e1f89013d4..e1f536535d 100644
--- a/libs/cua-driver/rust/crates/cua-driver-contract/src/compatibility.rs
+++ b/libs/cua-driver/rust/crates/cua-driver-contract/src/compatibility.rs
@@ -17,6 +17,7 @@ const SUPPORTED_KEYWORDS: &[&str] = &[
"type",
"const",
"enum",
+ "pattern",
"minimum",
"maximum",
"minLength",
@@ -76,6 +77,7 @@ fn compare_schema(path: &str, portable: &Value, live: &Value, violations: &mut V
compare_type(path, portable, live, violations);
compare_allowed_values(path, portable, live, violations);
+ compare_pattern(path, portable, live, violations);
compare_lower_bound(path, "minimum", portable, live, violations);
compare_upper_bound(path, "maximum", portable, live, violations);
compare_lower_bound(path, "minLength", portable, live, violations);
@@ -192,6 +194,35 @@ fn compare_allowed_values(
}
}
+fn compare_pattern(path: &str, portable: &Value, live: &Value, violations: &mut Vec<String>) {
+ let Some(live_pattern) = live.get("pattern") else {
+ // A portable-only pattern narrows the client domain and is therefore
+ // still a subset of an unconstrained live string.
+ return;
+ };
+ let Some(live_pattern) = live_pattern.as_str() else {
+ violations.push(format!("{path}.pattern: live pattern must be a string"));
+ return;
+ };
+ let Some(portable_pattern) = portable.get("pattern") else {
+ violations.push(format!(
+ "{path}: live defines pattern {live_pattern:?} but portable does not"
+ ));
+ return;
+ };
+ let Some(portable_pattern) = portable_pattern.as_str() else {
+ violations.push(format!("{path}.pattern: portable pattern must be a string"));
+ return;
+ };
+ // This intentionally small implication checker does not attempt regex
+ // language inclusion. Equality is the only pattern relation we claim.
+ if portable_pattern != live_pattern {
+ violations.push(format!(
+ "{path}: portable pattern {portable_pattern:?} is not proven to be a subset of live pattern {live_pattern:?}"
+ ));
+ }
+}
+
fn numeric_keyword<'a>(schema: &'a Value, keyword: &str) -> Option<&'a Value> {
schema.get(keyword)
}
@@ -397,13 +428,39 @@ mod tests {
assert!(violations.iter().any(|value| value.contains("minimum")));
}
+ #[test]
+ fn pattern_constraints_are_compared_conservatively() {
+ let portable = json!({ "type": "string", "pattern": "^s[0-9a-f]{8}:[0-9]+$" });
+ let same =
+ json!({ "type": "string", "pattern": "^s[0-9a-f]{8}:[0-9]+$", "description": "live" });
+ assert!(schema_subset_violations(&portable, &same).is_empty());
+
+ let live_only = json!({ "type": "string", "pattern": "^s[0-9a-f]{8}:[0-9]+$" });
+ let no_portable_pattern = json!({ "type": "string" });
+ assert!(schema_subset_violations(&no_portable_pattern, &live_only)
+ .iter()
+ .any(|value| value.contains("live defines pattern")));
+
+ let different = json!({ "type": "string", "pattern": "^s[0-9A-F]{8}:[0-9]+$" });
+ assert!(schema_subset_violations(&different, &live_only)
+ .iter()
+ .any(|value| value.contains("not proven to be a subset")));
+ }
+
+ #[test]
+ fn portable_only_pattern_is_safe_narrowing() {
+ let portable = json!({ "type": "string", "pattern": "^s[0-9a-f]{8}:[0-9]+$" });
+ let unconstrained_live = json!({ "type": "string", "description": "free text" });
+ assert!(schema_subset_violations(&portable, &unconstrained_live).is_empty());
+ }
+
#[test]
fn unknown_validation_keyword_fails_closed() {
- let portable = json!({ "type": "string", "pattern": "^[a-z]+$" });
+ let portable = json!({ "type": "string", "format": "uuid" });
let live = json!({ "type": "string" });
let violations = schema_subset_violations(&portable, &live);
assert_eq!(violations.len(), 1);
- assert!(violations[0].contains("unsupported portable schema keyword `pattern`"));
+ assert!(violations[0].contains("unsupported portable schema keyword `format`"));
}
#[test]
diff --git a/libs/cua-driver/rust/crates/cua-driver-contract/src/inputs.rs b/libs/cua-driver/rust/crates/cua-driver-contract/src/inputs.rs
index 643fb67f97..66a5c9f779 100644
--- a/libs/cua-driver/rust/crates/cua-driver-contract/src/inputs.rs
+++ b/libs/cua-driver/rust/crates/cua-driver-contract/src/inputs.rs
@@ -64,6 +64,10 @@ fn nonempty_string_schema(_: &mut SchemaGenerator) -> Schema {
json_schema!({ "type": "string", "minLength": 1 })
}
+fn element_token_schema(_: &mut SchemaGenerator) -> Schema {
+ json_schema!({ "type": "string", "pattern": "^s[0-9a-f]{8}:[0-9]+$" })
+}
+
pub const MULTI_CALL_SESSION_DESCRIPTION: &str =
"For multi-call work, prefer a short public session label and repeat it on every call that \
accepts it. Omit it to use the authenticated transport's implicit lifecycle session.";
@@ -695,7 +699,7 @@ struct ClickWireInput {
#[schemars(schema_with = "number_schema")]
y: Option<f64>,
#[serde(default, deserialize_with = "present_click_field")]
- #[schemars(schema_with = "string_schema")]
+ #[schemars(schema_with = "element_token_schema")]
element_token: Option<String>,
#[serde(default, deserialize_with = "present_click_field")]
#[schemars(schema_with = "nonempty_string_schema")]
@@ -1035,7 +1039,7 @@ mod tests {
for position in [
json!({"x":-1.5,"y":2.0}),
json!({"x":-1.5,"y":2.0,"capture_id":"capture-1"}),
- json!({"element_token":"s1:0"}),
+ json!({"element_token":"s00000001:0"}),
] {
let mut wire = json!({"target":{"kind":"window","pid":7,"window_id":9007199254740993_u64},"delivery_mode":"background"});
wire.as_object_mut()
@@ -1050,6 +1054,10 @@ mod tests {
assert!(schema["properties"].get("position").is_none());
assert!(schema["properties"].get("capture_id").is_some());
assert_eq!(schema["properties"]["capture_id"]["minLength"], 1);
+ assert_eq!(
+ schema["properties"]["element_token"]["pattern"],
+ "^s[0-9a-f]{8}:[0-9]+$"
+ );
}
#[test]
@@ -1058,11 +1066,11 @@ mod tests {
json!({}),
json!({"x":1}),
json!({"y":2}),
- json!({"x":1,"y":2,"element_token":"s1:0"}),
- json!({"element_token":"s1:0","capture_id":"capture-1"}),
+ json!({"x":1,"y":2,"element_token":"s00000001:0"}),
+ json!({"element_token":"s00000001:0","capture_id":"capture-1"}),
json!({"x":1,"y":2,"capture_id":" "}),
- json!({"x":1,"element_token":"s1:0"}),
- json!({"x":null,"element_token":"s1:0"}),
+ json!({"x":1,"element_token":"s00000001:0"}),
+ json!({"x":null,"element_token":"s00000001:0"}),
json!({"element_token":" "}),
json!({"x":1,"y":2,"unknown":true}),
] {
@@ -1093,7 +1101,7 @@ mod tests {
};
assert!(input.validate().is_err());
input.position = ClickPosition::Element {
- element_token: "s1:0".into(),
+ element_token: "s00000001:0".into(),
};
assert!(input.validate().is_err());
input.position = ClickPosition::Coordinates { x: 1.0, y: 2.0 };
diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/tool_schema.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/tool_schema.rs
index ea77fc3b35..b43a2b5aba 100644
--- a/libs/cua-driver/rust/crates/cua-driver-core/src/tool_schema.rs
+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/tool_schema.rs
@@ -95,6 +95,7 @@ pub fn scope_schema() -> Value {
pub fn element_token_schema() -> Value {
json!({
"type": "string",
+ "pattern": "^s[0-9a-f]{8}:[0-9]+$",
"description": "Opaque per-snapshot element handle from \
`structuredContent.elements[].element_token`. Returns an explicit \
stale error naming the current snapshots once a newer read \
@@ -192,8 +193,8 @@ fn required_canonical(tool: &str) -> Option<&'static [&'static str]> {
}
/// Reduce a param schema to the parts that govern client compatibility —
-/// `type`, `enum`, and (recursively) `items` — dropping `description` and any
-/// other prose so per-tool wording differences don't trip the gate.
+/// `type`, `enum`, `pattern`, and (recursively) `items` — dropping
+/// `description` and other prose so per-tool wording differences don't trip the gate.
fn structural(schema: &Value) -> Value {
let mut out = serde_json::Map::new();
if let Some(t) = schema.get("type") {
@@ -202,6 +203,9 @@ fn structural(schema: &Value) -> Value {
if let Some(e) = schema.get("enum") {
out.insert("enum".into(), e.clone());
}
+ if let Some(pattern) = schema.get("pattern") {
+ out.insert("pattern".into(), pattern.clone());
+ }
if let Some(items) = schema.get("items") {
out.insert("items".into(), structural(items));
}
@@ -384,4 +388,49 @@ mod tests {
let tool = json!({ "type": "object", "required": [], "properties": {} });
assert!(shared_schema_violations("click", &tool).is_empty());
}
+ #[test]
+ fn element_token_pattern_matches_the_minted_wire_shape() {
+ let schema = element_token_schema();
+ assert_eq!(schema["pattern"], "^s[0-9a-f]{8}:[0-9]+$");
+ let validator = jsonschema::validator_for(&schema).expect("element token schema compiles");
+ for (snapshot_id, element_index) in
+ [(0_u32, 0_usize), (1_u32, 42_usize), (u32::MAX, usize::MAX)]
+ {
+ let token = crate::element_token::token_for(snapshot_id, element_index);
+ assert!(
+ validator.is_valid(&json!(token)),
+ "advertised element_token schema rejected minted token {token}"
+ );
+ }
+ }
+
+ #[test]
+ fn missing_shared_element_token_pattern_is_flagged() {
+ let tool = json!({
+ "type": "object",
+ "properties": {
+ "element_token": { "type": "string" }
+ }
+ });
+ let violations = shared_schema_violations("click", &tool);
+ assert!(
+ violations.iter().any(|item| item.contains("element_token")),
+ "missing element_token pattern must be flagged: {violations:?}"
+ );
+ }
+
+ #[test]
+ fn element_token_pattern_drift_is_flagged() {
+ let tool = json!({
+ "type": "object",
+ "properties": {
+ "element_token": { "type": "string", "pattern": "^wrong$" }
+ }
+ });
+ let violations = shared_schema_violations("click", &tool);
+ assert!(
+ violations.iter().any(|item| item.contains("element_token")),
+ "element_token pattern drift must be flagged: {violations:?}"
+ );
+ }
}
diff --git a/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs b/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs
index 34a99e0e85..7f07305d40 100644
--- a/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs
+++ b/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs
@@ -4476,7 +4476,7 @@ impl Tool for TypeTextTool {
"pid":{"type":"integer","description":"Target process ID."},
"text":{"type":"string","description":"Text to insert at the focused element's cursor."},
"window_id":{"type":"integer","description":"HWND of the target window. Omit when element_token is supplied (the token carries it)."},
- "element_token":{"type":"string","description":"Opaque element handle from get_window_state. When supplied, type_text writes through UIA ValuePattern and reads that exact element back by handle. The shared ActionResult is confirmed only when the complete expected value is synchronously visible. If SetValue succeeds but read-back is stale or unavailable, the result is unverifiable with no escalation; take a fresh snapshot before retrying because deferred providers may publish only after this call returns."},
+ "element_token":{"type":"string","pattern":"^s[0-9a-f]{8}:[0-9]+$","description":"Opaque element handle from get_window_state. When supplied, type_text writes through UIA ValuePattern and reads that exact element back by handle. The shared ActionResult is confirmed only when the complete expected value is synchronously visible. If SetValue succeeds but read-back is stale or unavailable, the result is unverifiable with no escalation; take a fresh snapshot before retrying because deferred providers may publish only after this call returns."},
"x":{"type":"number","description":"Window-local screenshot-pixel X of the field to type into — the element px action form. Pass x,y (no element_token) and the tool pixel-clicks there to establish real renderer focus, then types. Use for Chromium/Electron inputs the UIA/WM_CHAR path can't reach. Read straight off the get_window_state PNG, same convention as click."},
"y":{"type":"number","description":"Window-local screenshot-pixel Y of the field (see x)."},
"delay_ms":{"type":"integer","minimum":0,"maximum":200,"description":"Milliseconds between characters. Default 30."},
diff --git a/scripts/docs-generators/cli-specs/cua-driver-mcp-linux.json b/scripts/docs-generators/cli-specs/cua-driver-mcp-linux.json
index 08194e30c9..d9b74f5575 100644
--- a/scripts/docs-generators/cli-specs/cua-driver-mcp-linux.json
+++ b/scripts/docs-generators/cli-specs/cua-driver-mcp-linux.json
@@ -522,6 +522,7 @@
},
"element_token": {
"description": "Opaque per-snapshot element handle from `structuredContent.elements[].element_token`. Returns an explicit stale error naming the current snapshots once a newer read supersedes it.",
+ "pattern": "^s[0-9a-f]{8}:[0-9]+$",
"type": "string"
},
"from_zoom": {
@@ -646,6 +647,7 @@
},
"element_token": {
"description": "Opaque per-snapshot element handle from `structuredContent.elements[].element_token`. Returns an explicit stale error naming the current snapshots once a newer read supersedes it.",
+ "pattern": "^s[0-9a-f]{8}:[0-9]+$",
"type": "string"
},
"from_zoom": {
@@ -711,6 +713,7 @@
},
"element_token": {
"description": "Opaque per-snapshot element handle from `structuredContent.elements[].element_token`. Returns an explicit stale error naming the current snapshots once a newer read supersedes it.",
+ "pattern": "^s[0-9a-f]{8}:[0-9]+$",
"type": "string"
},
"from_zoom": {
@@ -1219,6 +1222,7 @@
},
"element_token": {
"description": "Opaque per-snapshot element handle from `structuredContent.elements[].element_token`. Returns an explicit stale error naming the current snapshots once a newer read supersedes it.",
+ "pattern": "^s[0-9a-f]{8}:[0-9]+$",
"type": "string"
},
"pid": {
@@ -1342,6 +1346,7 @@
},
"element_token": {
"description": "Opaque per-snapshot element handle from `structuredContent.elements[].element_token`. Returns an explicit stale error naming the current snapshots once a newer read supersedes it.",
+ "pattern": "^s[0-9a-f]{8}:[0-9]+$",
"type": "string"
},
"key": {
@@ -1464,6 +1469,7 @@
},
"element_token": {
"description": "Opaque per-snapshot element handle from `structuredContent.elements[].element_token`. Returns an explicit stale error naming the current snapshots once a newer read supersedes it.",
+ "pattern": "^s[0-9a-f]{8}:[0-9]+$",
"type": "string"
},
"keys": {
@@ -1583,6 +1589,7 @@
},
"element_token": {
"description": "Opaque per-snapshot element handle from `structuredContent.elements[].element_token`. Returns an explicit stale error naming the current snapshots once a newer read supersedes it.",
+ "pattern": "^s[0-9a-f]{8}:[0-9]+$",
"type": "string"
},
"pid": {
@@ -1668,6 +1675,7 @@
},
"element_token": {
"description": "Opaque per-snapshot element handle from `structuredContent.elements[].element_token`. Returns an explicit stale error naming the current snapshots once a newer read supersedes it.",
+ "pattern": "^s[0-9a-f]{8}:[0-9]+$",
"type": "string"
},
"pid": {
diff --git a/scripts/docs-generators/cli-specs/cua-driver-mcp-macos.json b/scripts/docs-generators/cli-specs/cua-driver-mcp-macos.json
index 06e61fd65c..a34de45638 100644
--- a/scripts/docs-generators/cli-specs/cua-driver-mcp-macos.json
+++ b/scripts/docs-generators/cli-specs/cua-driver-mcp-macos.json
@@ -521,6 +521,7 @@
},
"element_token": {
"description": "Opaque per-snapshot element handle from `structuredContent.elements[].element_token`. Returns an explicit stale error naming the current snapshots once a newer read supersedes it.",
+ "pattern": "^s[0-9a-f]{8}:[0-9]+$",
"type": "string"
},
"from_zoom": {
@@ -631,6 +632,7 @@
},
"element_token": {
"description": "Opaque per-snapshot element handle from `structuredContent.elements[].element_token`. Returns an explicit stale error naming the current snapshots once a newer read supersedes it.",
+ "pattern": "^s[0-9a-f]{8}:[0-9]+$",
"type": "string"
},
"pid": {
@@ -679,6 +681,7 @@
},
"element_token": {
"description": "Opaque per-snapshot element handle from `structuredContent.elements[].element_token`. Returns an explicit stale error naming the current snapshots once a newer read supersedes it.",
+ "pattern": "^s[0-9a-f]{8}:[0-9]+$",
"type": "string"
},
"modifier": {
@@ -889,6 +892,7 @@
},
"element_token": {
"description": "Opaque per-snapshot element handle from `structuredContent.elements[].element_token`. Returns an explicit stale error naming the current snapshots once a newer read supersedes it.",
+ "pattern": "^s[0-9a-f]{8}:[0-9]+$",
"type": "string"
},
"pid": {
@@ -1003,6 +1007,7 @@
},
"element_token": {
"description": "Opaque per-snapshot element handle from `structuredContent.elements[].element_token`. Returns an explicit stale error naming the current snapshots once a newer read supersedes it.",
+ "pattern": "^s[0-9a-f]{8}:[0-9]+$",
"type": "string"
},
"key": {
@@ -1124,6 +1129,7 @@
},
"element_token": {
"description": "Opaque per-snapshot element handle from `structuredContent.elements[].element_token`. Returns an explicit stale error naming the current snapshots once a newer read supersedes it.",
+ "pattern": "^s[0-9a-f]{8}:[0-9]+$",
"type": "string"
},
"keys": {
@@ -1234,6 +1240,7 @@
"properties": {
"element_token": {
"description": "Opaque per-snapshot element handle from `structuredContent.elements[].element_token`. Returns an explicit stale error naming the current snapshots once a newer read supersedes it.",
+ "pattern": "^s[0-9a-f]{8}:[0-9]+$",
"type": "string"
},
"pid": {
@@ -1303,6 +1310,7 @@
},
"element_token": {
"description": "Opaque per-snapshot element handle from `structuredContent.elements[].element_token`. Returns an explicit stale error naming the current snapshots once a newer read supersedes it.",
+ "pattern": "^s[0-9a-f]{8}:[0-9]+$",
"type": "string"
},
"pid": {
diff --git a/scripts/docs-generators/cli-specs/cua-driver-mcp-windows.json b/scripts/docs-generators/cli-specs/cua-driver-mcp-windows.json
index e6f28d5376..8b0f09d484 100644
--- a/scripts/docs-generators/cli-specs/cua-driver-mcp-windows.json
+++ b/scripts/docs-generators/cli-specs/cua-driver-mcp-windows.json
@@ -560,6 +560,7 @@
},
"element_token": {
"description": "Opaque per-snapshot element handle from `structuredContent.elements[].element_token`. Returns an explicit stale error naming the current snapshots once a newer read supersedes it.",
+ "pattern": "^s[0-9a-f]{8}:[0-9]+$",
"type": "string"
},
"from_zoom": {
@@ -670,6 +671,7 @@
},
"element_token": {
"description": "Opaque per-snapshot element handle from `structuredContent.elements[].element_token`. Returns an explicit stale error naming the current snapshots once a newer read supersedes it.",
+ "pattern": "^s[0-9a-f]{8}:[0-9]+$",
"type": "string"
},
"from_zoom": {
@@ -729,6 +731,7 @@
},
"element_token": {
"description": "Opaque per-snapshot element handle from `structuredContent.elements[].element_token`. Returns an explicit stale error naming the current snapshots once a newer read supersedes it.",
+ "pattern": "^s[0-9a-f]{8}:[0-9]+$",
"type": "string"
},
"from_zoom": {
@@ -942,6 +945,7 @@
},
"element_token": {
"description": "Opaque element handle from get_window_state. When supplied, type_text writes through UIA ValuePattern and reads that exact element back by handle. The shared ActionResult is confirmed only when the complete expected value is synchronously visible. If SetValue succeeds but read-back is stale or unavailable, the result is unverifiable with no escalation; take a fresh snapshot before retrying because deferred providers may publish only after this call returns.",
+ "pattern": "^s[0-9a-f]{8}:[0-9]+$",
"type": "string"
},
"pid": {
@@ -1055,6 +1059,7 @@
},
"element_token": {
"description": "Opaque per-snapshot element handle from `structuredContent.elements[].element_token`. Returns an explicit stale error naming the current snapshots once a newer read supersedes it.",
+ "pattern": "^s[0-9a-f]{8}:[0-9]+$",
"type": "string"
},
"key": {
@@ -1175,6 +1180,7 @@
},
"element_token": {
"description": "Opaque per-snapshot element handle from `structuredContent.elements[].element_token`. Returns an explicit stale error naming the current snapshots once a newer read supersedes it.",
+ "pattern": "^s[0-9a-f]{8}:[0-9]+$",
"type": "string"
},
"keys": {
@@ -1292,6 +1298,7 @@
},
"element_token": {
"description": "Opaque per-snapshot element handle from `structuredContent.elements[].element_token`. Returns an explicit stale error naming the current snapshots once a newer read supersedes it.",
+ "pattern": "^s[0-9a-f]{8}:[0-9]+$",
"type": "string"
},
"pid": {
@@ -1361,6 +1368,7 @@
},
"element_token": {
"description": "Opaque per-snapshot element handle from `structuredContent.elements[].element_token`. Returns an explicit stale error naming the current snapshots once a newer read supersedes it.",
+ "pattern": "^s[0-9a-f]{8}:[0-9]+$",
"type": "string"
},
"pid": {