Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

perf(cua-driver): skip verifier AX walk for screenshot evidence (#4164)

Kevin Rajan78c42a38ab27
diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/expectation.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/expectation.rs
index d107e328fe..063b6287b1 100644
--- a/libs/cua-driver/rust/crates/cua-driver-core/src/expectation.rs
+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/expectation.rs
@@ -135,17 +135,12 @@ impl ObservationProvider for ToolObservationProvider {
let state = self
.get_window_state
- .invoke(json!({
- "pid": pid,
- "window_id": window_id,
- "include_screenshot": include_screenshot,
- // Internal direct-tool flag: verification must not refresh the
- // shared action index/token cache.
- // Registry ingress removes underscore-prefixed arguments before
- // public dispatch. Direct platform-tool invocation is the
- // trusted in-process channel for this non-mutating mode.
- "_observation_only": true,
- }))
+ .invoke(window_state_observation_args(
+ pid,
+ window_id,
+ include_elements,
+ include_screenshot,
+ ))
.await;
if state.is_error == Some(true) {
return Err(tool_error_text(&state, "get_window_state failed"));
@@ -181,6 +176,26 @@ impl ObservationProvider for ToolObservationProvider {
}
}
+fn window_state_observation_args(
+ pid: i64,
+ window_id: u64,
+ include_elements: bool,
+ include_screenshot: bool,
+) -> Value {
+ json!({
+ "pid": pid,
+ "window_id": window_id,
+ "include_accessibility_tree": include_elements,
+ "include_screenshot": include_screenshot,
+ // Internal direct-tool flag: verification must not refresh the
+ // shared action index/token cache.
+ // Registry ingress removes underscore-prefixed arguments before
+ // public dispatch. Direct platform-tool invocation is the
+ // trusted in-process channel for this non-mutating mode.
+ "_observation_only": true,
+ })
+}
+
fn tool_error_text(result: &ToolResult, fallback: &str) -> String {
result
.content
@@ -801,6 +816,18 @@ mod tests {
}
}
+ #[test]
+ fn observation_args_request_only_needed_modalities() {
+ let screenshot_only = window_state_observation_args(42, 7, false, true);
+ assert_eq!(screenshot_only["include_accessibility_tree"], json!(false));
+ assert_eq!(screenshot_only["include_screenshot"], json!(true));
+ assert_eq!(screenshot_only["_observation_only"], json!(true));
+
+ let elements_only = window_state_observation_args(42, 7, true, false);
+ assert_eq!(elements_only["include_accessibility_tree"], json!(true));
+ assert_eq!(elements_only["include_screenshot"], json!(false));
+ }
+
#[test]
fn evaluates_window_bounds_with_tolerance() {
let predicates = vec![StatePredicate {