Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
perf(cua-driver): skip verifier AX walk for screenshot evidence (#4164)
Kevin Rajan78c42a38ab27
diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/expectation.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/expectation.rsindex d107e328fe..063b6287b1 100644--- a/libs/cua-driver/rust/crates/cua-driver-core/src/expectation.rs+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/expectation.rs@@ -135,17 +135,12 @@ impl ObservationProvider for ToolObservationProvider {let state = self.get_window_state- .invoke(json!({- "pid": pid,- "window_id": window_id,- "include_screenshot": include_screenshot,- // Internal direct-tool flag: verification must not refresh the- // shared action index/token cache.- // Registry ingress removes underscore-prefixed arguments before- // public dispatch. Direct platform-tool invocation is the- // trusted in-process channel for this non-mutating mode.- "_observation_only": true,- }))+ .invoke(window_state_observation_args(+ pid,+ window_id,+ include_elements,+ include_screenshot,+ )).await;if state.is_error == Some(true) {return Err(tool_error_text(&state, "get_window_state failed"));@@ -181,6 +176,26 @@ impl ObservationProvider for ToolObservationProvider {}}+fn window_state_observation_args(+ pid: i64,+ window_id: u64,+ include_elements: bool,+ include_screenshot: bool,+) -> Value {+ json!({+ "pid": pid,+ "window_id": window_id,+ "include_accessibility_tree": include_elements,+ "include_screenshot": include_screenshot,+ // Internal direct-tool flag: verification must not refresh the+ // shared action index/token cache.+ // Registry ingress removes underscore-prefixed arguments before+ // public dispatch. Direct platform-tool invocation is the+ // trusted in-process channel for this non-mutating mode.+ "_observation_only": true,+ })+}+fn tool_error_text(result: &ToolResult, fallback: &str) -> String {result.content@@ -801,6 +816,18 @@ mod tests {}}+ #[test]+ fn observation_args_request_only_needed_modalities() {+ let screenshot_only = window_state_observation_args(42, 7, false, true);+ assert_eq!(screenshot_only["include_accessibility_tree"], json!(false));+ assert_eq!(screenshot_only["include_screenshot"], json!(true));+ assert_eq!(screenshot_only["_observation_only"], json!(true));++ let elements_only = window_state_observation_args(42, 7, true, false);+ assert_eq!(elements_only["include_accessibility_tree"], json!(true));+ assert_eq!(elements_only["include_screenshot"], json!(false));+ }+#[test]fn evaluates_window_bounds_with_tolerance() {let predicates = vec![StatePredicate {