Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
fix(cua-driver): deliver trusted background browser clicks
trycua-release[bot]00f33c4cfaff
diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/browser/tools.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/browser/tools.rsindex 7d71fb8127..89bf52a122 100644--- a/libs/cua-driver/rust/crates/cua-driver-core/src/browser/tools.rs+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/browser/tools.rs@@ -910,8 +910,27 @@ impl Tool for BrowserClickTool {(None, None) => unreachable!("validated above"),};+ if let Err(error) = conn+ .call(+ Some(cdp),+ "Emulation.setFocusEmulationEnabled",+ json!({ "enabled": true }),+ )+ .await+ {+ return BrowserRefusal::new(+ BrowserRefusalCode::BrowserInputTrustUnavailable,+ format!(+ "the target tab could not enter CDP focus emulation for trusted input: {error}"+ ),+ )+ .to_tool_result();+ }+ tokio::time::sleep(std::time::Duration::from_millis(25)).await;++ let mut delivery_error = None;for (event_type, click_count) in [("mousePressed", 1), ("mouseReleased", 1)] {- if let Err(e) = conn+ if let Err(error) = conn.call(Some(cdp),"Input.dispatchMouseEvent",@@ -925,18 +944,40 @@ impl Tool for BrowserClickTool {).await{- // Trusted input is the contract; we never silently fall- // back to synthetic events.- return BrowserRefusal::new(- BrowserRefusalCode::BrowserInputTrustUnavailable,- format!(- "trusted Input route failed ({e}) — re-run with \- input_route=\"dom_event\" to explicitly request a synthetic click"- ),- )- .to_tool_result();+ delivery_error = Some(error);+ break;}}+ let cleanup_error = conn+ .call(+ Some(cdp),+ "Emulation.setFocusEmulationEnabled",+ json!({ "enabled": false }),+ )+ .await+ .err();+ if let Some(error) = delivery_error {+ // Trusted input is the contract; we never silently fall back to+ // synthetic events. Focus emulation has already been unwound.+ return BrowserRefusal::new(+ BrowserRefusalCode::BrowserInputTrustUnavailable,+ format!(+ "trusted Input route failed ({error}) — re-run with \+ input_route=\"dom_event\" to explicitly request a synthetic click"+ ),+ )+ .to_tool_result();+ }+ if let Some(error) = cleanup_error {+ return BrowserRefusal::new(+ BrowserRefusalCode::BrowserInputTrustUnavailable,+ format!(+ "trusted click was acknowledged but CDP focus emulation could not be restored ({error}); delivery is unknown and must not be retried automatically"+ ),+ )+ .with_detail(json!({ "delivery": "unknown", "retryable": false }))+ .to_tool_result();+ }ToolResult::text(format!("clicked ({x:.0}, {y:.0}) in {tab_id}")).with_structured(json!({"status": "ok","route": "trusted",diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/browser/v2_tests.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/browser/v2_tests.rsindex d2a7d547de..7e5c24f128 100644--- a/libs/cua-driver/rust/crates/cua-driver-core/src/browser/v2_tests.rs+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/browser/v2_tests.rs@@ -1577,6 +1577,12 @@ async fn click_routes_oopif_refs_through_the_contained_child_session() {.all(|(sess, _)| sess.as_deref().unwrap_or("").starts_with("oopif-sess-")),"OOPIF input must dispatch on the child session: {mouse:?}");+ let focus_emulation = recorded_calls(&f, "Emulation.setFocusEmulationEnabled");+ assert_eq!(focus_emulation.len(), 2);+ assert_eq!(focus_emulation[0].1["enabled"], true);+ assert_eq!(focus_emulation[1].1["enabled"], false);+ assert!(recorded_calls(&f, "Page.bringToFront").is_empty());+ assert!(recorded_calls(&f, "Target.activateTarget").is_empty());}#[tokio::test]