Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
fix(cua-driver): refuse clicks fast while a detached session's dialog blocks the tab (#6646)
injaneitya550948860dd
diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/browser/cdp_ws.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/browser/cdp_ws.rsindex 0d2b22ee74..40235a40ea 100644--- a/libs/cua-driver/rust/crates/cua-driver-core/src/browser/cdp_ws.rs+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/browser/cdp_ws.rs@@ -122,6 +122,10 @@ pub struct CdpDialogState {pub generation: u64,pub kind: String,pub session_id: String,+ /// The session that saw the dialog open detached while it was showing.+ /// Chromium answers every other session `No dialog is showing`, so the+ /// driver cannot resolve it, yet the page stays blocked until it closes.+ pub detached: bool,}/// What the reader routed back for one in-flight call.@@ -219,6 +223,7 @@ async fn read_loop(mut read: SplitStream<WsStream>, demux: Arc<Demux>) {generation,kind: kind.to_owned(),session_id: session_id.to_owned(),+ detached: false,},);}@@ -231,17 +236,17 @@ async fn read_loop(mut read: SplitStream<WsStream>, demux: Arc<Demux>) {_ => {}}} else if event.method == "Target.detachedFromTarget" {- // A detached session reports nothing more, its dialog close- // included, and cannot resolve the dialog, so its record- // would lock the tab for good. Drop it; the next attach- // subscribes a new session.+ // A detached session reports nothing more and cannot resolve+ // its dialog, but the page stays blocked. Keep the record,+ // marked unresolvable, so callers fail fast; the next attach+ // subscribes a new session whose close event clears it.if let Some(session_id) = event.params.get("sessionId").and_then(Value::as_str) {demux.session_targets.lock().unwrap().remove(session_id);- demux- .dialogs- .lock()- .unwrap()- .retain(|_, dialog| dialog.session_id != session_id);+ for dialog in demux.dialogs.lock().unwrap().values_mut() {+ if dialog.session_id == session_id {+ dialog.detached = true;+ }+ }}}demux@@ -1377,7 +1382,7 @@ mod tests {}#[tokio::test]- async fn a_detached_dialog_session_drops_its_dialog_record() {+ async fn a_detached_dialog_session_marks_its_dialog_unresolvable() {let server = MockCdpServer::start(StdArc::new(|call| match call.method.as_str() {"Dialog.open" => MockReply::ok(json!({})).with_events(vec![MockEvent {method: "Page.javascriptDialogOpening".into(),@@ -1389,6 +1394,11 @@ mod tests {session_id: None,params: json!({ "sessionId": call.params["sessionId"], "targetId": "page-target" }),}]),+ "Dialog.close" => MockReply::ok(json!({})).with_events(vec![MockEvent {+ method: "Page.javascriptDialogClosed".into(),+ session_id: call.session_id.clone(),+ params: json!({}),+ }]),_ => MockReply::ok(json!({})),})).await;@@ -1405,8 +1415,16 @@ mod tests {).await.unwrap();- assert!(conn.dialog_state("page-target").is_none());+ // The page is still blocked, so the record stays, marked unresolvable.+ let dialog = conn.dialog_state("page-target").expect("dialog kept");+ assert!(dialog.detached);assert!(!conn.has_dialog_session("page-target"));+ // A new subscriber's close event clears it.+ conn.register_dialog_session("tab-session-b", "page-target");+ conn.call(Some("tab-session-b"), "Dialog.close", json!({}))+ .await+ .unwrap();+ assert!(conn.dialog_state("page-target").is_none());}#[tokio::test]diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/browser/engine.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/browser/engine.rsindex 776d17f7a7..40d2bc5664 100644--- a/libs/cua-driver/rust/crates/cua-driver-core/src/browser/engine.rs+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/browser/engine.rs@@ -1294,7 +1294,12 @@ impl BrowserEngine {cdp_session: &str,cdp_target_id: &str,) {- if conn.dialog_state(cdp_target_id).is_some() || conn.has_dialog_session(cdp_target_id) {+ // A dialog whose session detached still blocks the page; a new+ // subscriber is the only way its close can clear the record.+ let dialog = conn.dialog_state(cdp_target_id);+ if dialog.as_ref().is_some_and(|dialog| !dialog.detached)+ || conn.has_dialog_session(cdp_target_id)+ {return;}conn.register_dialog_session(cdp_session, cdp_target_id);@@ -1309,6 +1314,11 @@ impl BrowserEngine {if enabled.is_err() && conn.dialog_state(cdp_target_id).is_none() {conn.unregister_dialog_session(cdp_session, cdp_target_id);}+ // A blocked renderer cannot answer Page.enable, so a reply means the+ // detached dialog closed while no session was listening.+ if let (Ok(_), Some(stale)) = (&enabled, dialog) {+ conn.clear_dialog_state(cdp_target_id, stale.generation);+ }}// ── Binding (get_browser_state, pid + window_id mode) ──────────────diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/browser/tools.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/browser/tools.rsindex 571d0c639a..bdd8e47956 100644--- a/libs/cua-driver/rust/crates/cua-driver-core/src/browser/tools.rs+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/browser/tools.rs@@ -1308,10 +1308,15 @@ impl Tool for BrowserClickTool {.conn.dialog_state(validated.tab.cdp_target_id.as_str()){+ let remedy = if open.detached {+ "its CDP session was detached, so browser_dialog cannot resolve it; close it in the browser"+ } else {+ "resolve it with browser_dialog before clicking"+ };return BrowserRefusal::new(BrowserRefusalCode::BrowserActionUnavailable,format!(- "a {} dialog is already open on the exact tab; resolve it with browser_dialog before clicking",+ "a {} dialog is already open on the exact tab; {remedy}",open.kind),)@@ -1695,6 +1700,7 @@ fn dialog_block(dialog: &super::cdp_ws::CdpDialogState) -> Value {"present": true,"kind": dialog.kind,"dialog_id": dialog_capability_id(dialog.generation),+ "resolvable": !dialog.detached,})}@@ -2894,9 +2900,20 @@ impl Tool for BrowserDialogTool {return ToolResult::text(format!("{} dialog is open in {tab_id}", dialog.kind)).with_structured(json!({"status": "ok", "target_id": target_id, "tab_id": tab_id,- "present": true, "dialog_id": dialog_id, "kind": dialog.kind+ "present": true, "dialog_id": dialog_id, "kind": dialog.kind,+ "resolvable": !dialog.detached}));}+ if dialog.detached {+ // Chromium lets only the session that saw the dialog open resolve+ // it; every other session is told none is showing.+ return BrowserRefusal::new(+ BrowserRefusalCode::BrowserActionUnavailable,+ "the dialog's CDP session was detached, so the driver cannot resolve it; close it in the browser",+ )+ .with_detail(json!({ "dialog": dialog_block(&dialog) }))+ .to_tool_result();+ }let supplied_id = match args.require_str("dialog_id") {Ok(id) => id,diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/browser/v2_tests.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/browser/v2_tests.rsindex bb774e0870..7ea7147cd6 100644--- a/libs/cua-driver/rust/crates/cua-driver-core/src/browser/v2_tests.rs+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/browser/v2_tests.rs@@ -73,6 +73,11 @@ pub(super) struct FixtureState {/// or emulation command until it closes.trusted_click_opens_dialog: Option<&'static str>,renderer_blocked: bool,+ /// Something outside the driver detaches the dialog's Page session on+ /// the next attach. While `dialog_blocks_page`, the renderer cannot+ /// answer `Page.enable` on the new session.+ detach_dialog_session: bool,+ dialog_blocks_page: bool,/// Tab sessions that enabled the Page domain. Chromium reports a dialog/// to each of them, not to the session whose call opened it.page_sessions: Vec<String>,@@ -141,6 +146,8 @@ impl Default for FixtureState {dialog_vanished: false,trusted_click_opens_dialog: None,renderer_blocked: false,+ detach_dialog_session: false,+ dialog_blocks_page: false,page_sessions: Vec::new(),semantic_large_page: false,semantic_html_style: String::new(),@@ -538,9 +545,26 @@ fn fixture_handler(state: SharedState) -> MockHandler {})),"Target.attachToTarget" => {st.tab_sessions += 1;- MockReply::ok(json!({ "sessionId": format!("tab-sess-{}", st.tab_sessions) }))+ let reply =+ MockReply::ok(json!({ "sessionId": format!("tab-sess-{}", st.tab_sessions) }));+ if !std::mem::take(&mut st.detach_dialog_session) {+ return reply;+ }+ let detached = std::mem::take(&mut st.page_sessions);+ reply.with_events(+ detached+ .into_iter()+ .map(|session| MockEvent {+ method: "Target.detachedFromTarget".into(),+ params: json!({ "sessionId": session, "targetId": "T1" }),+ session_id: None,+ })+ .collect(),+ )+ }+ "Page.enable" if is_tab && (st.page_enable_hangs || st.dialog_blocks_page) => {+ MockReply::hang()}- "Page.enable" if is_tab && st.page_enable_hangs => MockReply::hang(),"Page.enable" if is_tab => {st.page_sessions.push(sess.clone());let mut reply = MockReply::ok(json!({}));@@ -4161,3 +4185,77 @@ async fn trusted_click_returns_when_a_dialog_opens_on(opener: &'static str) {.await;assert_eq!(structured(&accepted)["status"], "ok");}++/// Opens a confirm with a dom_event click, then has the next attach detach+/// the session that saw it open.+async fn click_then_detach_dialog_session(f: &Fixture, page_blocked: bool) -> (String, String) {+ let (target, tab) = bind(f).await;+ let snap = snapshot(f, &target, &tab).await;+ let button = ref_of(&snap, "main", "main-btn");+ let first = BrowserClickTool::new(f.engine.clone())+ .invoke(json!({+ "target_id": target, "tab_id": tab, "ref": button,+ "input_route": "dom_event", "session": SESSION+ }))+ .await;+ assert_eq!(structured(&first)["dialog"]["present"], true);+ let mut state = f.state.lock().unwrap();+ state.click_opens_dialog = false;+ state.detach_dialog_session = true;+ state.dialog_blocks_page = page_blocked;+ (target, tab)+}++#[tokio::test]+async fn a_dialog_whose_session_detached_refuses_clicks_without_waiting() {+ let f = fixture_with(|state| state.click_opens_dialog = true).await;+ let (target, tab) = click_then_detach_dialog_session(&f, true).await;++ let started = std::time::Instant::now();+ let clicked = BrowserClickTool::new(f.engine.clone())+ .invoke(json!({+ "target_id": target, "tab_id": tab, "session": SESSION, "x": 10, "y": 20+ }))+ .await;+ let c = structured(&clicked);+ assert_eq!(c["refusal"]["code"], "browser_action_unavailable", "{c}");+ assert_eq!(c["refusal"]["detail"]["dialog"]["present"], true, "{c}");+ assert_eq!(c["refusal"]["detail"]["dialog"]["resolvable"], false, "{c}");+ // Only the bounded dialog subscription waits; the 20 s call timeout never runs.+ assert!(started.elapsed() < std::time::Duration::from_secs(10));+ assert!(recorded_calls(&f, "Input.dispatchMouseEvent").is_empty());++ let inspected = BrowserDialogTool::new(f.engine.clone())+ .invoke(json!({+ "target_id": target, "tab_id": tab, "action": "inspect", "session": SESSION+ }))+ .await;+ let i = structured(&inspected);+ assert_eq!(i["present"], true, "{i}");+ assert_eq!(i["resolvable"], false, "{i}");+ let dismissed = BrowserDialogTool::new(f.engine.clone())+ .invoke(json!({+ "target_id": target, "tab_id": tab, "action": "dismiss",+ "dialog_id": i["dialog_id"], "session": SESSION, "delivery_mode": "foreground"+ }))+ .await;+ let d = structured(&dismissed);+ assert_eq!(d["refusal"]["code"], "browser_action_unavailable", "{d}");+ assert_eq!(d["refusal"]["detail"]["dialog"]["resolvable"], false, "{d}");+ assert!(recorded_calls(&f, "Page.handleJavaScriptDialog").is_empty());+}++#[tokio::test]+async fn a_detached_dialog_that_closed_unseen_stops_blocking_clicks() {+ let f = fixture_with(|state| state.click_opens_dialog = true).await;+ let (target, tab) = click_then_detach_dialog_session(&f, false).await;++ // The new session's Page.enable answers, so the renderer is no longer blocked.+ let clicked = BrowserClickTool::new(f.engine.clone())+ .invoke(json!({+ "target_id": target, "tab_id": tab, "session": SESSION, "x": 10, "y": 20+ }))+ .await;+ assert_ne!(clicked.is_error, Some(true), "{:?}", structured(&clicked));+ assert!(!recorded_calls(&f, "Input.dispatchMouseEvent").is_empty());+}