Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
fix(cua-driver): clean up unresolved Chrome consent UI (#3468)
Francesco Bonaccie7295472e196
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/browser/platform.rs b/libs/cua-driver/rust/crates/platform-macos/src/browser/platform.rsindex 261139794e..603f2ae4ec 100644--- a/libs/cua-driver/rust/crates/platform-macos/src/browser/platform.rs+++ b/libs/cua-driver/rust/crates/platform-macos/src/browser/platform.rs@@ -20,8 +20,182 @@ use cua_driver_core::browser::types::{EndpointOwnershipMethod, EndpointOwnershipProof, EndpointTransport, NativeOwnershipMethod,NativeOwnershipProof, NativeWindowInfo, OwnedEndpoint, ProcessFingerprint, Rect,};+use serde::Serialize;use tokio::io::{AsyncReadExt, AsyncWriteExt};+const CONSENT_CLEANUP_PASSES: usize = 20;++#[derive(Clone, Debug, Serialize)]+struct ConsentSurfaceEvidence {+ window_id: u32,+ pid: i32,+ bounds: [f64; 4],+}++fn consent_surface_evidence(+ windows: impl IntoIterator<Item = crate::windows::WindowInfo>,+ pid: i32,+) -> Vec<ConsentSurfaceEvidence> {+ let mut surfaces = windows+ .into_iter()+ .filter(|window| {+ window.pid == pid+ && window.title == "Allow remote debugging?"+ && window.is_on_screen+ && window.bounds.width > 0.0+ && window.bounds.height > 0.0+ })+ .map(|window| ConsentSurfaceEvidence {+ window_id: window.window_id,+ pid: window.pid,+ bounds: [+ window.bounds.x,+ window.bounds.y,+ window.bounds.width,+ window.bounds.height,+ ],+ })+ .collect::<Vec<_>>();+ surfaces.sort_by_key(|surface| surface.window_id);+ surfaces+}++fn visible_consent_surfaces(pid: i32) -> Vec<ConsentSurfaceEvidence> {+ consent_surface_evidence(crate::windows::all_windows(), pid)+}++fn add_consent_cleanup_detail(+ mut error: BrowserRefusal,+ cleanup: serde_json::Value,+) -> BrowserRefusal {+ let prior = error.detail.take();+ error.detail = Some(match prior {+ Some(serde_json::Value::Object(mut detail)) => {+ detail.insert("consent_cleanup".to_owned(), cleanup);+ serde_json::Value::Object(detail)+ }+ Some(cause) => serde_json::json!({+ "cause": cause,+ "consent_cleanup": cleanup,+ }),+ None => serde_json::json!({ "consent_cleanup": cleanup }),+ });+ error+}++fn run_failed_prepare_consent_cleanup<D, R, O, P>(+ error: BrowserRefusal,+ max_passes: usize,+ mut dismiss: D,+ rollback: R,+ mut observe: O,+ mut pause: P,+) -> BrowserRefusal+where+ D: FnMut() -> Result<bool, BrowserRefusal>,+ R: FnOnce(BrowserRefusal) -> BrowserRefusal,+ O: FnMut() -> Vec<ConsentSurfaceEvidence>,+ P: FnMut(),+{+ let mut dismissed_via_ax = false;+ let mut dismiss_error = None;+ match dismiss() {+ Ok(dismissed) => dismissed_via_ax |= dismissed,+ Err(error) => dismiss_error = Some(error),+ }++ // The retained setup handle is the exact authorization to undo the+ // setting change. Always run it even when consent UI is AX-ambiguous.+ let error = rollback(error);+ let mut retained_surfaces = Vec::new();+ for pass in 0..max_passes {+ match dismiss() {+ Ok(dismissed) => {+ dismissed_via_ax |= dismissed;+ dismiss_error = None;+ }+ Err(error) => dismiss_error = Some(error),+ }+ retained_surfaces = observe();+ if pass + 1 < max_passes {+ pause();+ }+ }++ let status = if retained_surfaces.is_empty() {+ "dismissed"+ } else {+ "retained"+ };+ add_consent_cleanup_detail(+ error,+ serde_json::json!({+ "status": status,+ "dismissed_via_ax": dismissed_via_ax,+ "retained_surfaces": retained_surfaces,+ "dismiss_error": dismiss_error,+ }),+ )+}++fn run_committed_consent_cleanup<D, C, O, P>(+ max_passes: usize,+ mut dismiss: D,+ cleanup_setup: C,+ mut observe: O,+ mut pause: P,+) -> Result<bool, BrowserRefusal>+where+ D: FnMut() -> Result<bool, BrowserRefusal>,+ C: FnOnce() -> Result<bool, BrowserRefusal>,+ O: FnMut() -> Vec<ConsentSurfaceEvidence>,+ P: FnMut(),+{+ let mut changed = false;+ let mut dismiss_error = None;+ match dismiss() {+ Ok(dismissed) => changed |= dismissed,+ Err(error) => dismiss_error = Some(error),+ }+ let cleanup_result = cleanup_setup();+ let cleanup_error = cleanup_result.as_ref().err().cloned();+ if let Ok(cleaned) = cleanup_result.as_ref() {+ changed |= *cleaned;+ }++ let mut retained_surfaces = Vec::new();+ for pass in 0..max_passes {+ match dismiss() {+ Ok(dismissed) => {+ changed |= dismissed;+ dismiss_error = None;+ }+ Err(error) => dismiss_error = Some(error),+ }+ retained_surfaces = observe();+ if pass + 1 < max_passes {+ pause();+ }+ }++ if !retained_surfaces.is_empty() {+ let mut error = cleanup_error.unwrap_or_else(|| {+ refusal(+ BrowserRefusalCode::BrowserWrongTargetRefused,+ "remote-debugging cleanup retained browser consent UI",+ )+ });+ error.detail = Some(serde_json::json!({+ "status": "retained",+ "retained_surfaces": retained_surfaces,+ "dismiss_error": dismiss_error,+ }));+ return Err(error);+ }++ cleanup_result.map(|_| changed)+}+#[derive(Clone)]pub struct MacOsBrowserPlatform {cursor_registry: Arc<crate::cursor::CursorRegistry>,@@ -1112,10 +1286,13 @@ impl BrowserPlatform for MacOsBrowserPlatform {"the approved browser window is outside the macOS window-id range",)})?;- let dismissed_before = super::consent_ui::dismiss(pid, window_id)?;- let closed_setup_page = super::setup_ui::disable(pid, window_id, descriptor)?;- let dismissed_after = super::consent_ui::dismiss(pid, window_id)?;- Ok(dismissed_before || closed_setup_page || dismissed_after)+ run_committed_consent_cleanup(+ CONSENT_CLEANUP_PASSES,+ || super::consent_ui::dismiss(pid, window_id),+ || super::setup_ui::disable(pid, window_id, descriptor),+ || visible_consent_surfaces(pid),+ || std::thread::sleep(Duration::from_millis(100)),+ )}async fn abort_existing_profile_setup(@@ -1129,14 +1306,23 @@ impl BrowserPlatform for MacOsBrowserPlatform {let Ok(window_id) = u32::try_from(request.window_id) else {return error;};- tokio::task::spawn_blocking(move || super::setup_ui::abort_pending(pid, window_id, error))- .await- .unwrap_or_else(|join_error| {- refusal(- BrowserRefusalCode::BrowserRouteUnavailable,- format!("could not roll back exact browser setup: {join_error}"),- )- })+ tokio::task::spawn_blocking(move || {+ run_failed_prepare_consent_cleanup(+ error,+ CONSENT_CLEANUP_PASSES,+ || super::consent_ui::dismiss(pid, window_id),+ |error| super::setup_ui::abort_pending(pid, window_id, error),+ || visible_consent_surfaces(pid),+ || std::thread::sleep(Duration::from_millis(100)),+ )+ })+ .await+ .unwrap_or_else(|join_error| {+ refusal(+ BrowserRefusalCode::BrowserRouteUnavailable,+ format!("could not roll back exact browser setup: {join_error}"),+ )+ })}async fn handle_existing_profile_consent(@@ -1186,6 +1372,160 @@ impl BrowserPlatform for MacOsBrowserPlatform {mod tests {use super::*;+ fn consent_surface(window_id: u32) -> ConsentSurfaceEvidence {+ ConsentSurfaceEvidence {+ window_id,+ pid: 42,+ bounds: [120.0, 120.0, 520.0, 260.0],+ }+ }++ #[test]+ fn consent_cleanup_evidence_requires_exact_pid_title_visibility_and_bounds() {+ let exact = window(90, 42, "Allow remote debugging?");+ let mut foreign = window(91, 7, "Allow remote debugging?");+ foreign.app_name = "Google Chrome".to_owned();+ let mut hidden = window(92, 42, "Allow remote debugging?");+ hidden.is_on_screen = false;+ let mut empty = window(93, 42, "Allow remote debugging?");+ empty.bounds.width = 0.0;++ let surfaces = consent_surface_evidence(+ [exact, foreign, hidden, empty, window(94, 42, "Unrelated")],+ 42,+ );+ assert_eq!(surfaces.len(), 1);+ assert_eq!(surfaces[0].window_id, 90);+ assert_eq!(surfaces[0].pid, 42);+ }++ #[test]+ fn failed_prepare_cleanup_drains_stacked_and_late_consent_surfaces() {+ let events = std::cell::RefCell::new(Vec::new());+ let mut observations = vec![+ vec![consent_surface(90), consent_surface(91)],+ vec![consent_surface(92)],+ Vec::new(),+ Vec::new(),+ ]+ .into_iter();+ let error = run_failed_prepare_consent_cleanup(+ refusal(+ BrowserRefusalCode::BrowserWrongTargetRefused,+ "prepare failed",+ ),+ 4,+ || {+ events.borrow_mut().push("dismiss");+ Ok(false)+ },+ |error| {+ events.borrow_mut().push("rollback");+ error+ },+ || observations.next().unwrap_or_default(),+ || {},+ );++ assert_eq!(+ *events.borrow(),+ ["dismiss", "rollback", "dismiss", "dismiss", "dismiss", "dismiss"]+ );+ assert_eq!(+ error.detail.as_ref().unwrap()["consent_cleanup"]["status"],+ "dismissed"+ );+ assert_eq!(+ error.detail.as_ref().unwrap()["consent_cleanup"]["retained_surfaces"],+ serde_json::json!([])+ );+ }++ #[test]+ fn failed_prepare_cleanup_returns_bounded_evidence_for_retained_surfaces() {+ let mut rollback_calls = 0;+ let error = run_failed_prepare_consent_cleanup(+ refusal(+ BrowserRefusalCode::BrowserWrongTargetRefused,+ "prepare failed",+ ),+ 3,+ || {+ Err(refusal(+ BrowserRefusalCode::BrowserWrongTargetRefused,+ "ambiguous cancel actions",+ ))+ },+ |error| {+ rollback_calls += 1;+ error+ },+ || vec![consent_surface(90), consent_surface(91)],+ || {},+ );++ assert_eq!(+ rollback_calls, 1,+ "ambiguity must not skip exact setup rollback"+ );+ assert_eq!(+ error.detail.as_ref().unwrap()["consent_cleanup"]["status"],+ "retained"+ );+ assert_eq!(+ error.detail.as_ref().unwrap()["consent_cleanup"]["retained_surfaces"]+ .as_array()+ .unwrap()+ .len(),+ 2+ );+ assert_eq!(+ error.detail.as_ref().unwrap()["consent_cleanup"]["dismiss_error"]["message"],+ "ambiguous cancel actions"+ );+ }++ #[test]+ fn committed_cleanup_refuses_while_an_unresolved_consent_surface_is_retained() {+ let result = run_committed_consent_cleanup(+ 3,+ || Ok(false),+ || Ok(true),+ || vec![consent_surface(90)],+ || {},+ );++ let error = result.expect_err("retained consent UI must retain the core cleanup request");+ assert_eq!(error.code, BrowserRefusalCode::BrowserWrongTargetRefused);+ assert_eq!(error.detail.as_ref().unwrap()["status"], "retained");+ assert_eq!(+ error.detail.as_ref().unwrap()["retained_surfaces"][0]["window_id"],+ 90+ );+ }++ #[test]+ fn committed_cleanup_observes_the_full_window_for_late_surfaces() {+ let mut observations = vec![+ Vec::new(),+ Vec::new(),+ Vec::new(),+ vec![consent_surface(90)],+ Vec::new(),+ Vec::new(),+ ]+ .into_iter();+ let result = run_committed_consent_cleanup(+ 6,+ || Ok(false),+ || Ok(true),+ || observations.next().unwrap_or_default(),+ || {},+ );++ assert_eq!(result.unwrap(), true);+ }+#[test]fn isolated_browser_candidates_are_vendor_attested_system_installs() {let candidates = isolated_browser_candidates();