Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
test(cua-driver): attach before the background oracles on macOS, then check the same-Space refusal (#5063)
Francesco Bonacci88602d15650a
diff --git a/libs/cua-driver/rust/crates/cua-driver-e2e/tests/standalone_browser_behavior_test.rs b/libs/cua-driver/rust/crates/cua-driver-e2e/tests/standalone_browser_behavior_test.rsindex 59806451eb..888d432462 100644--- a/libs/cua-driver/rust/crates/cua-driver-e2e/tests/standalone_browser_behavior_test.rs+++ b/libs/cua-driver/rust/crates/cua-driver-e2e/tests/standalone_browser_behavior_test.rs@@ -349,6 +349,9 @@ struct BrowserFixture {cdp_port: u16,pid: u32,window_id: u64,+ /// Sessions attached by `prepare_same_space_attachment` before the+ /// background oracles started (macOS only).+ attached_before_observation: Vec<String>,}fn browser_specs() -> Vec<BrowserSpec> {@@ -1290,6 +1293,7 @@ fn launch_browser_with_driver(cdp_port,pid,window_id,+ attached_before_observation: Vec::new(),}}@@ -1326,6 +1330,7 @@ fn launch_unprepared_browser(spec: &BrowserSpec, label: &str) -> BrowserFixturecdp_port: 0,pid,window_id,+ attached_before_observation: Vec::new(),}}@@ -1434,11 +1439,30 @@ fn semantic_ref_by_name(snapshot: &ToolResponse, name: &str, action: &str) -> St.to_owned()}-fn bind(fixture: &mut BrowserFixture, session: &str) -> (String, String, ToolResponse) {+/// On macOS, attaching to an existing Chrome or Edge profile shows Chrome's+/// "Allow remote debugging?" prompt, which brings the browser forward. When+/// the browser shares the user's Space, the driver therefore attaches only+/// while the browser is already in front (or with+/// `delivery_mode:"foreground"`), and refuses a background attachment with+/// `background_unavailable`. Attach here, before the background oracles+/// start, while the test-owned browser is in front. `bind` then checks the+/// refusal under the oracles and reuses this attachment.+fn prepare_same_space_attachment(+ fixture: &mut BrowserFixture,+ session: &str,+) -> Option<ToolResponse> {+ if !cfg!(target_os = "macos") {+ return None;+ }let started = fixture.driver.call("start_session", serde_json::json!({ "session": session }));assert!(!started.is_error(), "start_session failed: {}", started.raw);+ let raised = fixture.driver.call(+ "bring_to_front",+ serde_json::json!({ "pid": fixture.pid, "window_id": fixture.window_id }),+ );+ assert!(!raised.is_error(), "bring_to_front failed: {}", raised.raw);let prepared = fixture.driver.call("browser_prepare",serde_json::json!({@@ -1455,6 +1479,64 @@ fn bind(fixture: &mut BrowserFixture, session: &str) -> (String, String, ToolRes"{}",prepared.raw);+ fixture.attached_before_observation.push(session.to_owned());+ Some(prepared)+}++/// A background existing-profile attachment while another app is in front+/// on the same Space must refuse without visible effects (macOS).+fn assert_same_space_background_refusal(prepared: &ToolResponse) {+ assert_eq!(+ prepared.structured()["refusal"]["code"],+ "background_unavailable",+ "{}",+ prepared.raw+ );+ assert_eq!(+ prepared.structured()["refusal"]["detail"]["visible_effects"],+ false,+ "{}",+ prepared.raw+ );+ assert_eq!(+ prepared.structured()["refusal"]["detail"]["alternative_delivery_mode"],+ "foreground",+ "{}",+ prepared.raw+ );+}++fn bind(fixture: &mut BrowserFixture, session: &str) -> (String, String, ToolResponse) {+ let attached_before = fixture+ .attached_before_observation+ .iter()+ .any(|attached| attached == session);+ if !attached_before {+ let started = fixture+ .driver+ .call("start_session", serde_json::json!({ "session": session }));+ assert!(!started.is_error(), "start_session failed: {}", started.raw);+ }+ let prepared = fixture.driver.call(+ "browser_prepare",+ serde_json::json!({+ "pid": fixture.pid as i64,+ "window_id": fixture.window_id,+ "session": session,+ "strategy": {"kind": "existing_profile"},+ }),+ );+ if attached_before {+ assert_same_space_background_refusal(&prepared);+ } else {+ assert_eq!(prepared.structured()["status"], "ok", "{}", prepared.raw);+ assert_eq!(+ prepared.structured()["action"],+ "attached_existing_profile",+ "{}",+ prepared.raw+ );+ }// A newly mapped Wayland toplevel can briefly appear with a protocol-local// id before the compositor publishes its stable pid/geometry identity.// Model the real client preamble: re-list windows and bind only the id the@@ -1768,8 +1850,9 @@ fn run_roundtrip(spec: &BrowserSpec) {execute_case(case(&spec.name, "browser_tool_roundtrip"), |evidence| {let mut fixture = launch_browser(spec, &scenario);*evidence = recording_evidence(fixture.driver.recording_dir());+ let session = format!("standalone-roundtrip-{}", fixture.pid);+ prepare_same_space_attachment(&mut fixture, &session);run_with_background_oracles(&mut fixture, |fixture| {- let session = format!("standalone-roundtrip-{}", fixture.pid);let (target, tab, snapshot) = bind(fixture, &session);let click_ref = ref_by_label(&snapshot, "id=btn-increment");let click = fixture.driver.call(@@ -1822,8 +1905,9 @@ fn run_trust_gated_dom_click(spec: &BrowserSpec) {let mut fixture =launch_browser_with_html(spec, &scenario, standalone_trust_gated_click_html());*evidence = recording_evidence(fixture.driver.recording_dir());+ let session = format!("standalone-trust-gated-dom-click-{}", fixture.pid);+ prepare_same_space_attachment(&mut fixture, &session);run_with_background_oracles(&mut fixture, |fixture| {- let session = format!("standalone-trust-gated-dom-click-{}", fixture.pid);let (target, tab, snapshot) = bind(fixture, &session);let click_ref = ref_by_label(&snapshot, "id=standalone-trust-gated");let click = fixture.driver.call(@@ -1884,8 +1968,9 @@ fn run_semantic_state(spec: &BrowserSpec) {let mut fixture =launch_browser_with_html(spec, &scenario, standalone_semantic_fixture_html());*evidence = recording_evidence(fixture.driver.recording_dir());+ let session = format!("standalone-semantic-state-{}", fixture.pid);+ prepare_same_space_attachment(&mut fixture, &session);run_with_background_oracles(&mut fixture, |fixture| {- let session = format!("standalone-semantic-state-{}", fixture.pid);let (target, tab, _) = bind(fixture, &session);let snapshot = fixture.driver.call("get_browser_state",@@ -1992,8 +2077,9 @@ fn run_background_type(spec: &BrowserSpec) {execute_case(case(&spec.name, "background_type"), |evidence| {let mut fixture = launch_browser(spec, &scenario);*evidence = recording_evidence(fixture.driver.recording_dir());+ let session = format!("standalone-background-type-{}", fixture.pid);+ prepare_same_space_attachment(&mut fixture, &session);run_with_background_oracles(&mut fixture, |fixture| {- let session = format!("standalone-background-type-{}", fixture.pid);let (target, tab, snapshot) = bind(fixture, &session);let input_ref = ref_by_label(&snapshot, "id=txt-input");let typed = fixture.driver.call(@@ -2358,8 +2444,9 @@ fn run_trusted_click(spec: &BrowserSpec) {execute_case(case, |evidence| {let mut fixture = launch_browser(spec, &scenario);*evidence = recording_evidence(fixture.driver.recording_dir());+ let session = format!("standalone-trusted-click-{}", fixture.pid);+ prepare_same_space_attachment(&mut fixture, &session);run_with_background_oracles(&mut fixture, |fixture| {- let session = format!("standalone-trusted-click-{}", fixture.pid);let (target, tab, snapshot) = bind(fixture, &session);let click_ref = ref_by_label(&snapshot, "id=btn-increment");let click = fixture.driver.call(@@ -2612,12 +2699,15 @@ fn run_existing_profile_attach(spec: &BrowserSpec) {|evidence| {let mut fixture = launch_browser(spec, &scenario);*evidence = recording_evidence(fixture.driver.recording_dir());+ let session = format!("standalone-existing-profile-{}", fixture.pid);+ let attached_before = prepare_same_space_attachment(&mut fixture, &session);run_with_background_oracles(&mut fixture, |fixture| {- let session = format!("standalone-existing-profile-{}", fixture.pid);- let started = fixture- .driver- .call("start_session", serde_json::json!({ "session": session }));- assert!(!started.is_error(), "start_session failed: {}", started.raw);+ if attached_before.is_none() {+ let started = fixture+ .driver+ .call("start_session", serde_json::json!({ "session": session }));+ assert!(!started.is_error(), "start_session failed: {}", started.raw);+ }fixture.driver.start_behavior_recording();let prepared = fixture.driver.call(@@ -2629,50 +2719,62 @@ fn run_existing_profile_attach(spec: &BrowserSpec) {"strategy": {"kind": "existing_profile"},}),);- assert_eq!(prepared.structured()["status"], "ok", "{}", prepared.raw);- assert_eq!(- prepared.structured()["action"],- "attached_existing_profile",- "{}",- prepared.raw- );- assert_eq!(- prepared.structured()["attachment"]["kind"],- "existing_profile"- );- assert_eq!(- prepared.structured()["attachment"]["capabilities_invalidated"],- true- );- assert_eq!(- prepared.structured()["attachment"]["next_action"],- "get_browser_state"- );- assert_eq!(- prepared.structured()["side_effects"]["launched_browser"],- false- );- assert_eq!(- prepared.structured()["side_effects"]["created_profile"],- false- );- assert_eq!(- prepared.structured()["side_effects"]["copied_profile_data"],- false- );+ // On macOS the attachment was made before observation, while+ // the browser was in front; check its result, and that a+ // background attachment now refuses without visible effects.+ let prepared = match &attached_before {+ Some(attached) => {+ assert_same_space_background_refusal(&prepared);+ attached+ }+ None => &prepared,+ };+ {+ assert_eq!(prepared.structured()["status"], "ok", "{}", prepared.raw);+ assert_eq!(+ prepared.structured()["action"],+ "attached_existing_profile",+ "{}",+ prepared.raw+ );+ assert_eq!(+ prepared.structured()["attachment"]["kind"],+ "existing_profile"+ );+ assert_eq!(+ prepared.structured()["attachment"]["capabilities_invalidated"],+ true+ );+ assert_eq!(+ prepared.structured()["attachment"]["next_action"],+ "get_browser_state"+ );+ assert_eq!(+ prepared.structured()["side_effects"]["launched_browser"],+ false+ );+ assert_eq!(+ prepared.structured()["side_effects"]["created_profile"],+ false+ );+ assert_eq!(+ prepared.structured()["side_effects"]["copied_profile_data"],+ false+ );- let public_result = prepared.raw.to_string();- assert!(!public_result.contains("ws://"), "{}", prepared.raw);- assert!(- !public_result.contains("webSocketDebuggerUrl"),- "{}",- prepared.raw- );- assert!(- !public_result.contains(&fixture._profile.path().display().to_string()),- "{}",- prepared.raw- );+ let public_result = prepared.raw.to_string();+ assert!(!public_result.contains("ws://"), "{}", prepared.raw);+ assert!(+ !public_result.contains("webSocketDebuggerUrl"),+ "{}",+ prepared.raw+ );+ assert!(+ !public_result.contains(&fixture._profile.path().display().to_string()),+ "{}",+ prepared.raw+ );+ }let state = fixture.driver.call("get_browser_state",@@ -3054,8 +3156,9 @@ fn run_stale_ref(spec: &BrowserSpec) {execute_case(case(&spec.name, "browser_ref_stale"), |evidence| {let mut fixture = launch_browser(spec, &scenario);*evidence = recording_evidence(fixture.driver.recording_dir());+ let session = format!("standalone-stale-ref-{}", fixture.pid);+ prepare_same_space_attachment(&mut fixture, &session);run_with_background_oracles(&mut fixture, |fixture| {- let session = format!("standalone-stale-ref-{}", fixture.pid);let (target, tab, first) = bind(fixture, &session);let stale_ref = ref_by_label(&first, "id=btn-increment");let newer = fixture.driver.call(@@ -3105,8 +3208,9 @@ fn run_frame_roundtrip(spec: &BrowserSpec) {wait_for_text(&fixture.server, "standalone-frame-state", "iframe=ready");wait_for_observed(&child_server, "WEB_HARNESS_MARKER_v1");+ let session = format!("standalone-frame-roundtrip-{}", fixture.pid);+ prepare_same_space_attachment(&mut fixture, &session);run_with_background_oracles(&mut fixture, |fixture| {- let session = format!("standalone-frame-roundtrip-{}", fixture.pid);let (target, tab, snapshot) = bind(fixture, &session);assert_eq!(snapshot.structured()["oopif"]["status"],@@ -4398,8 +4502,9 @@ fn run_dialog_background_refusal(spec: &BrowserSpec) {let mut fixture =launch_browser_with_html(spec, &scenario, standalone_browser_completeness_html());*evidence = recording_evidence(fixture.driver.recording_dir());+ let session = format!("standalone-dialog-refusal-{}", fixture.pid);+ prepare_same_space_attachment(&mut fixture, &session);run_with_background_oracles(&mut fixture, |fixture| {- let session = format!("standalone-dialog-refusal-{}", fixture.pid);let (target_id, tab_id, _) = bind(fixture, &session);let refused = fixture.driver.call("browser_dialog",@@ -4568,8 +4673,9 @@ fn run_upload(spec: &BrowserSpec) {let mut fixture =launch_browser_with_html(spec, &scenario, standalone_browser_completeness_html());*evidence = recording_evidence(fixture.driver.recording_dir());+ let session = format!("standalone-upload-{}", fixture.pid);+ prepare_same_space_attachment(&mut fixture, &session);run_with_background_oracles(&mut fixture, |fixture| {- let session = format!("standalone-upload-{}", fixture.pid);let (target, tab, _) = bind(fixture, &session);let snapshot = fixture.driver.call("get_browser_state",@@ -4621,8 +4727,9 @@ fn run_pointer_actions(spec: &BrowserSpec) {let mut fixture =launch_browser_with_html(spec, &scenario, standalone_browser_completeness_html());*evidence = recording_evidence(fixture.driver.recording_dir());+ let session = format!("standalone-pointer-{}", fixture.pid);+ prepare_same_space_attachment(&mut fixture, &session);run_with_background_oracles(&mut fixture, |fixture| {- let session = format!("standalone-pointer-{}", fixture.pid);let (target, tab, _) = bind(fixture, &session);let snapshot = fixture.driver.call("get_browser_state",@@ -4747,8 +4854,9 @@ fn run_download(spec: &BrowserSpec) {let mut fixture =launch_browser_with_html(spec, &scenario, standalone_browser_completeness_html());*evidence = recording_evidence(fixture.driver.recording_dir());+ let session = format!("standalone-download-{}", fixture.pid);+ prepare_same_space_attachment(&mut fixture, &session);run_with_background_oracles(&mut fixture, |fixture| {- let session = format!("standalone-download-{}", fixture.pid);let (target, tab, snapshot) = bind(fixture, &session);// Sandboxed browser packages such as Ubuntu's Chromium snap use a// private /tmp namespace. A temporary directory under HOME remains@@ -4886,8 +4994,9 @@ fn run_type_replace(spec: &BrowserSpec) {execute_case(case(&spec.name, "browser_type_replace"), |evidence| {let mut fixture = launch_browser(spec, &scenario);*evidence = recording_evidence(fixture.driver.recording_dir());+ let session = format!("standalone-type-replace-{}", fixture.pid);+ prepare_same_space_attachment(&mut fixture, &session);run_with_background_oracles(&mut fixture, |fixture| {- let session = format!("standalone-type-replace-{}", fixture.pid);let (target, tab, snapshot) = bind(fixture, &session);let input_ref = ref_by_label(&snapshot, "id=txt-input");let number_input_ref = ref_by_label(&snapshot, "id=number-input");