Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
fix(cua-driver): declare browser chrome capture coverage (#2599)
injaneitya36865c236e7
diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/lib.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/lib.rsindex 62e517a4be..f98663ed3a 100644--- a/libs/cua-driver/rust/crates/cua-driver-core/src/lib.rs+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/lib.rs@@ -80,6 +80,7 @@ pub mod tool_args;pub mod tool_schema;pub mod video;pub mod video_ffmpeg;+pub mod window_inspection;pub use cua_driver_contract::{CaptureScope, EscalationReason};pub use recording::RecordingSession;diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/window_inspection.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/window_inspection.rsnew file mode 100644index 0000000000..7da45026d2--- /dev/null+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/window_inspection.rs@@ -0,0 +1,103 @@+//! Cross-platform window-snapshot coverage signals.+//!+//! A browser permission bubble is browser chrome, not page content. Chromium+//! may composite that chrome in a popup/child surface which is visible in a+//! desktop capture but absent from a capture of the requested native window.+//! A single-window backend therefore cannot prove that no browser-owned blocker+//! exists. Keep that limitation explicit and machine-readable.++use serde_json::{json, Value};++pub const BROWSER_CHROME_COVERAGE_STATUS: &str = "not_observable_in_window_scope";++/// Describe the browser-chrome coverage limit of a Chromium-family window+/// snapshot.+///+/// This deliberately does **not** claim that a prompt is present. Presence is+/// not observable from a single native-window surface on every supported+/// platform. It also carries no prompt text or choices, so routine traces can+/// retain the recovery signal without retaining permission content.+pub fn mark_browser_chrome_capture_coverage(structured: &mut Value, chromium_family_window: bool) {+ if !chromium_family_window {+ return;+ }++ structured["capture_coverage"] = json!({+ "browser_chrome": {+ "status": BROWSER_CHROME_COVERAGE_STATUS+ },+ "recovery": {+ "when": "verified_window_action_ineffective",+ "escalate": {+ "tool": "escalate_session",+ "reason": "foreground_ineffective"+ },+ "inspect": "get_desktop_state",+ "act_scope": "desktop",+ "verify": "get_desktop_state"+ }+ });+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn browser_window_declares_capture_coverage_without_claiming_presence() {+ let mut before = json!({+ "window_id": 7,+ "pid": 42,+ "tree_markdown": "page=unchanged",+ "screenshot_width": 900,+ "screenshot_height": 640+ });+ let mut after_visible_browser_blocker = before.clone();++ mark_browser_chrome_capture_coverage(&mut before, true);+ mark_browser_chrome_capture_coverage(&mut after_visible_browser_blocker, true);++ // Even when the page-owned state is byte-for-byte unchanged, callers+ // receive an explicit recovery branch instead of treating the window+ // snapshot as proof that the action was ignored.+ assert_eq!(before, after_visible_browser_blocker);+ assert_eq!(+ before["capture_coverage"]["browser_chrome"]["status"],+ BROWSER_CHROME_COVERAGE_STATUS+ );+ assert_eq!(+ before["capture_coverage"]["recovery"]["when"],+ "verified_window_action_ineffective"+ );+ assert_eq!(+ before["capture_coverage"]["recovery"]["escalate"],+ json!({+ "tool": "escalate_session",+ "reason": "foreground_ineffective"+ })+ );+ assert_eq!(+ before["capture_coverage"]["recovery"]["inspect"],+ "get_desktop_state"+ );+ assert!(before.get("browser_chrome_prompt").is_none());+ }++ #[test]+ fn signal_is_privacy_minimal_and_does_not_change_non_browser_snapshots() {+ let mut ordinary = json!({"window_id": 9, "pid": 3});+ let unchanged = ordinary.clone();+ mark_browser_chrome_capture_coverage(&mut ordinary, false);+ assert_eq!(ordinary, unchanged);++ let mut browser = unchanged;+ mark_browser_chrome_capture_coverage(&mut browser, true);+ let public = browser.to_string();+ for sensitive_key in ["text", "message", "choice", "allow", "deny"] {+ assert!(+ !public.contains(sensitive_key),+ "coverage signal leaked a prompt-content field: {public}"+ );+ }+ }+}diff --git a/libs/cua-driver/rust/crates/cua-driver/tests/standalone_browser_behavior_test.rs b/libs/cua-driver/rust/crates/cua-driver/tests/standalone_browser_behavior_test.rsindex 11558754aa..614972eea4 100644--- a/libs/cua-driver/rust/crates/cua-driver/tests/standalone_browser_behavior_test.rs+++ b/libs/cua-driver/rust/crates/cua-driver/tests/standalone_browser_behavior_test.rs@@ -104,6 +104,36 @@ fn standalone_browser_completeness_html() -> String {)}+/// Sanitized fixture for browser-owned permission chrome. The page contains no+/// origin names, prompt copy, or user choices; it records only lifecycle state.+/// A fresh Chromium profile should keep the returned promise pending while its+/// browser-owned notification permission bubble is visible.+fn standalone_browser_permission_prompt_html() -> String {+ standalone_fixture_html().replace(+ "</body>",+ r#"<fieldset style="position:fixed;left:16px;top:120px;z-index:10000;background:white">+ <legend>browser-owned permission</legend>+ <button id="standalone-browser-permission" data-cua-id="standalone-browser-permission"+ aria-label="Request browser permission">Request browser permission</button>+ <span id="standalone-browser-permission-state"+ data-cua-id="standalone-browser-permission-state"+ style="display:none">permission=idle</span>+</fieldset>+<script>+ document.getElementById('standalone-browser-permission').addEventListener('click', () => {+ const state = document.getElementById('standalone-browser-permission-state');+ state.textContent = 'permission=requested';+ Notification.requestPermission().then(result => {+ state.textContent = `permission=resolved:${result}`;+ }, () => {+ state.textContent = 'permission=error';+ });+ });+</script>+</body>"#,+ )+}+fn standalone_semantic_fixture_html() -> String {let hidden = (0..320).map(|index| {@@ -1108,6 +1138,7 @@ fn spawn_browser_command(url: &str,position: (i32, i32),force_high_device_scale: bool,+ disable_quiet_notification_prompts: bool,) {let mut command = command_for_browser(spec,@@ -1117,6 +1148,12 @@ fn spawn_browser_command(position,force_high_device_scale,);+ if cfg!(target_os = "windows") && disable_quiet_notification_prompts {+ // Edge defaults to Chromium's quiet notification UI, which exposes+ // only an address-bar indicator and no permission surface to compare.+ // Keep this certification row on the full browser-owned prompt.+ command.arg("--disable-features=QuietNotificationPrompts");+ }if force_high_device_scale {command.arg("--force-device-scale-factor=2");}@@ -1159,6 +1196,7 @@ fn launch_browser_with_driver("about:blank",TEST_BROWSER_INITIAL_POSITION,label.contains("multi-tab"),+ label.contains("browser-owned-permission"),);navigate_initial_page(cdp_port, &server);record_browser_provenance(spec, cdp_port);@@ -3297,6 +3335,492 @@ fn run_same_title_tabs(spec: &BrowserSpec) {);}+fn browser_owned_permission_evidence_dir(spec: &BrowserSpec) -> (PathBuf, PathBuf) {+ let relative = PathBuf::from("capture-evidence").join(format!(+ "{}-{}-browser-owned-permission",+ std::env::consts::OS,+ spec.name+ ));+ let root = std::env::var_os("CUA_E2E_ARTIFACT_DIR")+ .map(PathBuf::from)+ .unwrap_or_else(|| std::env::temp_dir().join("cua-driver-browser-evidence"));+ let absolute = root.join(&relative);+ std::fs::create_dir_all(&absolute).expect("create browser-owned permission evidence directory");+ (absolute, relative)+}++fn browser_window_bounds(driver: &mut McpDriver, pid: u32, window_id: u64) -> (f64, f64, f64, f64) {+ let windows = driver.call("list_windows", serde_json::json!({"pid": pid as i64}));+ let bounds = windows.structured()["windows"]+ .as_array()+ .and_then(|windows| {+ windows+ .iter()+ .find(|window| window["window_id"].as_u64() == Some(window_id))+ })+ .map(|window| &window["bounds"])+ .unwrap_or_else(|| {+ panic!(+ "browser window {window_id} missing while collecting capture evidence: {}",+ windows.raw+ )+ });+ let value = |name: &str| {+ bounds[name]+ .as_f64()+ .unwrap_or_else(|| panic!("browser window bound {name:?} missing: {}", windows.raw))+ };+ (value("x"), value("y"), value("width"), value("height"))+}++fn load_capture(path: &Path) -> image::RgbaImage {+ image::open(path)+ .unwrap_or_else(|error| panic!("decode capture {}: {error}", path.display()))+ .to_rgba8()+}++fn desktop_window_crop(+ desktop_path: &Path,+ desktop: &ToolResponse,+ bounds: (f64, f64, f64, f64),+ output_size: (u32, u32),+) -> image::RgbaImage {+ let image = load_capture(desktop_path);+ let screen_width = desktop.structured()["screen_width"]+ .as_f64()+ .expect("desktop screen width");+ let screen_height = desktop.structured()["screen_height"]+ .as_f64()+ .expect("desktop screen height");+ let scale_x = f64::from(image.width()) / screen_width;+ let scale_y = f64::from(image.height()) / screen_height;+ let (x, y, width, height) = bounds;+ let left = (x * scale_x).round().max(0.0) as u32;+ let top = (y * scale_y).round().max(0.0) as u32;+ let width = (width * scale_x)+ .round()+ .max(1.0)+ .min(f64::from(image.width().saturating_sub(left))) as u32;+ let height = (height * scale_y)+ .round()+ .max(1.0)+ .min(f64::from(image.height().saturating_sub(top))) as u32;+ assert!(+ width > 0 && height > 0,+ "browser window bounds {bounds:?} are outside desktop capture {}x{}",+ image.width(),+ image.height()+ );+ let crop = image::imageops::crop_imm(&image, left, top, width, height).to_image();+ image::imageops::resize(+ &crop,+ output_size.0,+ output_size.1,+ image::imageops::FilterType::Triangle,+ )+}++fn changed_pixel_count(before: &image::RgbaImage, after: &image::RgbaImage) -> u64 {+ assert_eq!(before.dimensions(), after.dimensions());+ before+ .pixels()+ .zip(after.pixels())+ .filter(|(before, after)| {+ before+ .0+ .iter()+ .zip(after.0.iter())+ .take(3)+ .any(|(before, after)| before.abs_diff(*after) >= 32)+ })+ .count() as u64+}++fn browser_chrome_changed_pixel_center(+ before: &image::RgbaImage,+ after: &image::RgbaImage,+ desktop: &ToolResponse,+ bounds: (f64, f64, f64, f64),+) -> Option<(f64, f64)> {+ assert_eq!(before.dimensions(), after.dimensions());+ let screen_width = desktop.structured()["screen_width"].as_f64()?;+ let screen_height = desktop.structured()["screen_height"].as_f64()?;+ let scale_x = f64::from(before.width()) / screen_width;+ let scale_y = f64::from(before.height()) / screen_height;+ let left = (bounds.0 * scale_x).round().max(0.0) as u32;+ let top = (bounds.1 * scale_y).round().max(0.0) as u32;+ let right = ((bounds.0 + bounds.2) * scale_x)+ .round()+ .min(f64::from(before.width())) as u32;+ // Browser permission indicators live in the top chrome. Limiting the+ // search excludes any unrelated page repaint below the toolbar.+ let bottom = ((bounds.1 + bounds.3.min(120.0)) * scale_y)+ .round()+ .min(f64::from(before.height())) as u32;+ let mut changed = 0_u64;+ let mut x_sum = 0_u64;+ let mut y_sum = 0_u64;+ for y in top..bottom {+ for x in left..right {+ let before = before.get_pixel(x, y);+ let after = after.get_pixel(x, y);+ if before+ .0+ .iter()+ .zip(after.0.iter())+ .take(3)+ .any(|(before, after)| before.abs_diff(*after) >= 32)+ {+ changed += 1;+ x_sum += u64::from(x);+ y_sum += u64::from(y);+ }+ }+ }+ (changed > 0).then(|| {+ (+ (x_sum as f64 / changed as f64) / scale_x,+ (y_sum as f64 / changed as f64) / scale_y,+ )+ })+}++fn run_browser_owned_permission_prompt(spec: &BrowserSpec) {+ let scenario = format!(+ "{}-{}-standalone-browser-owned-permission",+ std::env::consts::OS,+ spec.name+ );+ let mut spec_case = case(&spec.name, "browser_owned_permission");+ spec_case.oracles = vec![OracleKind::FixtureState, OracleKind::Pixels];+ execute_case(spec_case, |evidence| {+ let mut fixture =+ launch_browser_with_html(spec, &scenario, standalone_browser_permission_prompt_html());+ *evidence = recording_evidence(fixture.driver.recording_dir());+ let (evidence_dir, relative_evidence_dir) = browser_owned_permission_evidence_dir(spec);+ let window_before_path = evidence_dir.join("window-before.png");+ let desktop_before_path = evidence_dir.join("desktop-before.png");+ let window_after_path = evidence_dir.join("window-after.png");+ let desktop_after_path = evidence_dir.join("desktop-after.png");+ let desktop_crop_before_path = evidence_dir.join("desktop-window-crop-before.png");+ let desktop_crop_after_path = evidence_dir.join("desktop-window-crop-after.png");+ let metrics_path = evidence_dir.join("capture-metrics.json");+ let window_session = format!("standalone-browser-permission-window-{}", fixture.pid);+ let after_window_session =+ format!("standalone-browser-permission-window-after-{}", fixture.pid);+ let browser_session = format!("standalone-browser-permission-page-{}", fixture.pid);+ let bounds = browser_window_bounds(&mut fixture.driver, fixture.pid, fixture.window_id);+ let before = fixture.driver.call(+ "get_window_state",+ serde_json::json!({+ "pid": fixture.pid as i64,+ "window_id": fixture.window_id,+ "session": window_session,+ "screenshot_out_file": window_before_path,+ }),+ );+ if cfg!(target_os = "linux") {+ assert!(+ before.structured()["capture_coverage"]["browser_chrome"].is_null(),+ "Linux window capture already includes the browser-owned prompt surface: {}",+ before.raw+ );+ } else {+ assert_eq!(+ before.structured()["capture_coverage"]["browser_chrome"]["status"],+ "not_observable_in_window_scope",+ "{}",+ before.raw+ );+ assert_eq!(+ before.structured()["capture_coverage"]["recovery"]["when"],+ "verified_window_action_ineffective",+ "{}",+ before.raw+ );+ assert_eq!(+ before.structured()["capture_coverage"]["recovery"]["escalate"],+ serde_json::json!({+ "tool": "escalate_session",+ "reason": "foreground_ineffective",+ }),+ "{}",+ before.raw+ );+ }+ let escalated = fixture.driver.call(+ "escalate_session",+ serde_json::json!({+ "session": window_session,+ "reason": "foreground_ineffective",+ "detail": "browser chrome may be outside window capture",+ }),+ );+ assert!(+ !escalated.is_error(),+ "desktop inspection escalation failed: {}",+ escalated.raw+ );+ assert_eq!(+ escalated.structured()["effective_scope"],+ "desktop",+ "{}",+ escalated.raw+ );+ let desktop_before = fixture.driver.call(+ "get_desktop_state",+ serde_json::json!({+ "session": window_session,+ "screenshot_out_file": desktop_before_path,+ }),+ );+ assert!(+ !desktop_before.is_error(),+ "desktop baseline capture failed: {}",+ desktop_before.raw+ );+ let (target_id, tab_id, snapshot) = bind(&mut fixture, &browser_session);+ fixture.driver.start_behavior_recording();+ // Windows can prove and deliver a native trusted pointer route for+ // both Chrome and Edge. Edge rejects a synthetic DOM click before it+ // opens browser chrome. Linux/macOS currently expose only dom_event+ // through browser_click; Linux Chrome accepts it for this API.+ let input_route = if cfg!(target_os = "windows") {+ "trusted"+ } else {+ "dom_event"+ };+ let clicked = fixture.driver.call(+ "browser_click",+ serde_json::json!({+ "target_id": target_id,+ "tab_id": tab_id,+ "ref": ref_by_label(&snapshot, "id=standalone-browser-permission"),+ "input_route": input_route,+ "session": browser_session,+ }),+ );+ assert!(+ !clicked.is_error(),+ "permission trigger failed: {}",+ clicked.raw+ );+ wait_for_text(+ &fixture.server,+ "standalone-browser-permission-state",+ "permission=requested",+ );+ thread::sleep(Duration::from_millis(250));++ let window = fixture.driver.call(+ "get_window_state",+ serde_json::json!({+ "pid": fixture.pid as i64,+ "window_id": fixture.window_id,+ "session": after_window_session,+ "screenshot_out_file": window_after_path,+ }),+ );+ if cfg!(target_os = "linux") {+ assert!(+ window.structured()["capture_coverage"]["browser_chrome"].is_null(),+ "Linux window capture already includes the browser-owned prompt surface: {}",+ window.raw+ );+ } else {+ assert_eq!(+ window.structured()["capture_coverage"]["browser_chrome"]["status"],+ "not_observable_in_window_scope",+ "{}",+ window.raw+ );+ }+ assert!(+ !window+ .raw+ .to_string()+ .contains("Notification.requestPermission"),+ "window contract leaked fixture prompt internals: {}",+ window.raw+ );+ let mut desktop = fixture.driver.call(+ "get_desktop_state",+ serde_json::json!({+ "session": window_session,+ "screenshot_out_file": desktop_after_path,+ }),+ );+ assert!(+ !desktop.is_error(),+ "desktop fallback failed: {}",+ desktop.raw+ );++ let window_before = load_capture(&window_before_path);+ let mut window_after = load_capture(&window_after_path);+ assert_eq!(+ window_before.dimensions(),+ window_after.dimensions(),+ "browser window changed size while permission prompt was open"+ );+ let output_size = window_before.dimensions();+ let desktop_crop_before =+ desktop_window_crop(&desktop_before_path, &desktop_before, bounds, output_size);+ let mut desktop_crop_after =+ desktop_window_crop(&desktop_after_path, &desktop, bounds, output_size);+ let compared_pixels = u64::from(output_size.0) * u64::from(output_size.1);+ let minimum_prompt_pixels = (compared_pixels / 1_000).max(1_000);+ let initial_desktop_changed_pixels =+ changed_pixel_count(&desktop_crop_before, &desktop_crop_after);+ let mut quiet_prompt_expanded = false;+ let mut quiet_prompt_indicator = None;++ // Edge can collapse the notification request to a quiet indicator in+ // browser chrome. Expand the region that actually changed, then assess+ // the resulting browser-owned surface with the same pixel oracle.+ if cfg!(target_os = "windows")+ && spec.name == "edge"+ && initial_desktop_changed_pixels < minimum_prompt_pixels+ {+ let full_desktop_before = load_capture(&desktop_before_path);+ let full_desktop_after = load_capture(&desktop_after_path);+ let (x, y) = browser_chrome_changed_pixel_center(+ &full_desktop_before,+ &full_desktop_after,+ &desktop,+ bounds,+ )+ .expect("quiet permission indicator did not produce a changed browser chrome region");+ quiet_prompt_indicator = Some((x, y));+ let expanded = fixture.driver.call(+ "click",+ serde_json::json!({+ "session": window_session,+ "scope": "desktop",+ "x": x,+ "y": y,+ }),+ );+ assert!(+ !expanded.is_error(),+ "expand quiet permission indicator: {}",+ expanded.raw+ );+ thread::sleep(Duration::from_millis(250));+ let recaptured_window = fixture.driver.call(+ "get_window_state",+ serde_json::json!({+ "pid": fixture.pid as i64,+ "window_id": fixture.window_id,+ "session": after_window_session,+ "screenshot_out_file": window_after_path,+ }),+ );+ assert!(+ !recaptured_window.is_error(),+ "recapture expanded quiet permission window: {}",+ recaptured_window.raw+ );+ desktop = fixture.driver.call(+ "get_desktop_state",+ serde_json::json!({+ "session": window_session,+ "screenshot_out_file": desktop_after_path,+ }),+ );+ assert!(+ !desktop.is_error(),+ "recapture expanded quiet permission desktop: {}",+ desktop.raw+ );+ window_after = load_capture(&window_after_path);+ desktop_crop_after =+ desktop_window_crop(&desktop_after_path, &desktop, bounds, output_size);+ quiet_prompt_expanded = true;+ }++ desktop_crop_before+ .save(&desktop_crop_before_path)+ .expect("write normalized desktop baseline");+ desktop_crop_after+ .save(&desktop_crop_after_path)+ .expect("write normalized desktop permission capture");++ let desktop_changed_pixels = changed_pixel_count(&desktop_crop_before, &desktop_crop_after);+ let window_changed_pixels = changed_pixel_count(&window_before, &window_after);+ let minimum_desktop_to_window_ratio = 2_u64;+ let desktop_has_materially_more_prompt_pixels = window_changed_pixels+ .saturating_mul(minimum_desktop_to_window_ratio)+ < desktop_changed_pixels;+ let metrics = serde_json::json!({+ "platform": std::env::consts::OS,+ "browser": spec.name,+ "window_bounds": {+ "x": bounds.0,+ "y": bounds.1,+ "width": bounds.2,+ "height": bounds.3,+ },+ "compared_width": output_size.0,+ "compared_height": output_size.1,+ "compared_pixels": compared_pixels,+ "pixel_channel_threshold": 32,+ "minimum_prompt_pixels": minimum_prompt_pixels,+ "initial_desktop_changed_pixels": initial_desktop_changed_pixels,+ "desktop_changed_pixels": desktop_changed_pixels,+ "window_changed_pixels": window_changed_pixels,+ "minimum_desktop_to_window_ratio": minimum_desktop_to_window_ratio,+ "quiet_prompt_expanded": quiet_prompt_expanded,+ "quiet_prompt_indicator": quiet_prompt_indicator.map(|(x, y)| {+ serde_json::json!({"x": x, "y": y})+ }),+ "desktop_has_materially_more_prompt_pixels":+ desktop_has_materially_more_prompt_pixels,+ });+ std::fs::write(+ &metrics_path,+ serde_json::to_vec_pretty(&metrics).expect("serialize capture metrics"),+ )+ .expect("write capture metrics");++ evidence.screenshot = Some(+ relative_evidence_dir+ .join("desktop-after.png")+ .to_string_lossy()+ .replace('\\', "/"),+ );+ evidence.log = Some(+ relative_evidence_dir+ .join("capture-metrics.json")+ .to_string_lossy()+ .replace('\\', "/"),+ );+ assert!(+ desktop_changed_pixels >= minimum_prompt_pixels,+ "desktop capture did not show a material permission surface change: {metrics}"+ );+ if cfg!(target_os = "linux") {+ assert!(+ window_changed_pixels >= minimum_prompt_pixels,+ "Linux window capture omitted the material permission surface change: {metrics}"+ );+ assert!(+ !desktop_has_materially_more_prompt_pixels,+ "Linux desktop capture unexpectedly contained materially more permission UI than window capture: {metrics}"+ );+ } else {+ assert!(+ desktop_has_materially_more_prompt_pixels,+ "window capture omitted too little of the permission surface to justify desktop fallback: {metrics}"+ );+ }+ Observation::delivered(+ vec![OracleKind::FixtureState, OracleKind::Pixels],+ Evidence::default(),+ )+ });+}+fn run_dialogs(spec: &BrowserSpec) {let scenario = format!("{}-{}-standalone-dialogs", std::env::consts::OS, spec.name);let spec_case = if cfg!(target_os = "linux") {@@ -4006,6 +4530,10 @@ standalone_browser_test!(standalone_browser_frames, run_frame_roundtrip);standalone_browser_test!(standalone_browser_multi_tab, run_multi_tab);standalone_browser_test!(standalone_browser_same_title_tabs, run_same_title_tabs);standalone_browser_test!(standalone_browser_dialogs, run_dialogs);+standalone_browser_test!(+ standalone_browser_owned_permission_prompt,+ run_browser_owned_permission_prompt+);#[cfg(target_os = "linux")]standalone_browser_test!(standalone_browser_dialog_background_refusal,diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/get_window_state.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/get_window_state.rsindex 93f7e63bed..212290f954 100644--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/get_window_state.rs+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/get_window_state.rs@@ -105,6 +105,32 @@ fn def() -> &'static ToolDef {})}+fn chromium_browser_window(pid: i32) -> bool {+ let identity = format!(+ "{} {}",+ crate::apps::get_app_name_for_pid(pid).unwrap_or_default(),+ crate::apps::bundle_id_for_pid(pid).unwrap_or_default()+ )+ .to_ascii_lowercase();+ identity+ .split(|ch: char| !ch.is_ascii_alphanumeric())+ .any(|token| {+ matches!(+ token,+ "chrome"+ | "chromium"+ | "brave"+ | "edge"+ | "vivaldi"+ | "opera"+ | "arc"+ | "thorium"+ | "iridium"+ | "yandex"+ )+ })+}+#[async_trait]impl Tool for GetWindowStateTool {fn def(&self) -> &ToolDef {@@ -554,6 +580,10 @@ impl Tool for GetWindowStateTool {if let Some(ref fp) = screenshot_file_path {structured["screenshot_file_path"] = serde_json::json!(fp);}+ cua_driver_core::window_inspection::mark_browser_chrome_capture_coverage(+ &mut structured,+ chromium_browser_window(pid),+ );ToolResult {content,is_error: None,diff --git a/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs b/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rsindex 351b197018..6265c3b83c 100644--- a/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs+++ b/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs@@ -865,6 +865,14 @@ impl Tool for GetWindowStateTool {element trees. When applied, BOTH the markdown and the structured \elements are truncated identically. Omit both for current default behaviour \(≤5 000 elements, depth ≤25).\n\n\+ CHROMIUM COVERAGE: a browser-owned permission bubble can be \+ composited outside the requested native window. Chromium-family \+ snapshots therefore describe this limit in structuredContent.capture_coverage. \+ After a verified ineffective window action, call escalate_session, take a \+ fresh get_desktop_state snapshot, act explicitly in desktop scope if needed, \+ then verify with another fresh desktop snapshot. \+ This is separate from page JavaScript dialogs, which remain on \+ browser_dialog.\n\n\Windows requires no special permissions.".into(),input_schema: json!({"type":"object","required":["pid","window_id"],"properties":{"session": cua_driver_core::tool_schema::session_schema(),@@ -1197,6 +1205,11 @@ impl Tool for GetWindowStateTool {structured["screenshot_error"] = json!(err);}+ cua_driver_core::window_inspection::mark_browser_chrome_capture_coverage(+ &mut structured,+ is_standalone_chromium_browser_process(pid),+ );+ToolResult {content,is_error: None,@@ -1291,6 +1304,13 @@ fn is_chromium_browser_target(target: &str) -> bool {)}+fn is_standalone_chromium_browser_process(pid: u32) -> bool {+ crate::win32::list_processes()+ .into_iter()+ .find(|process| process.pid == pid)+ .is_some_and(|process| is_chromium_browser_target(&process.name))+}+/// Anti-throttling flags injected on hidden Chromium launches (#1620).////// `launch_app` uses `SW_SHOWNOACTIVATE` so the launched window isdiff --git a/scripts/ci/run-rust-standalone-browser-e2e.sh b/scripts/ci/run-rust-standalone-browser-e2e.shindex cf2dca15dc..ddaa128434 100755--- a/scripts/ci/run-rust-standalone-browser-e2e.sh+++ b/scripts/ci/run-rust-standalone-browser-e2e.sh@@ -108,6 +108,7 @@ elsetests=(standalone_browser_background_typestandalone_browser_type_replace+ standalone_browser_owned_permission_promptstandalone_browser_dialogs)if [[ "${HOST_OS}" == Linux ]]; thendiff --git a/scripts/ci/windows/run-rust-standalone-browser-e2e.ps1 b/scripts/ci/windows/run-rust-standalone-browser-e2e.ps1index ad0008f15d..cf26ab1713 100644--- a/scripts/ci/windows/run-rust-standalone-browser-e2e.ps1+++ b/scripts/ci/windows/run-rust-standalone-browser-e2e.ps1@@ -120,6 +120,7 @@ if (-not (Test-Path $env:CUA_TEST_DRIVER_BIN)) {$tests = @("standalone_browser_background_type","standalone_browser_type_replace",+ "standalone_browser_owned_permission_prompt","standalone_browser_dialogs","standalone_browser_download","standalone_browser_existing_profile",