Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
feat(cua-driver): capture inactive tabs and retain modal controls (#2426)
Francesco Bonaccic4d7ddc5bc7c
diff --git a/docs/content/docs/reference/cua-driver/mcp-tools.mdx b/docs/content/docs/reference/cua-driver/mcp-tools.mdxindex b05c08555d..a5aa30807a 100644--- a/docs/content/docs/reference/cua-driver/mcp-tools.mdx+++ b/docs/content/docs/reference/cua-driver/mcp-tools.mdx@@ -762,6 +762,7 @@ Read-only browser inspection. Mode 1 (bind): pass pid + window_id of a native br**Arguments:**- `continuation` (string, optional): Opaque continuation minted by an earlier semantic_v2 response.+- `include_screenshot` (boolean, optional): Capture the exact tab viewport as PNG through CDP without selecting the tab or foregrounding its native window. The request refuses if capture cannot be completed.- `pid` (integer, optional): Native browser process id (bind mode).- `query` (string, optional): Read-only semantic match over role, accessible name, and visible text.- `scope_ref` (string, optional): Current semantic/content ref whose subtree should be observed.diff --git a/libs/cua-driver/rust/Cargo.lock b/libs/cua-driver/rust/Cargo.lockindex 77eb00b3dd..61235438c5 100644--- a/libs/cua-driver/rust/Cargo.lock+++ b/libs/cua-driver/rust/Cargo.lock@@ -918,6 +918,7 @@ version = "0.10.0"dependencies = ["anyhow","async-trait",+ "base64","cua-driver-contract","futures-util","image",diff --git a/libs/cua-driver/rust/Skills/cua-driver/BROWSER.md b/libs/cua-driver/rust/Skills/cua-driver/BROWSER.mdindex 737ec7fb4b..78f24d82b4 100644--- a/libs/cua-driver/rust/Skills/cua-driver/BROWSER.md+++ b/libs/cua-driver/rust/Skills/cua-driver/BROWSER.md@@ -137,6 +137,30 @@ cua-driver get_browser_state \"session":"browser-run-1","snapshot_format":"semantic_v2"}'```+Set `include_screenshot:true` when the visual state matters, including when the+exact tab is open but unselected:++```bash+cua-driver get_browser_state \+ '{"target_id":"<target>","tab_id":"<tab>",+ "session":"browser-run-1","snapshot_format":"semantic_v2",+ "include_screenshot":true}'+```++The result includes a PNG image part, the flat compatibility fields+`screenshot_width`, `screenshot_height`, and `screenshot_mime_type`, plus a+structured `screenshot` object. That object identifies the coordinate space as+`viewport_css_px` and reports `viewport_css_width`, `viewport_css_height`,+`pixel_to_css_scale_x`, and `pixel_to_css_scale_y`. When grounding a coordinate+action from the PNG, convert image pixels to the browser action space with+`css_x = png_x * pixel_to_css_scale_x` and+`css_y = png_y * pixel_to_css_scale_y`; do not assume device scale factor 1.++Cua Driver captures the exact tab viewport through CDP. It does not select the+tab or foreground the browser window. Capture is opt-in because authenticated+pages may contain sensitive information, and a requested capture refuses when+the driver cannot return valid viewport metrics and a valid bounded PNG.+`semantic_v2` composes the page accessibility tree, pierced DOM, layout, andviewport state. Read the compact `outline` for page content, use `refs` onlyfor actions declared in each entry's `actions` array, and use `content_refs`diff --git a/libs/cua-driver/rust/crates/cua-driver-core/Cargo.toml b/libs/cua-driver/rust/crates/cua-driver-core/Cargo.tomlindex 2fa6f159a0..8f1430d0b0 100644--- a/libs/cua-driver/rust/crates/cua-driver-core/Cargo.toml+++ b/libs/cua-driver/rust/crates/cua-driver-core/Cargo.toml@@ -31,6 +31,7 @@ image = { workspace = true }tokio-tungstenite = "0.24"futures-util = "0.3"uuid = { workspace = true }+base64 = { workspace = true }sha2 = "0.10"url = "2.5"diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/browser/engine.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/browser/engine.rsindex 17ec8baf00..e01d45237e 100644--- a/libs/cua-driver/rust/crates/cua-driver-core/src/browser/engine.rs+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/browser/engine.rs@@ -26,6 +26,7 @@use std::collections::{HashMap, HashSet};use std::sync::{Arc, Weak};+use base64::{engine::general_purpose::STANDARD as BASE64, Engine as _};use serde_json::{json, Value};use uuid::Uuid;@@ -63,6 +64,9 @@ pub const BOUNDS_TOLERANCE_PX: f64 = 8.0;/// Cap on refs minted per snapshot — keeps snapshots bounded on/// pathological pages. The truncation is reported in the tool output.pub const MAX_REFS_PER_SNAPSHOT: usize = 300;+/// Hard cap for decoded tab screenshots returned through MCP. This bounds a+/// compromised or malformed endpoint before its response reaches consumers.+const MAX_BROWSER_SCREENSHOT_BYTES: usize = 16 * 1024 * 1024;pub struct BrowserEngine {pub(crate) platform: Arc<dyn BrowserPlatform>,@@ -282,6 +286,72 @@ pub(crate) struct SemanticSnapshotOutcome {pub oopif: OopifStatus,}+pub(crate) struct BrowserTabScreenshot {+ pub data_base64: String,+ pub width: u32,+ pub height: u32,+ pub viewport_css_width: f64,+ pub viewport_css_height: f64,+ pub pixel_to_css_scale_x: f64,+ pub pixel_to_css_scale_y: f64,+}++#[derive(Debug, Clone, Copy)]+struct BrowserScreenshotViewport {+ page_x: f64,+ page_y: f64,+ width: f64,+ height: f64,+}++fn browser_screenshot_viewport(+ metrics: &Value,+) -> Result<BrowserScreenshotViewport, BrowserRefusal> {+ let viewport = metrics+ .get("cssVisualViewport")+ .or_else(|| metrics.get("visualViewport"))+ .ok_or_else(|| {+ route_err(+ "Page.getLayoutMetrics returned malformed data",+ "missing visual viewport metrics",+ )+ })?;+ let number = |field: &str| {+ viewport+ .get(field)+ .and_then(Value::as_f64)+ .filter(|value| value.is_finite())+ .ok_or_else(|| {+ route_err(+ "Page.getLayoutMetrics returned malformed data",+ format!("missing or non-finite visual viewport field {field}"),+ )+ })+ };+ let page_x = number("pageX")?;+ let page_y = number("pageY")?;+ let width = number("clientWidth")?;+ let height = number("clientHeight")?;+ if page_x < 0.0 || page_y < 0.0 {+ return Err(route_err(+ "Page.getLayoutMetrics returned malformed data",+ "visual viewport page offsets must be non-negative",+ ));+ }+ if width <= 0.0 || height <= 0.0 {+ return Err(route_err(+ "Page.getLayoutMetrics returned malformed data",+ "visual viewport dimensions must be positive",+ ));+ }+ Ok(BrowserScreenshotViewport {+ page_x,+ page_y,+ width,+ height,+ })+}+/// Whether OOPIF content could be composed into the snapshot.#[derive(Debug, Clone, Copy, PartialEq, Eq)]pub(crate) enum OopifStatus {@@ -1478,6 +1548,103 @@ impl BrowserEngine {// ── Read-side: page snapshot (ref minting) ──────────────────────────+ /// Capture the exact page target's current viewport through its attached+ /// CDP session. This route never calls `Target.activateTarget`,+ /// `Page.bringToFront`, or a native foreground API, so an already-open+ /// inactive tab stays inactive.+ pub(crate) async fn capture_tab_screenshot(+ &self,+ session: &str,+ target_id: &str,+ tab_id: &str,+ ) -> Result<BrowserTabScreenshot, BrowserRefusal> {+ let record = self.store.get_target(session, target_id)?;+ let tab = record.tabs.get(tab_id).cloned().ok_or_else(|| {+ refuse(+ BrowserRefusalCode::BrowserTabNotFound,+ format!("tab {tab_id} is not known for target {target_id}"),+ )+ })?;+ let conn = self.connection_for_record(session, &record).await?;+ let cdp_session = self.attach(&conn, &tab.cdp_target_id).await?;+ let metrics = conn+ .call(Some(&cdp_session), "Page.getLayoutMetrics", json!({}))+ .await+ .map_err(|error| route_err("Page.getLayoutMetrics failed", error))?;+ let viewport = browser_screenshot_viewport(&metrics)?;+ let response = conn+ .call(+ Some(&cdp_session),+ "Page.captureScreenshot",+ json!({+ "format": "png",+ "fromSurface": true,+ "captureBeyondViewport": false,+ "clip": {+ "x": viewport.page_x,+ "y": viewport.page_y,+ "width": viewport.width,+ "height": viewport.height,+ "scale": 1.0,+ },+ }),+ )+ .await+ .map_err(|error| route_err("Page.captureScreenshot failed", error))?;+ let data_base64 = response+ .get("data")+ .and_then(Value::as_str)+ .ok_or_else(|| {+ route_err(+ "Page.captureScreenshot returned malformed data",+ "missing base64 PNG payload",+ )+ })?+ .to_owned();+ if data_base64.len() > MAX_BROWSER_SCREENSHOT_BYTES.saturating_mul(2) {+ return Err(route_err(+ "Page.captureScreenshot returned oversized data",+ "encoded payload exceeds the browser screenshot limit",+ ));+ }+ let png = BASE64.decode(&data_base64).map_err(|error| {+ route_err(+ "Page.captureScreenshot returned malformed data",+ format!("base64 decode failed: {error}"),+ )+ })?;+ if png.len() > MAX_BROWSER_SCREENSHOT_BYTES {+ return Err(route_err(+ "Page.captureScreenshot returned oversized data",+ format!(+ "decoded PNG is {} bytes; limit is {MAX_BROWSER_SCREENSHOT_BYTES}",+ png.len()+ ),+ ));+ }+ let (width, height) = crate::image_utils::png_dimensions(&png).map_err(|error| {+ route_err(+ "Page.captureScreenshot returned malformed data",+ format!("invalid PNG: {error}"),+ )+ })?;+ if width == 0 || height == 0 {+ return Err(route_err(+ "Page.captureScreenshot returned malformed data",+ "PNG dimensions must be non-zero",+ ));+ }+ Ok(BrowserTabScreenshot {+ data_base64,+ width,+ height,+ viewport_css_width: viewport.width,+ viewport_css_height: viewport.height,+ pixel_to_css_scale_x: viewport.width / f64::from(width),+ pixel_to_css_scale_y: viewport.height / f64::from(height),+ })+ }+/// Snapshot one tab's composed DOM (main frame + shadow DOM +/// same-process iframes + capability-tested OOPIFs) and mint/// `p<snap>:<index>` refs for interactive elements. Read-onlydiff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/browser/semantic.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/browser/semantic.rsindex 5ac64a75a8..5ff60badca 100644--- a/libs/cua-driver/rust/crates/cua-driver-core/src/browser/semantic.rs+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/browser/semantic.rs@@ -250,6 +250,30 @@ impl SemanticDocument {}}+impl DomIndex {+ fn is_ancestor_of(&self, ancestor: i64, mut descendant: i64) -> bool {+ let mut visited = HashSet::new();+ while visited.insert(descendant) {+ let Some(parent) = self+ .nodes+ .get(&descendant)+ .and_then(|node| node.parent_backend_node_id)+ else {+ return false;+ };+ if parent == ancestor {+ return true;+ }+ descendant = parent;+ }+ false+ }++ fn shares_dom_branch(&self, left: i64, right: i64) -> bool {+ self.is_ancestor_of(left, right) || self.is_ancestor_of(right, left)+ }+}+pub(crate) fn build_dom_index(root: &Value) -> DomIndex {fn walk(node: &Value,@@ -531,7 +555,7 @@ pub(crate) fn compose_accessibility_tree(}supplement_dom_actions(&mut nodes, dom, layout, viewport, &frame);- apply_page_occlusion(&mut nodes, layout);+ apply_page_occlusion(&mut nodes, dom, layout);remove_redundant_static_text(&mut nodes);SemanticDocument {nodes,@@ -541,7 +565,7 @@ pub(crate) fn compose_accessibility_tree(}}-fn apply_page_occlusion(nodes: &mut [SemanticNode], layout: &LayoutIndex) {+fn apply_page_occlusion(nodes: &mut [SemanticNode], dom: &DomIndex, layout: &LayoutIndex) {for node in nodes {if node.visibility != BrowserVisibility::InViewport {continue;@@ -557,6 +581,7 @@ fn apply_page_occlusion(nodes: &mut [SemanticNode], layout: &LayoutIndex) {};let covered = layout.nodes.iter().any(|(&backend, overlay)| {if backend == target_backend+ || dom.shares_dom_branch(backend, target_backend)|| overlay.paint_order.is_none_or(|paint| paint <= target_paint)@@ -1379,6 +1404,69 @@ mod tests {assert_eq!(page.omissions.page_occluded, 1);}+ #[test]+ fn dialog_container_does_not_occlude_its_own_actions() {+ let dom = build_dom_index(&json!({+ "nodeType": 9,+ "frameId": "F_MAIN",+ "children": [{+ "nodeType": 1,+ "nodeName": "DIV",+ "backendNodeId": 10,+ "attributes": ["role", "alertdialog", "aria-label", "Remove app"],+ "children": [+ {"nodeType": 1, "nodeName": "BUTTON", "backendNodeId": 11,+ "attributes": ["aria-label", "Cancel"]},+ {"nodeType": 1, "nodeName": "BUTTON", "backendNodeId": 12,+ "attributes": ["aria-label", "Remove"]}+ ]+ }]+ }));+ let layout = build_layout_index(&json!({+ "strings": ["block", "visible", "1", "auto", "default", "static", "0", "fixed", "100"],+ "documents": [{+ "nodes": {"backendNodeId": [10, 11, 12]},+ "layout": {+ "nodeIndex": [0, 1, 2],+ "bounds": [[0, 0, 800, 600], [250, 400, 120, 36], [390, 400, 120, 36]],+ "styles": [+ [0, 1, 2, 3, 4, 7, 8, 3, 3],+ [0, 1, 2, 3, 4, 5, 6, 3, 3],+ [0, 1, 2, 3, 4, 5, 6, 3, 3]+ ],+ "paintOrders": [100, 10, 11]+ }+ }]+ }));+ let viewport = parse_viewport(&json!({+ "cssVisualViewport": {"pageX": 0.0, "pageY": 0.0,+ "clientWidth": 800.0, "clientHeight": 600.0}+ }));+ let ax = json!({"nodes": [+ {"nodeId": "root", "ignored": false, "role": {"value": "RootWebArea"},+ "childIds": ["dialog"]},+ {"nodeId": "dialog", "parentId": "root", "ignored": false,+ "backendDOMNodeId": 10, "role": {"value": "alertdialog"},+ "name": {"value": "Remove app"}, "childIds": ["cancel", "remove"]},+ {"nodeId": "cancel", "parentId": "dialog", "ignored": false,+ "backendDOMNodeId": 11, "role": {"value": "button"},+ "name": {"value": "Cancel"}, "childIds": []},+ {"nodeId": "remove", "parentId": "dialog", "ignored": false,+ "backendDOMNodeId": 12, "role": {"value": "button"},+ "name": {"value": "Remove"}, "childIds": []}+ ]});+ let document = compose_accessibility_tree(&ax, &dom, &layout, &viewport, frame());+ let page = document.page(0, 300, None, None);+ let actions = page+ .selected+ .iter()+ .filter(|node| !node.actions.is_empty())+ .filter_map(|node| node.name.as_deref())+ .collect::<Vec<_>>();+ assert_eq!(actions, vec!["Cancel", "Remove"]);+ assert_eq!(page.omissions.page_occluded, 0);+ }+#[test]fn semantic_text_removes_icon_glyphs_and_nonbreaking_spaces() {assert_eq!(diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/browser/tools.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/browser/tools.rsindex 7318cbddf4..d15dc260bb 100644--- a/libs/cua-driver/rust/crates/cua-driver-core/src/browser/tools.rs+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/browser/tools.rs@@ -10,13 +10,13 @@ use std::sync::Arc;use async_trait::async_trait;use serde_json::{json, Value};-use crate::protocol::ToolResult;+use crate::protocol::{Content, ToolResult};use crate::tool::{Tool, ToolDef, ToolRegistry};use crate::tool_args::ArgsExt;use super::approval::MCP_HOST_APPROVAL_ARG;use super::download::BrowserDownloadTool;-use super::engine::BrowserEngine;+use super::engine::{BrowserEngine, BrowserTabScreenshot};use super::platform::{PrepareAuthorization, PrepareProfile, PrepareRequest, PrepareStrategy};use super::pointer::BrowserPointerTool;use super::refusal::{BrowserRefusal, BrowserRefusalCode};@@ -104,6 +104,32 @@ fn semantic_ref_value(listed: &super::engine::SemanticListedRef) -> Value {})}+fn with_tab_screenshot(mut result: ToolResult, screenshot: BrowserTabScreenshot) -> ToolResult {+ if let Some(structured) = result.structured_content.as_mut() {+ structured["screenshot"] = json!({+ "source": "cdp_tab",+ "scope": "viewport",+ "mime_type": "image/png",+ "width": screenshot.width,+ "height": screenshot.height,+ "coordinate_space": "viewport_css_px",+ "viewport_css_width": screenshot.viewport_css_width,+ "viewport_css_height": screenshot.viewport_css_height,+ "pixel_to_css_scale_x": screenshot.pixel_to_css_scale_x,+ "pixel_to_css_scale_y": screenshot.pixel_to_css_scale_y,+ "tab_activation": "not_requested",+ "window_foregrounding": "not_requested",+ });+ structured["screenshot_width"] = json!(screenshot.width);+ structured["screenshot_height"] = json!(screenshot.height);+ structured["screenshot_mime_type"] = json!("image/png");+ }+ result+ .content+ .insert(0, Content::image_png(screenshot.data_base64));+ result+}+// ── get_browser_state ────────────────────────────────────────────────────────pub struct GetBrowserStateTool {@@ -152,6 +178,11 @@ impl GetBrowserStateTool {"type": "string","description": "Opaque continuation minted by an earlier semantic_v2 response."},+ "include_screenshot": {+ "type": "boolean",+ "default": false,+ "description": "Capture the exact tab viewport as PNG through CDP without selecting the tab or foregrounding its native window. The request refuses if capture cannot be completed."+ },},"additionalProperties": true}),@@ -190,6 +221,15 @@ impl Tool for GetBrowserStateTool {let snapshot_format = args.opt_str("snapshot_format").unwrap_or_else(|| "dom_refs_v1".into());+ let include_screenshot = match args.get("include_screenshot") {+ None => false,+ Some(Value::Bool(include)) => *include,+ Some(_) => {+ return ToolResult::error(+ "Field include_screenshot has wrong type: expected boolean",+ )+ }+ };if snapshot_format != "dom_refs_v1" && snapshot_format != "semantic_v2" {return ToolResult::error(format!("snapshot_format must be \"dom_refs_v1\" or \"semantic_v2\", got {snapshot_format:?}"@@ -205,7 +245,7 @@ impl Tool for GetBrowserStateTool {);}if snapshot_format == "semantic_v2" {- return match self+ let snapshot = match self.engine.snapshot_tab_semantic(&session,@@ -272,10 +312,21 @@ impl Tool for GetBrowserStateTool {},}))}- Err(refusal) => refusal.to_tool_result(),+ Err(refusal) => return refusal.to_tool_result(),};+ if include_screenshot {+ return match self+ .engine+ .capture_tab_screenshot(&session, &target_id, &tab_id)+ .await+ {+ Ok(screenshot) => with_tab_screenshot(snapshot, screenshot),+ Err(refusal) => refusal.to_tool_result(),+ };+ }+ return snapshot;}- return match self+ let snapshot = match self.engine.snapshot_tab(&session, &target_id, &tab_id).await@@ -314,8 +365,19 @@ impl Tool for GetBrowserStateTool {},}))}- Err(refusal) => refusal.to_tool_result(),+ Err(refusal) => return refusal.to_tool_result(),};+ if include_screenshot {+ return match self+ .engine+ .capture_tab_screenshot(&session, &target_id, &tab_id)+ .await+ {+ Ok(screenshot) => with_tab_screenshot(snapshot, screenshot),+ Err(refusal) => refusal.to_tool_result(),+ };+ }+ return snapshot;}// Bind mode: pid + window_id.@@ -1936,6 +1998,10 @@ mod tests {"get_browser_state must be strictly read-only");assert!(state.def().idempotent);+ assert_eq!(+ state.def().input_schema["properties"]["include_screenshot"]["default"],+ false+ );let prepare = BrowserPrepareTool::new(e.clone());assert!(prepare.def().destructive);@@ -2055,6 +2121,23 @@ mod tests {}}+ #[tokio::test]+ async fn snapshot_rejects_non_boolean_include_screenshot() {+ let result = GetBrowserStateTool::new(engine())+ .invoke(json!({+ "target_id": "bt-fixture",+ "tab_id": "tab-fixture",+ "session": "browser-run",+ "include_screenshot": "yes"+ }))+ .await;+ assert_eq!(result.is_error, Some(true));+ assert!(matches!(+ &result.content[0],+ Content::Text { text, .. } if text.contains("expected boolean")+ ));+ }+#[tokio::test]async fn public_session_field_is_the_primary_capability_namespace() {let tool = GetBrowserStateTool::new(engine());diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/browser/v2_tests.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/browser/v2_tests.rsindex 49ba71c349..f58156473b 100644--- a/libs/cua-driver/rust/crates/cua-driver-core/src/browser/v2_tests.rs+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/browser/v2_tests.rs@@ -4,15 +4,18 @@//! revalidation, navigation invalidation, and unproven-capability//! omission/refusal.+use std::io::Cursor;use std::sync::{atomic::{AtomicBool, Ordering},Arc, Mutex as StdMutex,};use async_trait::async_trait;+use base64::{engine::general_purpose::STANDARD as BASE64, Engine as _};+use image::{DynamicImage, ImageFormat, Rgba, RgbaImage};use serde_json::{json, Value};-use crate::protocol::ToolResult;+use crate::protocol::{Content, ToolResult};use crate::tool::Tool;use super::engine::BrowserEngine;@@ -53,6 +56,9 @@ struct FixtureState {semantic_full_dom_fails: bool,semantic_full_dom_times_out: bool,semantic_truncated_dom: bool,+ screenshot_data: String,+ viewport_css_width: f64,+ viewport_css_height: f64,/// Every incoming CDP call: (sessionId, method, params).calls: Vec<(Option<String>, String, Value)>,}@@ -74,11 +80,23 @@ impl Default for FixtureState {semantic_full_dom_fails: false,semantic_full_dom_times_out: false,semantic_truncated_dom: false,+ screenshot_data: "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAusB9Y9ZJrAAAAAASUVORK5CYII=".into(),+ viewport_css_width: 800.0,+ viewport_css_height: 600.0,calls: Vec::new(),}}}+fn screenshot_png_base64(width: u32, height: u32) -> String {+ let image = RgbaImage::from_pixel(width, height, Rgba([18, 171, 52, 255]));+ let mut encoded = Cursor::new(Vec::new());+ DynamicImage::ImageRgba8(image)+ .write_to(&mut encoded, ImageFormat::Png)+ .expect("encode screenshot fixture PNG");+ BASE64.encode(encoded.into_inner())+}+type SharedState = Arc<StdMutex<FixtureState>>;/// Main-frame document: a plain button, an open shadow root with an@@ -509,10 +527,13 @@ fn fixture_handler(state: SharedState) -> MockHandler {"cssVisualViewport": {"pageX": 0.0,"pageY": 0.0,- "clientWidth": 800.0,- "clientHeight": 600.0+ "clientWidth": st.viewport_css_width,+ "clientHeight": st.viewport_css_height}})),+ "Page.captureScreenshot" if is_tab => {+ MockReply::ok(json!({"data": st.screenshot_data.clone()}))+ }"Page.navigate" if is_tab => MockReply::ok(json!({"frameId": "F_MAIN","loaderId": "L_MAIN_NAVIGATED",@@ -1345,6 +1366,146 @@ async fn semantic_snapshot_keeps_visible_content_after_hidden_node_pressure() {assert_eq!(snap["snapshot"]["omitted"]["css_hidden"], 320);}+#[tokio::test]+async fn semantic_snapshot_can_capture_an_inactive_tab_without_activation_calls() {+ let f = fixture().await;+ let (target, tab) = bind(&f).await;+ let result = GetBrowserStateTool::new(f.engine.clone())+ .invoke(json!({+ "target_id": target,+ "tab_id": tab,+ "session": SESSION,+ "snapshot_format": "semantic_v2",+ "include_screenshot": true+ }))+ .await;+ let snapshot = structured(&result);+ assert_eq!(snapshot["status"], "ok", "{snapshot}");+ assert_eq!(snapshot["screenshot"]["mime_type"], "image/png");+ assert_eq!(snapshot["screenshot"]["width"], 1);+ assert_eq!(snapshot["screenshot"]["height"], 1);+ assert_eq!(snapshot["screenshot"]["source"], "cdp_tab");+ assert_eq!(snapshot["screenshot_width"], 1);+ assert_eq!(snapshot["screenshot_height"], 1);+ assert_eq!(snapshot["screenshot_mime_type"], "image/png");+ assert_eq!(+ snapshot["screenshot"]["coordinate_space"],+ "viewport_css_px"+ );+ assert_eq!(snapshot["screenshot"]["viewport_css_width"], 800.0);+ assert_eq!(snapshot["screenshot"]["viewport_css_height"], 600.0);+ assert_eq!(snapshot["screenshot"]["pixel_to_css_scale_x"], 800.0);+ assert_eq!(snapshot["screenshot"]["pixel_to_css_scale_y"], 600.0);+ assert!(result.content.iter().any(|content| matches!(+ content,+ Content::Image { mime_type, .. } if mime_type == "image/png"+ )));++ let state = f.state.lock().unwrap();+ assert!(state.calls.iter().any(|(_, method, params)| {+ method == "Page.captureScreenshot"+ && params["format"] == "png"+ && params["fromSurface"] == true+ && params["captureBeyondViewport"] == false+ && params["clip"]["x"] == 0.0+ && params["clip"]["y"] == 0.0+ && params["clip"]["width"] == 800.0+ && params["clip"]["height"] == 600.0+ && params["clip"]["scale"] == 1.0+ }));+ assert!(state.calls.iter().all(|(_, method, _)| {+ method != "Target.activateTarget" && method != "Page.bringToFront"+ }));+}++#[tokio::test]+async fn semantic_snapshot_does_not_capture_unless_requested() {+ let f = fixture().await;+ let (target, tab) = bind(&f).await;+ let snapshot = semantic_snapshot(&f, &target, &tab).await;+ assert_eq!(snapshot["status"], "ok", "{snapshot}");+ assert_eq!(snapshot["screenshot"], Value::Null);+ assert_eq!(snapshot["screenshot_width"], Value::Null);+ assert_eq!(snapshot["screenshot_height"], Value::Null);+ assert_eq!(snapshot["screenshot_mime_type"], Value::Null);+ assert!(recorded_calls(&f, "Page.captureScreenshot").is_empty());+}++#[tokio::test]+async fn requested_tab_screenshot_maps_non_unit_png_pixels_to_viewport_css() {+ let f = fixture_with(|state| {+ state.viewport_css_width = 2.0;+ state.viewport_css_height = 1.0;+ state.screenshot_data = screenshot_png_base64(4, 2);+ })+ .await;+ let (target, tab) = bind(&f).await;+ let result = GetBrowserStateTool::new(f.engine.clone())+ .invoke(json!({+ "target_id": target,+ "tab_id": tab,+ "session": SESSION,+ "snapshot_format": "dom_refs_v1",+ "include_screenshot": true+ }))+ .await;+ let snapshot = structured(&result);+ assert_eq!(snapshot["status"], "ok", "{snapshot}");+ assert_eq!(snapshot["screenshot_width"], 4);+ assert_eq!(snapshot["screenshot_height"], 2);+ assert_eq!(snapshot["screenshot_mime_type"], "image/png");+ assert_eq!(snapshot["screenshot"]["viewport_css_width"], 2.0);+ assert_eq!(snapshot["screenshot"]["viewport_css_height"], 1.0);+ assert_eq!(snapshot["screenshot"]["pixel_to_css_scale_x"], 0.5);+ assert_eq!(snapshot["screenshot"]["pixel_to_css_scale_y"], 0.5);+ let capture = recorded_calls(&f, "Page.captureScreenshot");+ assert_eq!(capture.len(), 1, "{capture:?}");+ assert_eq!(capture[0].1["clip"]["width"], 2.0);+ assert_eq!(capture[0].1["clip"]["height"], 1.0);+ assert_eq!(capture[0].1["clip"]["scale"], 1.0);+}++#[tokio::test]+async fn requested_tab_screenshot_refuses_invalid_viewport_metrics_before_capture() {+ let f = fixture_with(|state| state.viewport_css_width = 0.0).await;+ let (target, tab) = bind(&f).await;+ let result = GetBrowserStateTool::new(f.engine.clone())+ .invoke(json!({+ "target_id": target,+ "tab_id": tab,+ "session": SESSION,+ "snapshot_format": "dom_refs_v1",+ "include_screenshot": true+ }))+ .await;+ let refusal = structured(&result);+ assert_eq!(refusal["status"], "refused", "{refusal}");+ assert_eq!(refusal["refusal"]["code"], "browser_route_unavailable");+ assert!(recorded_calls(&f, "Page.captureScreenshot").is_empty());+}++#[tokio::test]+async fn requested_tab_screenshot_refuses_malformed_image_data() {+ let f = fixture_with(|state| state.screenshot_data = "not-base64".into()).await;+ let (target, tab) = bind(&f).await;+ let result = GetBrowserStateTool::new(f.engine.clone())+ .invoke(json!({+ "target_id": target,+ "tab_id": tab,+ "session": SESSION,+ "snapshot_format": "semantic_v2",+ "include_screenshot": true+ }))+ .await;+ let refusal = structured(&result);+ assert_eq!(refusal["status"], "refused", "{refusal}");+ assert_eq!(refusal["refusal"]["code"], "browser_route_unavailable");+ assert!(result+ .content+ .iter()+ .all(|content| !matches!(content, Content::Image { .. })));+}+#[tokio::test]async fn semantic_continuation_is_opaque_single_use_and_reaches_offscreen_content() {let f = fixture_with(|st| st.semantic_large_page = true).await;diff --git a/libs/cua-driver/rust/crates/cua-driver/tests/protocol_schema_test.rs b/libs/cua-driver/rust/crates/cua-driver/tests/protocol_schema_test.rsindex 5b47d0ba9f..769fe15ecb 100644--- a/libs/cua-driver/rust/crates/cua-driver/tests/protocol_schema_test.rs+++ b/libs/cua-driver/rust/crates/cua-driver/tests/protocol_schema_test.rs@@ -76,6 +76,10 @@ fn tools_list_schema_shape() {"get_browser_state schema missing {field}");}+ assert_eq!(+ browser_state["inputSchema"]["properties"]["include_screenshot"]["default"], false,+ "get_browser_state should advertise opt-in exact-tab capture"+ );for (name, required) in [("browser_prepare", &["pid"][..]),("browser_navigate", &["target_id", "tab_id", "url"][..]),diff --git a/libs/cua-driver/rust/crates/cua-driver/tests/standalone_browser_behavior_test.rs b/libs/cua-driver/rust/crates/cua-driver/tests/standalone_browser_behavior_test.rsindex 4dc69d7fa0..e0acd99491 100644--- a/libs/cua-driver/rust/crates/cua-driver/tests/standalone_browser_behavior_test.rs+++ b/libs/cua-driver/rust/crates/cua-driver/tests/standalone_browser_behavior_test.rs@@ -14,6 +14,7 @@ use std::sync::Mutex;use std::thread;use std::time::{Duration, Instant};+use base64::Engine as _;use cua_driver_testkit::e2e::{append_json_line, execute_case, recording_evidence, write_environment_from_env, CaseSpec,Delivery, DriverRoute, EnvironmentRecord, Evidence, Observation, OracleKind, RefusalCode,@@ -615,31 +616,28 @@ fn cdp_target_for_url(port: u16, url: &str) -> String {.to_owned()}-fn bring_harness_page_to_front(port: u16, url: &str) {+fn cdp_page_websocket_for_url(port: u16, url: &str) -> String {let targets = browser_http_json(port, "/json/list").unwrap_or_else(|error| panic!("read harness CDP targets: {error}"));- let ws_url = targets+ targets.as_array().into_iter().flatten().find(|target| target["type"] == "page" && target["url"] == url).and_then(|target| target["webSocketDebuggerUrl"].as_str())- .unwrap_or_else(|| panic!("no harness page websocket for {url}"));- harness_cdp_call_at_url(ws_url, "Page.bringToFront", serde_json::json!({}));+ .unwrap_or_else(|| panic!("no harness page websocket for {url}"))+ .to_owned()+}++fn bring_harness_page_to_front(port: u16, url: &str) {+ let ws_url = cdp_page_websocket_for_url(port, url);+ harness_cdp_call_at_url(&ws_url, "Page.bringToFront", serde_json::json!({}));}fn harness_page_visibility(port: u16, url: &str) -> String {- let targets = browser_http_json(port, "/json/list")- .unwrap_or_else(|error| panic!("read harness CDP targets: {error}"));- let ws_url = targets- .as_array()- .into_iter()- .flatten()- .find(|target| target["type"] == "page" && target["url"] == url)- .and_then(|target| target["webSocketDebuggerUrl"].as_str())- .unwrap_or_else(|| panic!("no harness page websocket for {url}"));+ let ws_url = cdp_page_websocket_for_url(port, url);harness_cdp_call_at_url(- ws_url,+ &ws_url,"Runtime.evaluate",serde_json::json!({"expression": "document.visibilityState",@@ -2305,10 +2303,15 @@ fn run_multi_tab(spec: &BrowserSpec) {*evidence = recording_evidence(fixture.driver.recording_dir());let session = format!("standalone-multi-tab-{}", fixture.pid);let _ = bind(&mut fixture, &session);- let second_html = standalone_fixture_html().replace(- "<title>cua-driver Web Harness</title>",- "<title>cua-driver Background Harness</title>",- );+ let second_html = standalone_fixture_html()+ .replace(+ "<title>cua-driver Web Harness</title>",+ "<title>cua-driver Background Harness</title>",+ )+ .replace(+ "</style>",+ "body { background: rgb(18, 171, 52) !important; }</style>",+ );let second_server = BrowserFixtureServer::start(&second_html);let created = harness_cdp_call(fixture.cdp_port,@@ -2322,6 +2325,33 @@ fn run_multi_tab(spec: &BrowserSpec) {assert!(created["targetId"].is_string(), "{created}");wait_for_observed(&second_server, "WEB_HARNESS_MARKER_v1");+ // Exercise screenshot/coordinate parity under a non-1 device scale.+ // This is setup instrumentation and runs before the background sentinel;+ // the driver action below must still leave the tab selected state and+ // native foreground unchanged.+ let background_ws = cdp_page_websocket_for_url(fixture.cdp_port, &second_server.page_url());+ harness_cdp_call_at_url(+ &background_ws,+ "Emulation.setDeviceMetricsOverride",+ serde_json::json!({+ "width": 400,+ "height": 300,+ "deviceScaleFactor": 2,+ "mobile": false,+ }),+ );+ let device_scale = harness_cdp_call_at_url(+ &background_ws,+ "Runtime.evaluate",+ serde_json::json!({+ "expression": "window.devicePixelRatio",+ "returnByValue": true,+ }),+ )["result"]["value"]+ .as_f64()+ .expect("background tab device scale factor");+ assert!((device_scale - 2.0).abs() < f64::EPSILON, "{device_scale}");+// Establish a deterministic selected-tab baseline before the// foreground sentinel starts. Chromium does not consistently honor// createTarget(background=true) on every host, but Page.bringToFront@@ -2409,9 +2439,104 @@ fn run_multi_tab(spec: &BrowserSpec) {"session": session,"snapshot_format": "semantic_v2","query": "Increment txt-input",+ "include_screenshot": true,}),);assert_eq!(snapshot.structured()["status"], "ok", "{}", snapshot.raw);+ assert_eq!(+ snapshot.structured()["screenshot"]["source"],+ "cdp_tab",+ "{}",+ snapshot.raw+ );+ assert!(+ snapshot.structured()["screenshot"]["width"]+ .as_u64()+ .is_some_and(|width| width > 0)+ && snapshot.structured()["screenshot"]["height"]+ .as_u64()+ .is_some_and(|height| height > 0),+ "{}",+ snapshot.raw+ );+ assert_eq!(+ snapshot.structured()["screenshot_width"],+ snapshot.structured()["screenshot"]["width"]+ );+ assert_eq!(+ snapshot.structured()["screenshot_height"],+ snapshot.structured()["screenshot"]["height"]+ );+ assert_eq!(snapshot.structured()["screenshot_mime_type"], "image/png");+ assert_eq!(+ snapshot.structured()["screenshot"]["coordinate_space"],+ "viewport_css_px"+ );+ let png_width = snapshot.structured()["screenshot"]["width"]+ .as_f64()+ .expect("PNG width");+ let png_height = snapshot.structured()["screenshot"]["height"]+ .as_f64()+ .expect("PNG height");+ let viewport_width = snapshot.structured()["screenshot"]["viewport_css_width"]+ .as_f64()+ .expect("CSS viewport width");+ let viewport_height = snapshot.structured()["screenshot"]["viewport_css_height"]+ .as_f64()+ .expect("CSS viewport height");+ let pixel_to_css_x = snapshot.structured()["screenshot"]["pixel_to_css_scale_x"]+ .as_f64()+ .expect("screenshot x scale");+ let pixel_to_css_y = snapshot.structured()["screenshot"]["pixel_to_css_scale_y"]+ .as_f64()+ .expect("screenshot y scale");+ assert!((viewport_width - 400.0).abs() < 0.01, "{}", snapshot.raw);+ assert!((viewport_height - 300.0).abs() < 0.01, "{}", snapshot.raw);+ assert!(+ (pixel_to_css_x - viewport_width / png_width).abs() < 1e-9,+ "{}",+ snapshot.raw+ );+ assert!(+ (pixel_to_css_y - viewport_height / png_height).abs() < 1e-9,+ "{}",+ snapshot.raw+ );+ assert!(+ snapshot.raw["result"]["content"]+ .as_array()+ .is_some_and(|content| content.iter().any(|item| {+ item["type"] == "image"+ && item["mimeType"] == "image/png"+ && item["data"]+ .as_str()+ .is_some_and(|data| data.starts_with("iVBOR"))+ })),+ "{}",+ snapshot.raw+ );+ let screenshot_data = snapshot.raw["result"]["content"]+ .as_array()+ .and_then(|content| {+ content+ .iter()+ .find(|item| item["type"] == "image")+ .and_then(|item| item["data"].as_str())+ })+ .expect("inactive-tab PNG data");+ let screenshot_bytes = base64::engine::general_purpose::STANDARD+ .decode(screenshot_data)+ .expect("decode inactive-tab PNG");+ let screenshot = image::load_from_memory(&screenshot_bytes)+ .expect("decode inactive-tab image")+ .to_rgba8();+ let pixel = screenshot.get_pixel(2, 2).0;+ assert!(+ pixel[0].abs_diff(18) <= 2+ && pixel[1].abs_diff(171) <= 2+ && pixel[2].abs_diff(52) <= 2,+ "screenshot did not come from the green inactive tab: rgba={pixel:?}"+ );let click_ref = semantic_ref_by_name(&snapshot, "Increment", "click");let trusted = fixture.driver.call("browser_click",