Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

feat(cua-driver): capture inactive tabs and retain modal controls (#2426)

Francesco Bonaccic4d7ddc5bc7c
diff --git a/docs/content/docs/reference/cua-driver/mcp-tools.mdx b/docs/content/docs/reference/cua-driver/mcp-tools.mdx
index b05c08555d..a5aa30807a 100644
--- a/docs/content/docs/reference/cua-driver/mcp-tools.mdx
+++ b/docs/content/docs/reference/cua-driver/mcp-tools.mdx
@@ -762,6 +762,7 @@ Read-only browser inspection. Mode 1 (bind): pass pid + window_id of a native br
**Arguments:**
- `continuation` (string, optional): Opaque continuation minted by an earlier semantic_v2 response.
+- `include_screenshot` (boolean, optional): Capture the exact tab viewport as PNG through CDP without selecting the tab or foregrounding its native window. The request refuses if capture cannot be completed.
- `pid` (integer, optional): Native browser process id (bind mode).
- `query` (string, optional): Read-only semantic match over role, accessible name, and visible text.
- `scope_ref` (string, optional): Current semantic/content ref whose subtree should be observed.
diff --git a/libs/cua-driver/rust/Cargo.lock b/libs/cua-driver/rust/Cargo.lock
index 77eb00b3dd..61235438c5 100644
--- a/libs/cua-driver/rust/Cargo.lock
+++ b/libs/cua-driver/rust/Cargo.lock
@@ -918,6 +918,7 @@ version = "0.10.0"
dependencies = [
"anyhow",
"async-trait",
+ "base64",
"cua-driver-contract",
"futures-util",
"image",
diff --git a/libs/cua-driver/rust/Skills/cua-driver/BROWSER.md b/libs/cua-driver/rust/Skills/cua-driver/BROWSER.md
index 737ec7fb4b..78f24d82b4 100644
--- a/libs/cua-driver/rust/Skills/cua-driver/BROWSER.md
+++ b/libs/cua-driver/rust/Skills/cua-driver/BROWSER.md
@@ -137,6 +137,30 @@ cua-driver get_browser_state \
"session":"browser-run-1","snapshot_format":"semantic_v2"}'
```
+Set `include_screenshot:true` when the visual state matters, including when the
+exact tab is open but unselected:
+
+```bash
+cua-driver get_browser_state \
+ '{"target_id":"<target>","tab_id":"<tab>",
+ "session":"browser-run-1","snapshot_format":"semantic_v2",
+ "include_screenshot":true}'
+```
+
+The result includes a PNG image part, the flat compatibility fields
+`screenshot_width`, `screenshot_height`, and `screenshot_mime_type`, plus a
+structured `screenshot` object. That object identifies the coordinate space as
+`viewport_css_px` and reports `viewport_css_width`, `viewport_css_height`,
+`pixel_to_css_scale_x`, and `pixel_to_css_scale_y`. When grounding a coordinate
+action from the PNG, convert image pixels to the browser action space with
+`css_x = png_x * pixel_to_css_scale_x` and
+`css_y = png_y * pixel_to_css_scale_y`; do not assume device scale factor 1.
+
+Cua Driver captures the exact tab viewport through CDP. It does not select the
+tab or foreground the browser window. Capture is opt-in because authenticated
+pages may contain sensitive information, and a requested capture refuses when
+the driver cannot return valid viewport metrics and a valid bounded PNG.
+
`semantic_v2` composes the page accessibility tree, pierced DOM, layout, and
viewport state. Read the compact `outline` for page content, use `refs` only
for actions declared in each entry's `actions` array, and use `content_refs`
diff --git a/libs/cua-driver/rust/crates/cua-driver-core/Cargo.toml b/libs/cua-driver/rust/crates/cua-driver-core/Cargo.toml
index 2fa6f159a0..8f1430d0b0 100644
--- a/libs/cua-driver/rust/crates/cua-driver-core/Cargo.toml
+++ b/libs/cua-driver/rust/crates/cua-driver-core/Cargo.toml
@@ -31,6 +31,7 @@ image = { workspace = true }
tokio-tungstenite = "0.24"
futures-util = "0.3"
uuid = { workspace = true }
+base64 = { workspace = true }
sha2 = "0.10"
url = "2.5"
diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/browser/engine.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/browser/engine.rs
index 17ec8baf00..e01d45237e 100644
--- a/libs/cua-driver/rust/crates/cua-driver-core/src/browser/engine.rs
+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/browser/engine.rs
@@ -26,6 +26,7 @@
use std::collections::{HashMap, HashSet};
use std::sync::{Arc, Weak};
+use base64::{engine::general_purpose::STANDARD as BASE64, Engine as _};
use serde_json::{json, Value};
use uuid::Uuid;
@@ -63,6 +64,9 @@ pub const BOUNDS_TOLERANCE_PX: f64 = 8.0;
/// Cap on refs minted per snapshot — keeps snapshots bounded on
/// pathological pages. The truncation is reported in the tool output.
pub const MAX_REFS_PER_SNAPSHOT: usize = 300;
+/// Hard cap for decoded tab screenshots returned through MCP. This bounds a
+/// compromised or malformed endpoint before its response reaches consumers.
+const MAX_BROWSER_SCREENSHOT_BYTES: usize = 16 * 1024 * 1024;
pub struct BrowserEngine {
pub(crate) platform: Arc<dyn BrowserPlatform>,
@@ -282,6 +286,72 @@ pub(crate) struct SemanticSnapshotOutcome {
pub oopif: OopifStatus,
}
+pub(crate) struct BrowserTabScreenshot {
+ pub data_base64: String,
+ pub width: u32,
+ pub height: u32,
+ pub viewport_css_width: f64,
+ pub viewport_css_height: f64,
+ pub pixel_to_css_scale_x: f64,
+ pub pixel_to_css_scale_y: f64,
+}
+
+#[derive(Debug, Clone, Copy)]
+struct BrowserScreenshotViewport {
+ page_x: f64,
+ page_y: f64,
+ width: f64,
+ height: f64,
+}
+
+fn browser_screenshot_viewport(
+ metrics: &Value,
+) -> Result<BrowserScreenshotViewport, BrowserRefusal> {
+ let viewport = metrics
+ .get("cssVisualViewport")
+ .or_else(|| metrics.get("visualViewport"))
+ .ok_or_else(|| {
+ route_err(
+ "Page.getLayoutMetrics returned malformed data",
+ "missing visual viewport metrics",
+ )
+ })?;
+ let number = |field: &str| {
+ viewport
+ .get(field)
+ .and_then(Value::as_f64)
+ .filter(|value| value.is_finite())
+ .ok_or_else(|| {
+ route_err(
+ "Page.getLayoutMetrics returned malformed data",
+ format!("missing or non-finite visual viewport field {field}"),
+ )
+ })
+ };
+ let page_x = number("pageX")?;
+ let page_y = number("pageY")?;
+ let width = number("clientWidth")?;
+ let height = number("clientHeight")?;
+ if page_x < 0.0 || page_y < 0.0 {
+ return Err(route_err(
+ "Page.getLayoutMetrics returned malformed data",
+ "visual viewport page offsets must be non-negative",
+ ));
+ }
+ if width <= 0.0 || height <= 0.0 {
+ return Err(route_err(
+ "Page.getLayoutMetrics returned malformed data",
+ "visual viewport dimensions must be positive",
+ ));
+ }
+ Ok(BrowserScreenshotViewport {
+ page_x,
+ page_y,
+ width,
+ height,
+ })
+}
+
/// Whether OOPIF content could be composed into the snapshot.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum OopifStatus {
@@ -1478,6 +1548,103 @@ impl BrowserEngine {
// ── Read-side: page snapshot (ref minting) ──────────────────────────
+ /// Capture the exact page target's current viewport through its attached
+ /// CDP session. This route never calls `Target.activateTarget`,
+ /// `Page.bringToFront`, or a native foreground API, so an already-open
+ /// inactive tab stays inactive.
+ pub(crate) async fn capture_tab_screenshot(
+ &self,
+ session: &str,
+ target_id: &str,
+ tab_id: &str,
+ ) -> Result<BrowserTabScreenshot, BrowserRefusal> {
+ let record = self.store.get_target(session, target_id)?;
+ let tab = record.tabs.get(tab_id).cloned().ok_or_else(|| {
+ refuse(
+ BrowserRefusalCode::BrowserTabNotFound,
+ format!("tab {tab_id} is not known for target {target_id}"),
+ )
+ })?;
+ let conn = self.connection_for_record(session, &record).await?;
+ let cdp_session = self.attach(&conn, &tab.cdp_target_id).await?;
+ let metrics = conn
+ .call(Some(&cdp_session), "Page.getLayoutMetrics", json!({}))
+ .await
+ .map_err(|error| route_err("Page.getLayoutMetrics failed", error))?;
+ let viewport = browser_screenshot_viewport(&metrics)?;
+ let response = conn
+ .call(
+ Some(&cdp_session),
+ "Page.captureScreenshot",
+ json!({
+ "format": "png",
+ "fromSurface": true,
+ "captureBeyondViewport": false,
+ "clip": {
+ "x": viewport.page_x,
+ "y": viewport.page_y,
+ "width": viewport.width,
+ "height": viewport.height,
+ "scale": 1.0,
+ },
+ }),
+ )
+ .await
+ .map_err(|error| route_err("Page.captureScreenshot failed", error))?;
+ let data_base64 = response
+ .get("data")
+ .and_then(Value::as_str)
+ .ok_or_else(|| {
+ route_err(
+ "Page.captureScreenshot returned malformed data",
+ "missing base64 PNG payload",
+ )
+ })?
+ .to_owned();
+ if data_base64.len() > MAX_BROWSER_SCREENSHOT_BYTES.saturating_mul(2) {
+ return Err(route_err(
+ "Page.captureScreenshot returned oversized data",
+ "encoded payload exceeds the browser screenshot limit",
+ ));
+ }
+ let png = BASE64.decode(&data_base64).map_err(|error| {
+ route_err(
+ "Page.captureScreenshot returned malformed data",
+ format!("base64 decode failed: {error}"),
+ )
+ })?;
+ if png.len() > MAX_BROWSER_SCREENSHOT_BYTES {
+ return Err(route_err(
+ "Page.captureScreenshot returned oversized data",
+ format!(
+ "decoded PNG is {} bytes; limit is {MAX_BROWSER_SCREENSHOT_BYTES}",
+ png.len()
+ ),
+ ));
+ }
+ let (width, height) = crate::image_utils::png_dimensions(&png).map_err(|error| {
+ route_err(
+ "Page.captureScreenshot returned malformed data",
+ format!("invalid PNG: {error}"),
+ )
+ })?;
+ if width == 0 || height == 0 {
+ return Err(route_err(
+ "Page.captureScreenshot returned malformed data",
+ "PNG dimensions must be non-zero",
+ ));
+ }
+ Ok(BrowserTabScreenshot {
+ data_base64,
+ width,
+ height,
+ viewport_css_width: viewport.width,
+ viewport_css_height: viewport.height,
+ pixel_to_css_scale_x: viewport.width / f64::from(width),
+ pixel_to_css_scale_y: viewport.height / f64::from(height),
+ })
+ }
+
/// Snapshot one tab's composed DOM (main frame + shadow DOM +
/// same-process iframes + capability-tested OOPIFs) and mint
/// `p<snap>:<index>` refs for interactive elements. Read-only
diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/browser/semantic.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/browser/semantic.rs
index 5ac64a75a8..5ff60badca 100644
--- a/libs/cua-driver/rust/crates/cua-driver-core/src/browser/semantic.rs
+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/browser/semantic.rs
@@ -250,6 +250,30 @@ impl SemanticDocument {
}
}
+impl DomIndex {
+ fn is_ancestor_of(&self, ancestor: i64, mut descendant: i64) -> bool {
+ let mut visited = HashSet::new();
+ while visited.insert(descendant) {
+ let Some(parent) = self
+ .nodes
+ .get(&descendant)
+ .and_then(|node| node.parent_backend_node_id)
+ else {
+ return false;
+ };
+ if parent == ancestor {
+ return true;
+ }
+ descendant = parent;
+ }
+ false
+ }
+
+ fn shares_dom_branch(&self, left: i64, right: i64) -> bool {
+ self.is_ancestor_of(left, right) || self.is_ancestor_of(right, left)
+ }
+}
+
pub(crate) fn build_dom_index(root: &Value) -> DomIndex {
fn walk(
node: &Value,
@@ -531,7 +555,7 @@ pub(crate) fn compose_accessibility_tree(
}
supplement_dom_actions(&mut nodes, dom, layout, viewport, &frame);
- apply_page_occlusion(&mut nodes, layout);
+ apply_page_occlusion(&mut nodes, dom, layout);
remove_redundant_static_text(&mut nodes);
SemanticDocument {
nodes,
@@ -541,7 +565,7 @@ pub(crate) fn compose_accessibility_tree(
}
}
-fn apply_page_occlusion(nodes: &mut [SemanticNode], layout: &LayoutIndex) {
+fn apply_page_occlusion(nodes: &mut [SemanticNode], dom: &DomIndex, layout: &LayoutIndex) {
for node in nodes {
if node.visibility != BrowserVisibility::InViewport {
continue;
@@ -557,6 +581,7 @@ fn apply_page_occlusion(nodes: &mut [SemanticNode], layout: &LayoutIndex) {
};
let covered = layout.nodes.iter().any(|(&backend, overlay)| {
if backend == target_backend
+ || dom.shares_dom_branch(backend, target_backend)
|| overlay
.paint_order
.is_none_or(|paint| paint <= target_paint)
@@ -1379,6 +1404,69 @@ mod tests {
assert_eq!(page.omissions.page_occluded, 1);
}
+ #[test]
+ fn dialog_container_does_not_occlude_its_own_actions() {
+ let dom = build_dom_index(&json!({
+ "nodeType": 9,
+ "frameId": "F_MAIN",
+ "children": [{
+ "nodeType": 1,
+ "nodeName": "DIV",
+ "backendNodeId": 10,
+ "attributes": ["role", "alertdialog", "aria-label", "Remove app"],
+ "children": [
+ {"nodeType": 1, "nodeName": "BUTTON", "backendNodeId": 11,
+ "attributes": ["aria-label", "Cancel"]},
+ {"nodeType": 1, "nodeName": "BUTTON", "backendNodeId": 12,
+ "attributes": ["aria-label", "Remove"]}
+ ]
+ }]
+ }));
+ let layout = build_layout_index(&json!({
+ "strings": ["block", "visible", "1", "auto", "default", "static", "0", "fixed", "100"],
+ "documents": [{
+ "nodes": {"backendNodeId": [10, 11, 12]},
+ "layout": {
+ "nodeIndex": [0, 1, 2],
+ "bounds": [[0, 0, 800, 600], [250, 400, 120, 36], [390, 400, 120, 36]],
+ "styles": [
+ [0, 1, 2, 3, 4, 7, 8, 3, 3],
+ [0, 1, 2, 3, 4, 5, 6, 3, 3],
+ [0, 1, 2, 3, 4, 5, 6, 3, 3]
+ ],
+ "paintOrders": [100, 10, 11]
+ }
+ }]
+ }));
+ let viewport = parse_viewport(&json!({
+ "cssVisualViewport": {"pageX": 0.0, "pageY": 0.0,
+ "clientWidth": 800.0, "clientHeight": 600.0}
+ }));
+ let ax = json!({"nodes": [
+ {"nodeId": "root", "ignored": false, "role": {"value": "RootWebArea"},
+ "childIds": ["dialog"]},
+ {"nodeId": "dialog", "parentId": "root", "ignored": false,
+ "backendDOMNodeId": 10, "role": {"value": "alertdialog"},
+ "name": {"value": "Remove app"}, "childIds": ["cancel", "remove"]},
+ {"nodeId": "cancel", "parentId": "dialog", "ignored": false,
+ "backendDOMNodeId": 11, "role": {"value": "button"},
+ "name": {"value": "Cancel"}, "childIds": []},
+ {"nodeId": "remove", "parentId": "dialog", "ignored": false,
+ "backendDOMNodeId": 12, "role": {"value": "button"},
+ "name": {"value": "Remove"}, "childIds": []}
+ ]});
+ let document = compose_accessibility_tree(&ax, &dom, &layout, &viewport, frame());
+ let page = document.page(0, 300, None, None);
+ let actions = page
+ .selected
+ .iter()
+ .filter(|node| !node.actions.is_empty())
+ .filter_map(|node| node.name.as_deref())
+ .collect::<Vec<_>>();
+ assert_eq!(actions, vec!["Cancel", "Remove"]);
+ assert_eq!(page.omissions.page_occluded, 0);
+ }
+
#[test]
fn semantic_text_removes_icon_glyphs_and_nonbreaking_spaces() {
assert_eq!(
diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/browser/tools.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/browser/tools.rs
index 7318cbddf4..d15dc260bb 100644
--- a/libs/cua-driver/rust/crates/cua-driver-core/src/browser/tools.rs
+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/browser/tools.rs
@@ -10,13 +10,13 @@ use std::sync::Arc;
use async_trait::async_trait;
use serde_json::{json, Value};
-use crate::protocol::ToolResult;
+use crate::protocol::{Content, ToolResult};
use crate::tool::{Tool, ToolDef, ToolRegistry};
use crate::tool_args::ArgsExt;
use super::approval::MCP_HOST_APPROVAL_ARG;
use super::download::BrowserDownloadTool;
-use super::engine::BrowserEngine;
+use super::engine::{BrowserEngine, BrowserTabScreenshot};
use super::platform::{PrepareAuthorization, PrepareProfile, PrepareRequest, PrepareStrategy};
use super::pointer::BrowserPointerTool;
use super::refusal::{BrowserRefusal, BrowserRefusalCode};
@@ -104,6 +104,32 @@ fn semantic_ref_value(listed: &super::engine::SemanticListedRef) -> Value {
})
}
+fn with_tab_screenshot(mut result: ToolResult, screenshot: BrowserTabScreenshot) -> ToolResult {
+ if let Some(structured) = result.structured_content.as_mut() {
+ structured["screenshot"] = json!({
+ "source": "cdp_tab",
+ "scope": "viewport",
+ "mime_type": "image/png",
+ "width": screenshot.width,
+ "height": screenshot.height,
+ "coordinate_space": "viewport_css_px",
+ "viewport_css_width": screenshot.viewport_css_width,
+ "viewport_css_height": screenshot.viewport_css_height,
+ "pixel_to_css_scale_x": screenshot.pixel_to_css_scale_x,
+ "pixel_to_css_scale_y": screenshot.pixel_to_css_scale_y,
+ "tab_activation": "not_requested",
+ "window_foregrounding": "not_requested",
+ });
+ structured["screenshot_width"] = json!(screenshot.width);
+ structured["screenshot_height"] = json!(screenshot.height);
+ structured["screenshot_mime_type"] = json!("image/png");
+ }
+ result
+ .content
+ .insert(0, Content::image_png(screenshot.data_base64));
+ result
+}
+
// ── get_browser_state ────────────────────────────────────────────────────────
pub struct GetBrowserStateTool {
@@ -152,6 +178,11 @@ impl GetBrowserStateTool {
"type": "string",
"description": "Opaque continuation minted by an earlier semantic_v2 response."
},
+ "include_screenshot": {
+ "type": "boolean",
+ "default": false,
+ "description": "Capture the exact tab viewport as PNG through CDP without selecting the tab or foregrounding its native window. The request refuses if capture cannot be completed."
+ },
},
"additionalProperties": true
}),
@@ -190,6 +221,15 @@ impl Tool for GetBrowserStateTool {
let snapshot_format = args
.opt_str("snapshot_format")
.unwrap_or_else(|| "dom_refs_v1".into());
+ let include_screenshot = match args.get("include_screenshot") {
+ None => false,
+ Some(Value::Bool(include)) => *include,
+ Some(_) => {
+ return ToolResult::error(
+ "Field include_screenshot has wrong type: expected boolean",
+ )
+ }
+ };
if snapshot_format != "dom_refs_v1" && snapshot_format != "semantic_v2" {
return ToolResult::error(format!(
"snapshot_format must be \"dom_refs_v1\" or \"semantic_v2\", got {snapshot_format:?}"
@@ -205,7 +245,7 @@ impl Tool for GetBrowserStateTool {
);
}
if snapshot_format == "semantic_v2" {
- return match self
+ let snapshot = match self
.engine
.snapshot_tab_semantic(
&session,
@@ -272,10 +312,21 @@ impl Tool for GetBrowserStateTool {
},
}))
}
- Err(refusal) => refusal.to_tool_result(),
+ Err(refusal) => return refusal.to_tool_result(),
};
+ if include_screenshot {
+ return match self
+ .engine
+ .capture_tab_screenshot(&session, &target_id, &tab_id)
+ .await
+ {
+ Ok(screenshot) => with_tab_screenshot(snapshot, screenshot),
+ Err(refusal) => refusal.to_tool_result(),
+ };
+ }
+ return snapshot;
}
- return match self
+ let snapshot = match self
.engine
.snapshot_tab(&session, &target_id, &tab_id)
.await
@@ -314,8 +365,19 @@ impl Tool for GetBrowserStateTool {
},
}))
}
- Err(refusal) => refusal.to_tool_result(),
+ Err(refusal) => return refusal.to_tool_result(),
};
+ if include_screenshot {
+ return match self
+ .engine
+ .capture_tab_screenshot(&session, &target_id, &tab_id)
+ .await
+ {
+ Ok(screenshot) => with_tab_screenshot(snapshot, screenshot),
+ Err(refusal) => refusal.to_tool_result(),
+ };
+ }
+ return snapshot;
}
// Bind mode: pid + window_id.
@@ -1936,6 +1998,10 @@ mod tests {
"get_browser_state must be strictly read-only"
);
assert!(state.def().idempotent);
+ assert_eq!(
+ state.def().input_schema["properties"]["include_screenshot"]["default"],
+ false
+ );
let prepare = BrowserPrepareTool::new(e.clone());
assert!(prepare.def().destructive);
@@ -2055,6 +2121,23 @@ mod tests {
}
}
+ #[tokio::test]
+ async fn snapshot_rejects_non_boolean_include_screenshot() {
+ let result = GetBrowserStateTool::new(engine())
+ .invoke(json!({
+ "target_id": "bt-fixture",
+ "tab_id": "tab-fixture",
+ "session": "browser-run",
+ "include_screenshot": "yes"
+ }))
+ .await;
+ assert_eq!(result.is_error, Some(true));
+ assert!(matches!(
+ &result.content[0],
+ Content::Text { text, .. } if text.contains("expected boolean")
+ ));
+ }
+
#[tokio::test]
async fn public_session_field_is_the_primary_capability_namespace() {
let tool = GetBrowserStateTool::new(engine());
diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/browser/v2_tests.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/browser/v2_tests.rs
index 49ba71c349..f58156473b 100644
--- a/libs/cua-driver/rust/crates/cua-driver-core/src/browser/v2_tests.rs
+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/browser/v2_tests.rs
@@ -4,15 +4,18 @@
//! revalidation, navigation invalidation, and unproven-capability
//! omission/refusal.
+use std::io::Cursor;
use std::sync::{
atomic::{AtomicBool, Ordering},
Arc, Mutex as StdMutex,
};
use async_trait::async_trait;
+use base64::{engine::general_purpose::STANDARD as BASE64, Engine as _};
+use image::{DynamicImage, ImageFormat, Rgba, RgbaImage};
use serde_json::{json, Value};
-use crate::protocol::ToolResult;
+use crate::protocol::{Content, ToolResult};
use crate::tool::Tool;
use super::engine::BrowserEngine;
@@ -53,6 +56,9 @@ struct FixtureState {
semantic_full_dom_fails: bool,
semantic_full_dom_times_out: bool,
semantic_truncated_dom: bool,
+ screenshot_data: String,
+ viewport_css_width: f64,
+ viewport_css_height: f64,
/// Every incoming CDP call: (sessionId, method, params).
calls: Vec<(Option<String>, String, Value)>,
}
@@ -74,11 +80,23 @@ impl Default for FixtureState {
semantic_full_dom_fails: false,
semantic_full_dom_times_out: false,
semantic_truncated_dom: false,
+ screenshot_data: "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAusB9Y9ZJrAAAAAASUVORK5CYII=".into(),
+ viewport_css_width: 800.0,
+ viewport_css_height: 600.0,
calls: Vec::new(),
}
}
}
+fn screenshot_png_base64(width: u32, height: u32) -> String {
+ let image = RgbaImage::from_pixel(width, height, Rgba([18, 171, 52, 255]));
+ let mut encoded = Cursor::new(Vec::new());
+ DynamicImage::ImageRgba8(image)
+ .write_to(&mut encoded, ImageFormat::Png)
+ .expect("encode screenshot fixture PNG");
+ BASE64.encode(encoded.into_inner())
+}
+
type SharedState = Arc<StdMutex<FixtureState>>;
/// Main-frame document: a plain button, an open shadow root with an
@@ -509,10 +527,13 @@ fn fixture_handler(state: SharedState) -> MockHandler {
"cssVisualViewport": {
"pageX": 0.0,
"pageY": 0.0,
- "clientWidth": 800.0,
- "clientHeight": 600.0
+ "clientWidth": st.viewport_css_width,
+ "clientHeight": st.viewport_css_height
}
})),
+ "Page.captureScreenshot" if is_tab => {
+ MockReply::ok(json!({"data": st.screenshot_data.clone()}))
+ }
"Page.navigate" if is_tab => MockReply::ok(json!({
"frameId": "F_MAIN",
"loaderId": "L_MAIN_NAVIGATED",
@@ -1345,6 +1366,146 @@ async fn semantic_snapshot_keeps_visible_content_after_hidden_node_pressure() {
assert_eq!(snap["snapshot"]["omitted"]["css_hidden"], 320);
}
+#[tokio::test]
+async fn semantic_snapshot_can_capture_an_inactive_tab_without_activation_calls() {
+ let f = fixture().await;
+ let (target, tab) = bind(&f).await;
+ let result = GetBrowserStateTool::new(f.engine.clone())
+ .invoke(json!({
+ "target_id": target,
+ "tab_id": tab,
+ "session": SESSION,
+ "snapshot_format": "semantic_v2",
+ "include_screenshot": true
+ }))
+ .await;
+ let snapshot = structured(&result);
+ assert_eq!(snapshot["status"], "ok", "{snapshot}");
+ assert_eq!(snapshot["screenshot"]["mime_type"], "image/png");
+ assert_eq!(snapshot["screenshot"]["width"], 1);
+ assert_eq!(snapshot["screenshot"]["height"], 1);
+ assert_eq!(snapshot["screenshot"]["source"], "cdp_tab");
+ assert_eq!(snapshot["screenshot_width"], 1);
+ assert_eq!(snapshot["screenshot_height"], 1);
+ assert_eq!(snapshot["screenshot_mime_type"], "image/png");
+ assert_eq!(
+ snapshot["screenshot"]["coordinate_space"],
+ "viewport_css_px"
+ );
+ assert_eq!(snapshot["screenshot"]["viewport_css_width"], 800.0);
+ assert_eq!(snapshot["screenshot"]["viewport_css_height"], 600.0);
+ assert_eq!(snapshot["screenshot"]["pixel_to_css_scale_x"], 800.0);
+ assert_eq!(snapshot["screenshot"]["pixel_to_css_scale_y"], 600.0);
+ assert!(result.content.iter().any(|content| matches!(
+ content,
+ Content::Image { mime_type, .. } if mime_type == "image/png"
+ )));
+
+ let state = f.state.lock().unwrap();
+ assert!(state.calls.iter().any(|(_, method, params)| {
+ method == "Page.captureScreenshot"
+ && params["format"] == "png"
+ && params["fromSurface"] == true
+ && params["captureBeyondViewport"] == false
+ && params["clip"]["x"] == 0.0
+ && params["clip"]["y"] == 0.0
+ && params["clip"]["width"] == 800.0
+ && params["clip"]["height"] == 600.0
+ && params["clip"]["scale"] == 1.0
+ }));
+ assert!(state.calls.iter().all(|(_, method, _)| {
+ method != "Target.activateTarget" && method != "Page.bringToFront"
+ }));
+}
+
+#[tokio::test]
+async fn semantic_snapshot_does_not_capture_unless_requested() {
+ let f = fixture().await;
+ let (target, tab) = bind(&f).await;
+ let snapshot = semantic_snapshot(&f, &target, &tab).await;
+ assert_eq!(snapshot["status"], "ok", "{snapshot}");
+ assert_eq!(snapshot["screenshot"], Value::Null);
+ assert_eq!(snapshot["screenshot_width"], Value::Null);
+ assert_eq!(snapshot["screenshot_height"], Value::Null);
+ assert_eq!(snapshot["screenshot_mime_type"], Value::Null);
+ assert!(recorded_calls(&f, "Page.captureScreenshot").is_empty());
+}
+
+#[tokio::test]
+async fn requested_tab_screenshot_maps_non_unit_png_pixels_to_viewport_css() {
+ let f = fixture_with(|state| {
+ state.viewport_css_width = 2.0;
+ state.viewport_css_height = 1.0;
+ state.screenshot_data = screenshot_png_base64(4, 2);
+ })
+ .await;
+ let (target, tab) = bind(&f).await;
+ let result = GetBrowserStateTool::new(f.engine.clone())
+ .invoke(json!({
+ "target_id": target,
+ "tab_id": tab,
+ "session": SESSION,
+ "snapshot_format": "dom_refs_v1",
+ "include_screenshot": true
+ }))
+ .await;
+ let snapshot = structured(&result);
+ assert_eq!(snapshot["status"], "ok", "{snapshot}");
+ assert_eq!(snapshot["screenshot_width"], 4);
+ assert_eq!(snapshot["screenshot_height"], 2);
+ assert_eq!(snapshot["screenshot_mime_type"], "image/png");
+ assert_eq!(snapshot["screenshot"]["viewport_css_width"], 2.0);
+ assert_eq!(snapshot["screenshot"]["viewport_css_height"], 1.0);
+ assert_eq!(snapshot["screenshot"]["pixel_to_css_scale_x"], 0.5);
+ assert_eq!(snapshot["screenshot"]["pixel_to_css_scale_y"], 0.5);
+ let capture = recorded_calls(&f, "Page.captureScreenshot");
+ assert_eq!(capture.len(), 1, "{capture:?}");
+ assert_eq!(capture[0].1["clip"]["width"], 2.0);
+ assert_eq!(capture[0].1["clip"]["height"], 1.0);
+ assert_eq!(capture[0].1["clip"]["scale"], 1.0);
+}
+
+#[tokio::test]
+async fn requested_tab_screenshot_refuses_invalid_viewport_metrics_before_capture() {
+ let f = fixture_with(|state| state.viewport_css_width = 0.0).await;
+ let (target, tab) = bind(&f).await;
+ let result = GetBrowserStateTool::new(f.engine.clone())
+ .invoke(json!({
+ "target_id": target,
+ "tab_id": tab,
+ "session": SESSION,
+ "snapshot_format": "dom_refs_v1",
+ "include_screenshot": true
+ }))
+ .await;
+ let refusal = structured(&result);
+ assert_eq!(refusal["status"], "refused", "{refusal}");
+ assert_eq!(refusal["refusal"]["code"], "browser_route_unavailable");
+ assert!(recorded_calls(&f, "Page.captureScreenshot").is_empty());
+}
+
+#[tokio::test]
+async fn requested_tab_screenshot_refuses_malformed_image_data() {
+ let f = fixture_with(|state| state.screenshot_data = "not-base64".into()).await;
+ let (target, tab) = bind(&f).await;
+ let result = GetBrowserStateTool::new(f.engine.clone())
+ .invoke(json!({
+ "target_id": target,
+ "tab_id": tab,
+ "session": SESSION,
+ "snapshot_format": "semantic_v2",
+ "include_screenshot": true
+ }))
+ .await;
+ let refusal = structured(&result);
+ assert_eq!(refusal["status"], "refused", "{refusal}");
+ assert_eq!(refusal["refusal"]["code"], "browser_route_unavailable");
+ assert!(result
+ .content
+ .iter()
+ .all(|content| !matches!(content, Content::Image { .. })));
+}
+
#[tokio::test]
async fn semantic_continuation_is_opaque_single_use_and_reaches_offscreen_content() {
let f = fixture_with(|st| st.semantic_large_page = true).await;
diff --git a/libs/cua-driver/rust/crates/cua-driver/tests/protocol_schema_test.rs b/libs/cua-driver/rust/crates/cua-driver/tests/protocol_schema_test.rs
index 5b47d0ba9f..769fe15ecb 100644
--- a/libs/cua-driver/rust/crates/cua-driver/tests/protocol_schema_test.rs
+++ b/libs/cua-driver/rust/crates/cua-driver/tests/protocol_schema_test.rs
@@ -76,6 +76,10 @@ fn tools_list_schema_shape() {
"get_browser_state schema missing {field}"
);
}
+ assert_eq!(
+ browser_state["inputSchema"]["properties"]["include_screenshot"]["default"], false,
+ "get_browser_state should advertise opt-in exact-tab capture"
+ );
for (name, required) in [
("browser_prepare", &["pid"][..]),
("browser_navigate", &["target_id", "tab_id", "url"][..]),
diff --git a/libs/cua-driver/rust/crates/cua-driver/tests/standalone_browser_behavior_test.rs b/libs/cua-driver/rust/crates/cua-driver/tests/standalone_browser_behavior_test.rs
index 4dc69d7fa0..e0acd99491 100644
--- a/libs/cua-driver/rust/crates/cua-driver/tests/standalone_browser_behavior_test.rs
+++ b/libs/cua-driver/rust/crates/cua-driver/tests/standalone_browser_behavior_test.rs
@@ -14,6 +14,7 @@ use std::sync::Mutex;
use std::thread;
use std::time::{Duration, Instant};
+use base64::Engine as _;
use cua_driver_testkit::e2e::{
append_json_line, execute_case, recording_evidence, write_environment_from_env, CaseSpec,
Delivery, DriverRoute, EnvironmentRecord, Evidence, Observation, OracleKind, RefusalCode,
@@ -615,31 +616,28 @@ fn cdp_target_for_url(port: u16, url: &str) -> String {
.to_owned()
}
-fn bring_harness_page_to_front(port: u16, url: &str) {
+fn cdp_page_websocket_for_url(port: u16, url: &str) -> String {
let targets = browser_http_json(port, "/json/list")
.unwrap_or_else(|error| panic!("read harness CDP targets: {error}"));
- let ws_url = targets
+ targets
.as_array()
.into_iter()
.flatten()
.find(|target| target["type"] == "page" && target["url"] == url)
.and_then(|target| target["webSocketDebuggerUrl"].as_str())
- .unwrap_or_else(|| panic!("no harness page websocket for {url}"));
- harness_cdp_call_at_url(ws_url, "Page.bringToFront", serde_json::json!({}));
+ .unwrap_or_else(|| panic!("no harness page websocket for {url}"))
+ .to_owned()
+}
+
+fn bring_harness_page_to_front(port: u16, url: &str) {
+ let ws_url = cdp_page_websocket_for_url(port, url);
+ harness_cdp_call_at_url(&ws_url, "Page.bringToFront", serde_json::json!({}));
}
fn harness_page_visibility(port: u16, url: &str) -> String {
- let targets = browser_http_json(port, "/json/list")
- .unwrap_or_else(|error| panic!("read harness CDP targets: {error}"));
- let ws_url = targets
- .as_array()
- .into_iter()
- .flatten()
- .find(|target| target["type"] == "page" && target["url"] == url)
- .and_then(|target| target["webSocketDebuggerUrl"].as_str())
- .unwrap_or_else(|| panic!("no harness page websocket for {url}"));
+ let ws_url = cdp_page_websocket_for_url(port, url);
harness_cdp_call_at_url(
- ws_url,
+ &ws_url,
"Runtime.evaluate",
serde_json::json!({
"expression": "document.visibilityState",
@@ -2305,10 +2303,15 @@ fn run_multi_tab(spec: &BrowserSpec) {
*evidence = recording_evidence(fixture.driver.recording_dir());
let session = format!("standalone-multi-tab-{}", fixture.pid);
let _ = bind(&mut fixture, &session);
- let second_html = standalone_fixture_html().replace(
- "<title>cua-driver Web Harness</title>",
- "<title>cua-driver Background Harness</title>",
- );
+ let second_html = standalone_fixture_html()
+ .replace(
+ "<title>cua-driver Web Harness</title>",
+ "<title>cua-driver Background Harness</title>",
+ )
+ .replace(
+ "</style>",
+ "body { background: rgb(18, 171, 52) !important; }</style>",
+ );
let second_server = BrowserFixtureServer::start(&second_html);
let created = harness_cdp_call(
fixture.cdp_port,
@@ -2322,6 +2325,33 @@ fn run_multi_tab(spec: &BrowserSpec) {
assert!(created["targetId"].is_string(), "{created}");
wait_for_observed(&second_server, "WEB_HARNESS_MARKER_v1");
+ // Exercise screenshot/coordinate parity under a non-1 device scale.
+ // This is setup instrumentation and runs before the background sentinel;
+ // the driver action below must still leave the tab selected state and
+ // native foreground unchanged.
+ let background_ws = cdp_page_websocket_for_url(fixture.cdp_port, &second_server.page_url());
+ harness_cdp_call_at_url(
+ &background_ws,
+ "Emulation.setDeviceMetricsOverride",
+ serde_json::json!({
+ "width": 400,
+ "height": 300,
+ "deviceScaleFactor": 2,
+ "mobile": false,
+ }),
+ );
+ let device_scale = harness_cdp_call_at_url(
+ &background_ws,
+ "Runtime.evaluate",
+ serde_json::json!({
+ "expression": "window.devicePixelRatio",
+ "returnByValue": true,
+ }),
+ )["result"]["value"]
+ .as_f64()
+ .expect("background tab device scale factor");
+ assert!((device_scale - 2.0).abs() < f64::EPSILON, "{device_scale}");
+
// Establish a deterministic selected-tab baseline before the
// foreground sentinel starts. Chromium does not consistently honor
// createTarget(background=true) on every host, but Page.bringToFront
@@ -2409,9 +2439,104 @@ fn run_multi_tab(spec: &BrowserSpec) {
"session": session,
"snapshot_format": "semantic_v2",
"query": "Increment txt-input",
+ "include_screenshot": true,
}),
);
assert_eq!(snapshot.structured()["status"], "ok", "{}", snapshot.raw);
+ assert_eq!(
+ snapshot.structured()["screenshot"]["source"],
+ "cdp_tab",
+ "{}",
+ snapshot.raw
+ );
+ assert!(
+ snapshot.structured()["screenshot"]["width"]
+ .as_u64()
+ .is_some_and(|width| width > 0)
+ && snapshot.structured()["screenshot"]["height"]
+ .as_u64()
+ .is_some_and(|height| height > 0),
+ "{}",
+ snapshot.raw
+ );
+ assert_eq!(
+ snapshot.structured()["screenshot_width"],
+ snapshot.structured()["screenshot"]["width"]
+ );
+ assert_eq!(
+ snapshot.structured()["screenshot_height"],
+ snapshot.structured()["screenshot"]["height"]
+ );
+ assert_eq!(snapshot.structured()["screenshot_mime_type"], "image/png");
+ assert_eq!(
+ snapshot.structured()["screenshot"]["coordinate_space"],
+ "viewport_css_px"
+ );
+ let png_width = snapshot.structured()["screenshot"]["width"]
+ .as_f64()
+ .expect("PNG width");
+ let png_height = snapshot.structured()["screenshot"]["height"]
+ .as_f64()
+ .expect("PNG height");
+ let viewport_width = snapshot.structured()["screenshot"]["viewport_css_width"]
+ .as_f64()
+ .expect("CSS viewport width");
+ let viewport_height = snapshot.structured()["screenshot"]["viewport_css_height"]
+ .as_f64()
+ .expect("CSS viewport height");
+ let pixel_to_css_x = snapshot.structured()["screenshot"]["pixel_to_css_scale_x"]
+ .as_f64()
+ .expect("screenshot x scale");
+ let pixel_to_css_y = snapshot.structured()["screenshot"]["pixel_to_css_scale_y"]
+ .as_f64()
+ .expect("screenshot y scale");
+ assert!((viewport_width - 400.0).abs() < 0.01, "{}", snapshot.raw);
+ assert!((viewport_height - 300.0).abs() < 0.01, "{}", snapshot.raw);
+ assert!(
+ (pixel_to_css_x - viewport_width / png_width).abs() < 1e-9,
+ "{}",
+ snapshot.raw
+ );
+ assert!(
+ (pixel_to_css_y - viewport_height / png_height).abs() < 1e-9,
+ "{}",
+ snapshot.raw
+ );
+ assert!(
+ snapshot.raw["result"]["content"]
+ .as_array()
+ .is_some_and(|content| content.iter().any(|item| {
+ item["type"] == "image"
+ && item["mimeType"] == "image/png"
+ && item["data"]
+ .as_str()
+ .is_some_and(|data| data.starts_with("iVBOR"))
+ })),
+ "{}",
+ snapshot.raw
+ );
+ let screenshot_data = snapshot.raw["result"]["content"]
+ .as_array()
+ .and_then(|content| {
+ content
+ .iter()
+ .find(|item| item["type"] == "image")
+ .and_then(|item| item["data"].as_str())
+ })
+ .expect("inactive-tab PNG data");
+ let screenshot_bytes = base64::engine::general_purpose::STANDARD
+ .decode(screenshot_data)
+ .expect("decode inactive-tab PNG");
+ let screenshot = image::load_from_memory(&screenshot_bytes)
+ .expect("decode inactive-tab image")
+ .to_rgba8();
+ let pixel = screenshot.get_pixel(2, 2).0;
+ assert!(
+ pixel[0].abs_diff(18) <= 2
+ && pixel[1].abs_diff(171) <= 2
+ && pixel[2].abs_diff(52) <= 2,
+ "screenshot did not come from the green inactive tab: rgba={pixel:?}"
+ );
let click_ref = semantic_ref_by_name(&snapshot, "Increment", "click");
let trusted = fixture.driver.call(
"browser_click",