Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
fix(cua-driver): keep rendered controls visible in semantic_v2 and report not_rendered for background tabs (#5256)
injaneitya212cd026e26
diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/browser/engine.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/browser/engine.rsindex c4e14caa41..1d0940c17b 100644--- a/libs/cua-driver/rust/crates/cua-driver-core/src/browser/engine.rs+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/browser/engine.rs@@ -53,8 +53,8 @@ use super::semantic::{SEMANTIC_COMPUTED_STYLES,};use super::store::{- format_ref, BrowserStore, FrameIdentity, FrameKind, FrameRef, RefEntry, SemanticContinuation,- SnapshotRecord, TabRecord, TargetRecord,+ format_ref, BrowserStore, BrowserVisibility, FrameIdentity, FrameKind, FrameRef, RefEntry,+ SemanticContinuation, SnapshotRecord, TabRecord, TargetRecord,};use super::types::{BindingQuality, BrowserClassification, BrowserEngineFamily, BrowserProcessRole,@@ -1743,21 +1743,7 @@ impl BrowserEngine {// focusing its native window or invoking any CDP activation command.// Treat an unavailable or malformed proof as inactive: omitting// feedback is safer than drawing a cursor over another tab.- let tab_is_active = validated- .conn- .call(- Some(&validated.cdp_session),- "Runtime.evaluate",- json!({- "expression": "document.visibilityState === 'visible'",- "returnByValue": true,- "awaitPromise": false,- }),- )- .await- .ok()- .and_then(|result| result.pointer("/result/value").and_then(Value::as_bool))- .unwrap_or(false);+ let tab_is_active = tab_visibility_proven(&validated.conn, &validated.cdp_session).await;let screen_point = if cdp_session == validated.cdp_session {validated@@ -2787,6 +2773,24 @@ impl BrowserEngine {OopifStatus::Unsupported};+ if !semantic.nodes.iter().any(|node| {+ matches!(+ node.visibility,+ BrowserVisibility::InViewport+ | BrowserVisibility::NearViewport+ | BrowserVisibility::Offscreen+ )+ }) && !tokio::time::timeout(+ std::time::Duration::from_millis(500),+ tab_visibility_proven(&conn, &cdp_session),+ )+ .await+ .unwrap_or(false)+ {+ semantic.not_rendered_count = 1;+ semantic.complete = false;+ }+let page = semantic.page(0,DEFAULT_SEMANTIC_NODE_BUDGET,@@ -2850,6 +2854,22 @@ impl BrowserEngine {}}+async fn tab_visibility_proven(conn: &CdpConnection, cdp_session: &str) -> bool {+ conn.call(+ Some(cdp_session),+ "Runtime.evaluate",+ json!({+ "expression": "document.visibilityState === 'visible'",+ "returnByValue": true,+ "awaitPromise": false,+ }),+ )+ .await+ .ok()+ .and_then(|result| result.pointer("/result/value").and_then(Value::as_bool))+ .unwrap_or(false)+}+/// Attribute names that make an element interactive-enough to ref.const INTERACTIVE_ATTRS: &[&str] = &["onclick", "role", "contenteditable", "tabindex", "href"];/// Element names always considered interactive.@@ -2920,6 +2940,9 @@ fn collect_interactive(.find(|kv| kv[0].eq_ignore_ascii_case(key)).map(|kv| kv[1].as_str())};+ let closed_select = name == "select"+ && attr("multiple").is_none()+ && attr("size").and_then(|size| size.trim().parse::<u32>().ok()) <= Some(1);let interactive = INTERACTIVE_TAGS.contains(&name.as_str())|| INTERACTIVE_ATTRS.iter().any(|a| attr(a).is_some());if interactive {@@ -2938,6 +2961,9 @@ fn collect_interactive(});}}+ if closed_select {+ return;+ }}if let Some(children) = node.get("children").and_then(Value::as_array) {for child in children {@@ -3174,6 +3200,36 @@ mod tests {assert_eq!(inner.frame_id.as_deref(), Some("F_IFRAME"));}+ #[test]+ fn collector_refs_a_closed_select_without_its_options() {+ let option = |backend: i64| {+ json!({"nodeType": 1, "nodeName": "OPTION", "backendNodeId": backend,+ "attributes": ["value", format!("c{backend}")]})+ };+ let doc = json!({+ "nodeType": 9,+ "nodeName": "#document",+ "frameId": "F_MAIN",+ "children": [+ {"nodeType": 1, "nodeName": "SELECT", "backendNodeId": 1,+ "attributes": ["name", "country"],+ "children": (100..250).map(option).collect::<Vec<_>>()},+ {"nodeType": 1, "nodeName": "SELECT", "backendNodeId": 2,+ "attributes": ["name", "tags", "multiple", ""],+ "children": [option(300), option(301)]},+ {"nodeType": 1, "nodeName": "SELECT", "backendNodeId": 3,+ "attributes": ["name", "list", "size", "4"],+ "children": [option(400)]},+ {"nodeType": 1, "nodeName": "BUTTON", "backendNodeId": 4,+ "attributes": ["type", "submit"]}+ ]+ });+ let mut out = Vec::new();+ collect_interactive(&doc, Some("F_MAIN"), true, &mut out);+ let backends: Vec<i64> = out.iter().map(|e| e.backend_node_id).collect();+ assert_eq!(backends, vec![1, 2, 300, 301, 3, 400, 4]);+ }+#[test]fn parse_frame_tree_maps_frames_and_omits_malformed_children() {let tree = parse_frame_tree(&json!({diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/browser/semantic.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/browser/semantic.rsindex dc7cecebcf..0005f29954 100644--- a/libs/cua-driver/rust/crates/cua-driver-core/src/browser/semantic.rs+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/browser/semantic.rs@@ -80,6 +80,7 @@ struct DomMeta {attrs: HashMap<String, String>,order: usize,css_hidden: bool,+ transparent: bool,parent_backend_node_id: Option<i64>,frame_id: Option<String>,}@@ -149,6 +150,7 @@ pub(crate) struct OmissionCounts {pub(crate) unknown: usize,pub(crate) budget: usize,pub(crate) unprovable_frame: usize,+ pub(crate) not_rendered: usize,}#[derive(Debug, Clone)]@@ -166,6 +168,7 @@ pub(crate) struct SemanticDocument {pub(crate) nodes: Vec<SemanticNode>,pub(crate) css_hidden_dom_count: usize,pub(crate) unprovable_frame_count: usize,+ pub(crate) not_rendered_count: usize,pub(crate) complete: bool,}@@ -186,6 +189,7 @@ impl SemanticDocument {self.nodes.extend(other.nodes);self.css_hidden_dom_count += other.css_hidden_dom_count;self.unprovable_frame_count += other.unprovable_frame_count;+ self.not_rendered_count += other.not_rendered_count;self.complete = if was_empty {other.complete} else {@@ -232,6 +236,7 @@ impl SemanticDocument {let mut omissions = OmissionCounts {css_hidden: self.css_hidden_dom_count.max(semantic_css_hidden),unprovable_frame: self.unprovable_frame_count,+ not_rendered: self.not_rendered_count,..Default::default()};for node in &self.nodes {@@ -293,6 +298,10 @@ pub(crate) fn build_dom_index(root: &Value) -> DomIndex {let node_type = node.get("nodeType").and_then(Value::as_i64).unwrap_or(0);let attrs = attributes(node);let hidden = inherited_hidden || statically_hidden(&attrs);+ let transparent = statically_transparent(&attrs)+ || parent_backend_node_id+ .and_then(|parent| index.nodes.get(&parent))+ .is_some_and(|parent| parent.transparent);let backend_node_id = node.get("backendNodeId").and_then(Value::as_i64);let frame_id = if node_type == 9 {node.get("frameId").and_then(Value::as_str)@@ -315,6 +324,7 @@ pub(crate) fn build_dom_index(root: &Value) -> DomIndex {attrs,order: *order,css_hidden: hidden,+ transparent,parent_backend_node_id,frame_id: frame_id.map(str::to_owned),},@@ -568,6 +578,7 @@ pub(crate) fn compose_accessibility_tree(nodes,css_hidden_dom_count: dom.css_hidden_count,unprovable_frame_count: 0,+ not_rendered_count: 0,complete: true,}}@@ -725,32 +736,38 @@ fn statically_hidden(attrs: &HashMap<String, String>) -> bool {if attrs.contains_key("hidden") || attrs.get("aria-hidden").is_some_and(|v| v == "true") {return true;}- let style = attrs+ inline_style(attrs).any(|(property, value)| hiding_style(property, value))+}++fn statically_transparent(attrs: &HashMap<String, String>) -> bool {+ inline_style(attrs).any(|(property, value)| {+ property.trim().eq_ignore_ascii_case("opacity") && hiding_style(property, value)+ })+}++fn inline_style(attrs: &HashMap<String, String>) -> impl Iterator<Item = (&str, &str)> {+ attrs.get("style")- .map(|value| value.to_ascii_lowercase())- .unwrap_or_default();- style.contains("display:none")- || style.contains("display: none")- || style.contains("visibility:hidden")- || style.contains("visibility: hidden")- || style.contains("opacity:0")- || style.contains("opacity: 0")+ .into_iter()+ .flat_map(|style| style.split(';'))+ .filter_map(|declaration| declaration.split_once(':'))+}++fn hiding_style(property: &str, value: &str) -> bool {+ let value = value.trim().trim_end_matches("!important").trim_end();+ match property.trim().to_ascii_lowercase().as_str() {+ "display" => value.eq_ignore_ascii_case("none"),+ "visibility" => value.eq_ignore_ascii_case("hidden"),+ "opacity" => value.parse::<f64>().is_ok_and(|opacity| opacity <= 0.0),+ _ => false,+ }}fn layout_hidden(layout: &LayoutMeta) -> bool {layout.styles- .get("display")- .is_some_and(|value| value.eq_ignore_ascii_case("none"))- || layout- .styles- .get("visibility")- .is_some_and(|value| value.eq_ignore_ascii_case("hidden"))- || layout- .styles- .get("opacity")- .and_then(|value| value.parse::<f64>().ok())- .is_some_and(|opacity| opacity <= 0.0)+ .iter()+ .any(|(property, value)| hiding_style(property, value))}fn classify_visibility(@@ -758,7 +775,10 @@ fn classify_visibility(layout: Option<&LayoutMeta>,viewport: &Viewport,) -> BrowserVisibility {- if dom.is_some_and(|meta| meta.css_hidden) || layout.is_some_and(layout_hidden) {+ let rendered = layout.is_some_and(|layout| layout.bounds.is_some_and(Rect::has_area));+ if dom.is_some_and(|meta| meta.transparent || (meta.css_hidden && !rendered))+ || layout.is_some_and(layout_hidden)+ {return BrowserVisibility::CssHidden;}let Some(layout) = layout else {@@ -1173,6 +1193,7 @@ mod tests {attrs: HashMap::from([("type".into(), "file".into())]),order: 0,css_hidden: false,+ transparent: false,parent_backend_node_id: None,frame_id: None,};@@ -1187,6 +1208,19 @@ mod tests {FrameRef::main_unproven()}+ #[test]+ fn not_rendered_count_propagates_through_extend_and_page() {+ let mut document = SemanticDocument {+ not_rendered_count: 1,+ ..Default::default()+ };+ document.extend(SemanticDocument {+ complete: true,+ ..Default::default()+ });+ assert_eq!(document.page(0, 300, None, None).omissions.not_rendered, 1);+ }+#[test]fn hidden_dom_nodes_do_not_enter_the_visible_working_set() {let dom = build_dom_index(&json!({@@ -1226,6 +1260,90 @@ mod tests {assert_eq!(actionable[0].name.as_deref(), Some("Reply"));}+ #[test]+ fn inline_style_hides_only_on_exact_hiding_declarations() {+ let dom = build_dom_index(&json!({+ "nodeType": 9,+ "children": [+ {"nodeType": 1, "nodeName": "DIV", "backendNodeId": 1,+ "attributes": ["style", "opacity: 0.98; transition: opacity 0.2s"]},+ {"nodeType": 1, "nodeName": "DIV", "backendNodeId": 2,+ "attributes": ["style", "--overlay-display:none;opacity:0.5"]},+ {"nodeType": 1, "nodeName": "DIV", "backendNodeId": 3,+ "attributes": ["style", "color: red; display: none !important"]},+ {"nodeType": 1, "nodeName": "DIV", "backendNodeId": 4,+ "attributes": ["style", "OPACITY:0"]}+ ]+ }));+ let hidden = |backend| dom.nodes[&backend].css_hidden;+ assert!(!hidden(1));+ assert!(!hidden(2));+ assert!(hidden(3));+ assert!(hidden(4));+ assert_eq!(dom.css_hidden_count, 2);+ }++ #[test]+ fn rendered_layout_overrides_inherited_static_hiding() {+ let dom = build_dom_index(&json!({+ "nodeType": 9,+ "frameId": "F_MAIN",+ "children": [+ {"nodeType": 1, "nodeName": "FORM", "backendNodeId": 1,+ "attributes": ["style", "visibility: hidden"],+ "children": [+ {"nodeType": 1, "nodeName": "INPUT", "backendNodeId": 2,+ "attributes": ["aria-label", "Business name",+ "style", "visibility: visible"]},+ {"nodeType": 1, "nodeName": "INPUT", "backendNodeId": 3,+ "attributes": ["aria-label", "Unrendered"]}+ ]},+ {"nodeType": 1, "nodeName": "DIV", "backendNodeId": 4,+ "attributes": ["style", "opacity: 0"],+ "children": [+ {"nodeType": 1, "nodeName": "SPAN", "children": [+ {"nodeType": 1, "nodeName": "INPUT", "backendNodeId": 5,+ "attributes": ["aria-label", "Transparent"]}+ ]}+ ]}+ ]+ }));+ let layout = build_layout_index(&json!({+ "strings": ["block", "visible", "1", "auto"],+ "documents": [{+ "nodes": {"backendNodeId": [2, 5]},+ "layout": {+ "nodeIndex": [0, 1],+ "bounds": [[10, 10, 200, 30], [10, 50, 200, 30]],+ "styles": [[0, 1, 2, 3, 3, 3, 3], [0, 1, 2, 3, 3, 3, 3]],+ "paintOrders": [1, 2]+ }+ }]+ }));+ let viewport = parse_viewport(&json!({+ "cssVisualViewport": {"pageX": 0.0, "pageY": 0.0,+ "clientWidth": 800.0, "clientHeight": 600.0}+ }));+ let document = compose_accessibility_tree(+ &json!({"nodes": [{"nodeId": "root", "ignored": false,+ "role": {"value": "RootWebArea"}, "childIds": []}]}),+ &dom,+ &layout,+ &viewport,+ frame(),+ );+ let page = document.page(0, 300, None, None);+ let names = page+ .selected+ .iter()+ .filter_map(|node| Some((node.name.as_deref()?, node.visibility)))+ .collect::<Vec<_>>();+ assert_eq!(+ names,+ vec![("Business name", BrowserVisibility::InViewport)]+ );+ }+#[test]fn dom_supplement_adds_only_explicit_visible_custom_actions() {let dom = build_dom_index(&json!({diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/browser/tools.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/browser/tools.rsindex 15162d9e7c..7b31ed1a8e 100644--- a/libs/cua-driver/rust/crates/cua-driver-core/src/browser/tools.rs+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/browser/tools.rs@@ -454,8 +454,10 @@ impl Tool for GetBrowserStateTool {"unknown": outcome.omissions.unknown,"budget": outcome.omissions.budget,"unprovable_frame": outcome.omissions.unprovable_frame,+ "not_rendered": outcome.omissions.not_rendered,},"continuation": outcome.continuation,+ "hint": (outcome.omissions.not_rendered > 0).then_some("Chrome has not rendered this background tab, so this tree is partial. Bring the tab to the front in the browser, then re-run get_browser_state; observation never activates tabs."),},"page": {"url": outcome.url,diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/browser/v2_tests.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/browser/v2_tests.rsindex 67e81ea828..14683b10f0 100644--- a/libs/cua-driver/rust/crates/cua-driver-core/src/browser/v2_tests.rs+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/browser/v2_tests.rs@@ -59,6 +59,7 @@ pub(super) struct FixtureState {fail_key_down_after: Option<usize>,completed_key_pairs: usize,semantic_large_page: bool,+ semantic_html_style: String,semantic_full_dom_fails: bool,semantic_full_dom_times_out: bool,semantic_truncated_dom: bool,@@ -99,6 +100,7 @@ impl Default for FixtureState {fail_key_down_after: None,completed_key_pairs: 0,semantic_large_page: false,+ semantic_html_style: String::new(),semantic_full_dom_fails: false,semantic_full_dom_times_out: false,semantic_truncated_dom: false,@@ -210,7 +212,7 @@ fn main_document() -> Value {/// Large application document used to prove that hidden retained controls do/// not consume the semantic snapshot budget ahead of the active view.-fn large_semantic_document() -> Value {+fn large_semantic_document(html_style: &str) -> Value {let mut children = Vec::new();for id in 0..320_i64 {children.push(json!({@@ -278,6 +280,7 @@ fn large_semantic_document() -> Value {"nodeType": 1,"nodeName": "HTML","backendNodeId": 999,+ "attributes": ["style", html_style],"children": children,}]}@@ -522,7 +525,7 @@ fn fixture_handler(state: SharedState) -> MockHandler {MockReply::ok(truncated_semantic_document())} else {MockReply::ok(if st.semantic_large_page {- large_semantic_document()+ large_semantic_document(&st.semantic_html_style)} else {main_document()})@@ -530,7 +533,7 @@ fn fixture_handler(state: SharedState) -> MockHandler {}"DOM.describeNode" if is_tab && call.params["backendNodeId"] == 999 => {MockReply::ok(json!({- "node": large_semantic_document()["root"]["children"][0].clone()+ "node": large_semantic_document(&st.semantic_html_style)["root"]["children"][0].clone()}))}"DOM.getDocument" if is_oopif => MockReply::ok(oopif_document()),@@ -1967,6 +1970,117 @@ async fn semantic_snapshot_can_capture_an_inactive_tab_without_activation_calls(}));}+#[tokio::test]+async fn semantic_background_tab_without_layout_reports_not_rendered() {+ let visible = fixture_with(|st| st.oopif_present = false).await;+ let (target, tab) = bind(&visible).await;+ let visible_snap = semantic_snapshot(&visible, &target, &tab).await;++ let f = fixture_with(|st| {+ st.oopif_present = false;+ st.tab_visible = false;+ })+ .await;+ let (target, tab) = bind(&f).await;+ let snap = semantic_snapshot(&f, &target, &tab).await;++ assert_eq!(snap["status"], "ok", "{snap}");+ assert_eq!(snap["snapshot"]["complete"], false, "{snap}");+ assert_eq!(snap["snapshot"]["omitted"]["not_rendered"], 1, "{snap}");+ assert!(snap["snapshot"]["hint"].is_string(), "{snap}");+ assert_eq!(snap["outline"], visible_snap["outline"]);+ let without_ref_ids = |refs: &Value| {+ let mut refs = refs.clone();+ for entry in refs.as_array_mut().unwrap() {+ entry.as_object_mut().unwrap().remove("ref");+ }+ refs+ };+ assert_eq!(+ without_ref_ids(&snap["refs"]),+ without_ref_ids(&visible_snap["refs"])+ );+ assert_eq!(+ without_ref_ids(&snap["content_refs"]),+ without_ref_ids(&visible_snap["content_refs"])+ );+ for method in [+ "Page.bringToFront",+ "Target.activateTarget",+ "Page.setWebLifecycleState",+ "Emulation.setFocusEmulationEnabled",+ ] {+ assert!(recorded_calls(&f, method).is_empty(), "{method} was called");+ }+}++#[tokio::test]+async fn semantic_background_tab_with_layout_is_not_flagged() {+ let f = fixture_with(|st| {+ st.semantic_large_page = true;+ st.tab_visible = false;+ })+ .await;+ let (target, tab) = bind(&f).await;+ let snap = semantic_snapshot(&f, &target, &tab).await;++ assert_eq!(snap["status"], "ok", "{snap}");+ assert_eq!(snap["snapshot"]["omitted"]["not_rendered"], 0, "{snap}");+ assert_eq!(snap["snapshot"]["hint"], Value::Null, "{snap}");+ assert_eq!(snap["snapshot"]["omitted"]["css_hidden"], 320);+ assert!(recorded_calls(&f, "Runtime.evaluate").is_empty());+}++#[tokio::test]+async fn semantic_background_tab_with_faded_in_wrapper_is_not_flagged() {+ let f = fixture_with(|st| {+ st.semantic_large_page = true;+ st.semantic_html_style = "opacity: 0.98; transition: opacity 0.3s".into();+ st.oopif_present = false;+ st.tab_visible = false;+ })+ .await;+ let (target, tab) = bind(&f).await;+ let snap = semantic_snapshot(&f, &target, &tab).await;++ assert_eq!(snap["status"], "ok", "{snap}");+ assert_eq!(snap["snapshot"]["omitted"]["not_rendered"], 0, "{snap}");+ assert_eq!(snap["snapshot"]["omitted"]["css_hidden"], 320, "{snap}");+ let outline = snap["outline"].as_str().unwrap();+ assert!(outline.contains("Reply body"), "{outline}");+ assert!(!outline.contains("Retained control"), "{outline}");+}++#[tokio::test]+async fn semantic_background_tab_with_rendered_oopif_is_not_flagged() {+ let f = fixture_with(|st| st.tab_visible = false).await;+ let (target, tab) = bind(&f).await;+ let snap = semantic_snapshot(&f, &target, &tab).await;++ assert_eq!(snap["status"], "ok", "{snap}");+ assert_eq!(snap["oopif"]["frames"], 1, "{snap}");+ assert_eq!(snap["snapshot"]["omitted"]["not_rendered"], 0, "{snap}");+ assert_eq!(snap["snapshot"]["hint"], Value::Null, "{snap}");+}++#[tokio::test]+async fn semantic_visible_tab_without_layout_is_not_flagged() {+ let f = fixture_with(|st| st.oopif_present = false).await;+ let (target, tab) = bind(&f).await;+ let snap = semantic_snapshot(&f, &target, &tab).await;++ assert_eq!(snap["status"], "ok", "{snap}");+ assert_eq!(snap["snapshot"]["complete"], true, "{snap}");+ assert_eq!(snap["snapshot"]["omitted"]["not_rendered"], 0, "{snap}");+ assert_eq!(snap["snapshot"]["hint"], Value::Null, "{snap}");+ let probes = recorded_calls(&f, "Runtime.evaluate");+ assert_eq!(probes.len(), 1, "{probes:?}");+ assert_eq!(+ probes[0].1["expression"],+ "document.visibilityState === 'visible'"+ );+}+#[tokio::test]async fn semantic_snapshot_does_not_capture_unless_requested() {let f = fixture().await;