Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
fix(cua-driver): classify DOM click effects with readback
injaneity1bf24b83ed9d
diff --git a/libs/cua-driver/docs/action-result-contract.md b/libs/cua-driver/docs/action-result-contract.mdindex d382d8c490..8d48afd894 100644--- a/libs/cua-driver/docs/action-result-contract.md+++ b/libs/cua-driver/docs/action-result-contract.md@@ -123,6 +123,13 @@ in the same string. It is advice only: it has no `next_call`, it never rides onan error, and it is never sent for a step that already runs inside `run_steps`or `run_script`. The logic is in `rust/crates/cua-driver-core/src/batch_nudge.rs`.+`suspected_noop` is advisory. It means a bounded readable state remained+unchanged, not that the action failed. For a ref-targeted browser DOM click,+the readback covers bounded semantic and form state on the target plus the+document URL. An unchanged DOM click readback remains `unverifiable`, with+readback evidence: effects elsewhere on the page can be invisible to that+readback, so verify the expected postcondition before retrying.+## Window target resolutionWindow-scoped actions accept a PID without `window_id` only when that processdiff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/browser/mock_cdp.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/browser/mock_cdp.rsindex 404ddd2a30..a375cab7da 100644--- a/libs/cua-driver/rust/crates/cua-driver-core/src/browser/mock_cdp.rs+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/browser/mock_cdp.rs@@ -33,6 +33,7 @@ pub(crate) struct MockEvent {/// The scripted reaction to one command.pub(crate) struct MockReply {pub events: Vec<MockEvent>,+ pub suppress_response: bool,pub result: Result<Value, (i64, String)>,}@@ -41,6 +42,7 @@ impl MockReply {Self {events: Vec::new(),result: Ok(result),+ suppress_response: false,}}@@ -48,6 +50,7 @@ impl MockReply {Self {events: Vec::new(),result: Err((code, message.to_owned())),+ suppress_response: false,}}@@ -114,6 +117,9 @@ impl MockCdpServer {.map(str::to_owned),};let reply = handler(&call);+ if reply.suppress_response {+ continue;+ }for event in reply.events {let mut frame =json!({ "method": event.method, "params": event.params });diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/browser/tools.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/browser/tools.rsindex 7b31ed1a8e..df044019dd 100644--- a/libs/cua-driver/rust/crates/cua-driver-core/src/browser/tools.rs+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/browser/tools.rs@@ -6,10 +6,16 @@//! `{"status": "ok" | "refused", ...}`.use std::sync::Arc;+use std::time::{Duration, Instant};use async_trait::async_trait;use serde_json::{json, Value};+use crate::action_record::{+ effect_from_value_readback, ActionEffect, ActionEscalation, ActionEvidence,+ ActionExecutionRecord, ActionTransport, ActualDelivery, EscalationKind, EvidenceKind,+ RequestedDelivery,+};use crate::protocol::{Content, ToolResult};use crate::tool::{ProtectedResourceOwnership, Tool, ToolDef, ToolRegistry};use crate::tool_args::ArgsExt;@@ -45,6 +51,143 @@ pub fn register_browser_tools(engine: &Arc<BrowserEngine>, registry: &mut ToolRe// ── Shared helpers ───────────────────────────────────────────────────────────+const DOM_CLICK_READBACK: &str = r#"function() {+ const bounded = value => typeof value === 'string' ? value.slice(0, 512) : null;+ const attr = name => bounded(this.getAttribute(name));+ return {+ ariaPressed: attr('aria-pressed'),+ ariaSelected: attr('aria-selected'),+ ariaChecked: attr('aria-checked'),+ ariaExpanded: attr('aria-expanded'),+ ariaCurrent: attr('aria-current'),+ ariaDisabled: attr('aria-disabled'),+ disabled: 'disabled' in this ? Boolean(this.disabled) : null,+ checked: 'checked' in this ? Boolean(this.checked) : null,+ value: 'value' in this ? bounded(String(this.value)) : null,+ innerText: bounded(this.innerText),+ href: this.ownerDocument && this.ownerDocument.location+ ? bounded(this.ownerDocument.location.href)+ : null+ };+}"#;++const DOM_CLICK_READBACK_DEADLINE: Duration = Duration::from_millis(400);+const DOM_CLICK_READBACK_INTERVAL: Duration = Duration::from_millis(25);++struct DomClickObservation {+ effect: ActionEffect,+ readback_available: bool,+}++async fn read_dom_click_state(+ conn: &CdpConnection,+ cdp: &str,+ object_id: &str,+ budget: Duration,+) -> Option<Value> {+ tokio::time::timeout(+ budget,+ conn.call(+ Some(cdp),+ "Runtime.callFunctionOn",+ json!({+ "objectId": object_id,+ "functionDeclaration": DOM_CLICK_READBACK,+ "returnByValue": true,+ }),+ ),+ )+ .await+ .ok()+ .and_then(Result::ok)+ .and_then(|reply| reply.pointer("/result/value").cloned())+ .filter(Value::is_object)+}++fn dom_click_state_changed(before: &Value, after: &Value) -> bool {+ before+ .as_object()+ .zip(after.as_object())+ .is_some_and(|(before, after)| before != after)+}++async fn observe_dom_click(+ conn: &CdpConnection,+ cdp: &str,+ object_id: &str,+ before: Option<Value>,+) -> DomClickObservation {+ let Some(before) = before else {+ return DomClickObservation {+ effect: effect_from_value_readback(false, false, false),+ readback_available: false,+ };+ };+ let started = Instant::now();+ loop {+ if started.elapsed() >= DOM_CLICK_READBACK_DEADLINE {+ return DomClickObservation {+ effect: ActionEffect::Unverifiable,+ readback_available: true,+ };+ }+ if let Some(after) = read_dom_click_state(+ conn,+ cdp,+ object_id,+ DOM_CLICK_READBACK_DEADLINE.saturating_sub(started.elapsed()),+ )+ .await+ {+ if dom_click_state_changed(&before, &after) {+ return DomClickObservation {+ effect: effect_from_value_readback(true, true, true),+ readback_available: true,+ };+ }+ } else {+ return DomClickObservation {+ effect: effect_from_value_readback(false, false, false),+ readback_available: false,+ };+ }+ tokio::time::sleep(DOM_CLICK_READBACK_INTERVAL).await;+ }+}++fn dom_click_action_record(observation: &DomClickObservation) -> ActionExecutionRecord {+ let mut builder = ActionExecutionRecord::builder(+ observation.effect,+ ActionTransport::BrowserCdpRuntimeFunction,+ RequestedDelivery::NotApplicable,+ )+ .actual_delivery(ActualDelivery::Background);+ if observation.readback_available {+ builder = builder.evidence(ActionEvidence {+ kind: EvidenceKind::BrowserReadback,+ detail: if observation.effect == ActionEffect::Confirmed {+ "bounded semantic target state changed".to_owned()+ } else {+ "bounded semantic target state remained unchanged".to_owned()+ },+ });+ }+ if observation.effect != ActionEffect::Confirmed {+ builder = builder.escalation(ActionEscalation {+ kind: EscalationKind::RetryWithPageAction,+ detail: Some(if observation.readback_available {+ "bounded semantic target state remained unchanged; application effect is unverifiable"+ .to_owned()+ } else {+ "semantic target readback was unavailable".to_owned()+ }),+ });+ }+ builder+ .build()+ .expect("DOM click action record must satisfy the stable contract")+}+/// The public caller session id, falling back to the daemon's internal mirror.fn session_of(args: &Value) -> String {args.opt_str("session")@@ -1259,6 +1402,8 @@ impl Tool for BrowserClickTool {.await;}}+ let before =+ read_dom_click_state(conn, cdp, &object_id, DOM_CLICK_READBACK_DEADLINE).await;return match conn.call(Some(cdp),@@ -1270,25 +1415,30 @@ impl Tool for BrowserClickTool {).await{- Ok(_) => ToolResult::text(format!(- "dispatched synthetic DOM click on {} in {tab_id}; application effect not \- verified (trust-gated controls may ignore untrusted events). Refresh page \- state and verify the expected postcondition",- ext_ref.as_deref().unwrap_or("?")- ))- .with_structured(json!({- "status": "ok",- "effect": "unverifiable",- "route": "dom_event",- "target_id": target_id,- "tab_id": tab_id,- "ref": ext_ref,- "frame": frame_kind,- "escalation": {- "recommended": "page",- "reason": "synthetic DOM dispatch cannot prove control activation; refresh page state and verify the expected postcondition",- },- })),+ Ok(_) => {+ let observation = observe_dom_click(conn, cdp, &object_id, before).await;+ let text = match observation.effect {+ ActionEffect::Confirmed => format!(+ "dispatched synthetic DOM click on {} in {tab_id}; bounded semantic \+ readback confirmed a target-state change",+ ext_ref.as_deref().unwrap_or("?")+ ),+ _ if observation.readback_available => format!(+ "dispatched synthetic DOM click on {} in {tab_id}; bounded semantic \+ target state remained unchanged; application effect is unverifiable. \+ Refresh page state and verify the expected postcondition",+ ext_ref.as_deref().unwrap_or("?")+ ),+ _ => format!(+ "dispatched synthetic DOM click on {} in {tab_id}; application effect \+ not verified because semantic target readback was unavailable. Refresh \+ page state and verify the expected postcondition",+ ext_ref.as_deref().unwrap_or("?")+ ),+ };+ let action_record = dom_click_action_record(&observation);+ ToolResult::text(text).with_action_record(action_record)+ }Err(e) => ToolResult::error(format!("DOM click failed: {e}")),};}diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/browser/v2_tests.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/browser/v2_tests.rsindex 14683b10f0..d5f50d2a6e 100644--- a/libs/cua-driver/rust/crates/cua-driver-core/src/browser/v2_tests.rs+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/browser/v2_tests.rs@@ -76,6 +76,12 @@ pub(super) struct FixtureState {omit_primary_target: bool,/// Make `Browser.getWindowForTarget` for T1 answer this CDP error.primary_window_error: Option<(i64, String)>,+ dom_click_changes_readback: bool,+ dom_click_readback_available: bool,+ dom_click_stall_after: Option<usize>,+ dom_click_generation: u64,+ dom_click_focused: bool,+ dom_click_reads_before_failure: Option<usize>,/// Every incoming CDP call: (sessionId, method, params).calls: Vec<(Option<String>, String, Value)>,}@@ -109,6 +115,12 @@ impl Default for FixtureState {viewport_css_height: 600.0,tab_visible: true,select_outcome: Value::Null,+ dom_click_changes_readback: true,+ dom_click_readback_available: true,+ dom_click_stall_after: None,+ dom_click_generation: 0,+ dom_click_focused: false,+ dom_click_reads_before_failure: None,calls: Vec::new(),}}@@ -706,7 +718,44 @@ fn fixture_handler(state: SharedState) -> MockHandler {{MockReply::ok(json!({ "result": { "value": st.select_outcome.clone() } }))}- "Runtime.callFunctionOn" => MockReply::ok(json!({ "result": { "value": true } })),+ "Runtime.callFunctionOn"+ if call.params["functionDeclaration"]+ .as_str()+ .is_some_and(|function| function.contains("ariaPressed")) =>+ {+ if let Some(remaining) = st.dom_click_stall_after.as_mut() {+ if *remaining == 0 {+ let mut reply = MockReply::ok(json!({}));+ reply.suppress_response = true;+ return reply;+ }+ *remaining -= 1;+ }+ if let Some(remaining) = st.dom_click_reads_before_failure.as_mut() {+ if *remaining == 0 {+ return MockReply::err(-32000, "execution context destroyed");+ }+ *remaining -= 1;+ }+ if !st.dom_click_readback_available {+ return MockReply::err(-32000, "semantic click readback unavailable");+ }+ assert!(!call.params["functionDeclaration"]+ .as_str()+ .unwrap()+ .contains("activeElement"));+ let value = json!({+ "innerText": format!("counter={}", st.dom_click_generation)+ });+ MockReply::ok(json!({ "result": { "value": value } }))+ }+ "Runtime.callFunctionOn" => {+ st.dom_click_focused = true;+ if st.dom_click_changes_readback {+ st.dom_click_generation += 1;+ }+ MockReply::ok(json!({ "result": { "value": true } }))+ }other => MockReply::method_not_found(other),}})@@ -1322,10 +1371,12 @@ async fn approved_existing_profile_tools_stay_within_the_reviewed_cdp_surface()})).await;assert_eq!(- structured(&clicked)["status"],- "ok",- "{}",- structured(&clicked)+ clicked+ .action_record+ .as_ref()+ .expect("click action record")+ .effect,+ crate::action_record::ActionEffect::Confirmed);let typed = BrowserTypeTool::new(f.engine.clone()).invoke(json!({@@ -2321,10 +2372,8 @@ async fn browser_visual_feedback_probes_live_tab_visibility_without_activating_i.await;assert_eq!(- structured(&result)["status"],- "ok",- "{}",- structured(&result)+ result.action_record.as_ref().map(|record| record.effect),+ Some(crate::action_record::ActionEffect::Confirmed));let visibility = recorded_calls(&f, "Runtime.evaluate");assert_eq!(visibility.len(), 1, "{visibility:?}");@@ -2878,34 +2927,25 @@ async fn trusted_click_refuses_when_standalone_background_posture_is_unavailable"input_route": "dom_event", "session": SESSION});let synthetic = BrowserClickTool::new(f.engine.clone())- .invoke(synthetic_args.clone())+ .invoke(synthetic_args).await;- assert_eq!(structured(&synthetic)["status"], "ok");- assert_eq!(structured(&synthetic)["effect"], "unverifiable");- assert_eq!(structured(&synthetic)["escalation"]["recommended"], "page");assert!(synthetic.content.iter().any(|content| matches!(content,crate::protocol::Content::Text { text, .. }- if text.contains("application effect not verified")- && text.contains("trust-gated controls")+ if text.contains("confirmed a target-state change"))));- let public = ActionExecutionRecord::from_legacy(- "browser_click",- &synthetic_args,- structured(&synthetic),- )- .expect("browser click action record")- .public_result()- .expect("public browser click result");+ let public = synthetic+ .action_record+ .as_ref()+ .expect("typed browser click action record")+ .public_result()+ .expect("public browser click result");let public = serde_json::to_value(public).expect("serialize public result");- assert_eq!(public["effect"], "unverifiable", "{public}");+ assert_eq!(public["effect"], "confirmed", "{public}");assert_eq!(public["route"], "dom", "{public}");assert_eq!(public["delivery"]["mode"], "background", "{public}");- assert_eq!(public["escalation"]["target"], "page", "{public}");- assert_eq!(- public["escalation"]["reason"], "effect_unconfirmed",- "{public}"- );+ assert_eq!(public["evidence"][0]["kind"], "value_readback", "{public}");+ assert!(public["escalation"].is_null(), "{public}");assert!(public.get("status").is_none(), "{public}");let dispatches = recorded_calls(&f, "Runtime.callFunctionOn");let click_sequence = dispatches@@ -2943,6 +2983,148 @@ async fn trusted_click_refuses_when_standalone_background_posture_is_unavailableassert!(recorded_calls(&f, "Target.activateTarget").is_empty());}+#[tokio::test]+async fn dom_event_click_remains_unverifiable_when_only_focus_changes() {+ let f = fixture_with(|state| state.dom_click_changes_readback = false).await;+ assert!(!f.state.lock().unwrap().dom_click_focused);+ let (target, tab) = bind(&f).await;+ let snap = snapshot(&f, &target, &tab).await;+ let main_ref = ref_of(&snap, "main", "main-btn");++ let result = BrowserClickTool::new(f.engine.clone())+ .invoke(json!({+ "target_id": target,+ "tab_id": tab,+ "ref": main_ref,+ "input_route": "dom_event",+ "session": SESSION,+ }))+ .await;++ assert!(result.content.iter().any(|content| matches!(+ content,+ Content::Text { text, .. }+ if text.contains("target state remained unchanged; application effect is unverifiable")+ )));+ let public = result+ .action_record+ .as_ref()+ .expect("typed browser click action record")+ .public_result()+ .expect("public browser click result");+ let public = serde_json::to_value(public).expect("serialize public result");+ assert!(f.state.lock().unwrap().dom_click_focused);+ assert_eq!(public["effect"], "unverifiable", "{public}");+ assert_eq!(public["evidence"][0]["kind"], "value_readback", "{public}");+ assert_eq!(public["escalation"]["target"], "page", "{public}");+ assert_eq!(+ public["escalation"]["reason"], "effect_unconfirmed",+ "{public}"+ );+}++#[tokio::test]+async fn dom_event_click_remains_unverifiable_when_semantic_readback_is_unavailable() {+ let f = fixture_with(|state| state.dom_click_readback_available = false).await;+ let (target, tab) = bind(&f).await;+ let snap = snapshot(&f, &target, &tab).await;+ let main_ref = ref_of(&snap, "main", "main-btn");++ let result = BrowserClickTool::new(f.engine.clone())+ .invoke(json!({+ "target_id": target,+ "tab_id": tab,+ "ref": main_ref,+ "input_route": "dom_event",+ "session": SESSION,+ }))+ .await;++ assert!(result.content.iter().any(|content| matches!(+ content,+ Content::Text { text, .. }+ if text.contains("semantic target readback was unavailable")+ )));+ let public = result+ .action_record+ .as_ref()+ .expect("typed browser click action record")+ .public_result()+ .expect("public browser click result");+ let public = serde_json::to_value(public).expect("serialize public result");+ assert_eq!(public["effect"], "unverifiable", "{public}");+ assert!(public["evidence"].is_null(), "{public}");+ assert_eq!(+ public["escalation"]["reason"], "effect_unconfirmed",+ "{public}"+ );+}++#[tokio::test]+async fn dom_event_click_bounds_stalled_readbacks() {+ for reads in [0, 1, 2] {+ let f = fixture_with(|state| {+ state.dom_click_stall_after = Some(reads);+ state.dom_click_changes_readback = false;+ })+ .await;+ let (target, tab) = bind(&f).await;+ let snap = snapshot(&f, &target, &tab).await;+ let main_ref = ref_of(&snap, "main", "main-btn");+ let result = tokio::time::timeout(+ std::time::Duration::from_secs(2),+ BrowserClickTool::new(f.engine.clone()).invoke(json!({+ "target_id": target,+ "tab_id": tab,+ "ref": main_ref,+ "input_route": "dom_event",+ "session": SESSION,+ })),+ )+ .await+ .expect("click must not wait for CALL_TIMEOUT");+ assert!(result.content.iter().any(|content| matches!(+ content,+ Content::Text { text, .. }+ if text.contains("semantic target readback was unavailable")+ )));+ let public = result.action_record.unwrap().public_result().unwrap();+ let public = serde_json::to_value(public).unwrap();+ assert_eq!(public["effect"], "unverifiable");+ assert!(public["evidence"].is_null());+ assert_eq!(public["escalation"]["reason"], "effect_unconfirmed");+ }+}++#[tokio::test]+async fn dom_event_click_is_unverifiable_when_readback_disappears() {+ let f = fixture_with(|state| {+ state.dom_click_changes_readback = false;+ state.dom_click_reads_before_failure = Some(2);+ })+ .await;+ let (target, tab) = bind(&f).await;+ let snap = snapshot(&f, &target, &tab).await;+ let main_ref = ref_of(&snap, "main", "main-btn");+ let result = BrowserClickTool::new(f.engine.clone())+ .invoke(json!({+ "target_id": target,+ "tab_id": tab,+ "ref": main_ref,+ "input_route": "dom_event",+ "session": SESSION,+ }))+ .await;+ let public = result.action_record.unwrap().public_result().unwrap();+ let public = serde_json::to_value(public).unwrap();+ assert_eq!(public["effect"], "unverifiable", "{public}");+ assert!(public["evidence"].is_null(), "{public}");+ assert_eq!(+ f.state.lock().unwrap().dom_click_reads_before_failure,+ Some(0)+ );+}+#[tokio::test]async fn main_frame_navigation_invalidates_refs_via_loader_identity() {let f = fixture().await;diff --git a/libs/cua-driver/rust/crates/cua-driver-e2e/tests/standalone_browser_behavior_test.rs b/libs/cua-driver/rust/crates/cua-driver-e2e/tests/standalone_browser_behavior_test.rsindex f8b1e240f0..a88bcab775 100644--- a/libs/cua-driver/rust/crates/cua-driver-e2e/tests/standalone_browser_behavior_test.rs+++ b/libs/cua-driver/rust/crates/cua-driver-e2e/tests/standalone_browser_behavior_test.rs@@ -1878,7 +1878,19 @@ fn run_roundtrip(spec: &BrowserSpec) {"session": session,}),);+ assert_ne!(+ click.action_effect(),+ Some("suspected_noop"),+ "{}",+ click.raw+ );assert_eq!(click.action_effect(), Some("unverifiable"), "{}", click.raw);+ assert_eq!(+ click.structured()["evidence"][0]["kind"],+ "value_readback",+ "{}",+ click.raw+ );wait_for_text(&fixture.server, "lbl-counter", "counter=1");let snapshot = fixture.driver.call(@@ -1955,11 +1967,13 @@ fn run_trust_gated_dom_click(spec: &BrowserSpec) {click.raw);assert!(- click.text().contains("application effect not verified")- && click.text().contains("trust-gated controls"),+ click.text().contains(+ "target state remained unchanged; application effect is unverifiable"+ ),"{}",click.raw);+ assert_eq!(click.structured()["evidence"][0]["kind"], "value_readback");wait_for_text(&fixture.server,"standalone-trust-gated-state",