Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

fix(cua-driver): classify DOM click effects with readback

injaneity1bf24b83ed9d
diff --git a/libs/cua-driver/docs/action-result-contract.md b/libs/cua-driver/docs/action-result-contract.md
index d382d8c490..8d48afd894 100644
--- a/libs/cua-driver/docs/action-result-contract.md
+++ b/libs/cua-driver/docs/action-result-contract.md
@@ -123,6 +123,13 @@ in the same string. It is advice only: it has no `next_call`, it never rides on
an error, and it is never sent for a step that already runs inside `run_steps`
or `run_script`. The logic is in `rust/crates/cua-driver-core/src/batch_nudge.rs`.
+`suspected_noop` is advisory. It means a bounded readable state remained
+unchanged, not that the action failed. For a ref-targeted browser DOM click,
+the readback covers bounded semantic and form state on the target plus the
+document URL. An unchanged DOM click readback remains `unverifiable`, with
+readback evidence: effects elsewhere on the page can be invisible to that
+readback, so verify the expected postcondition before retrying.
+
## Window target resolution
Window-scoped actions accept a PID without `window_id` only when that process
diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/browser/mock_cdp.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/browser/mock_cdp.rs
index 404ddd2a30..a375cab7da 100644
--- a/libs/cua-driver/rust/crates/cua-driver-core/src/browser/mock_cdp.rs
+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/browser/mock_cdp.rs
@@ -33,6 +33,7 @@ pub(crate) struct MockEvent {
/// The scripted reaction to one command.
pub(crate) struct MockReply {
pub events: Vec<MockEvent>,
+ pub suppress_response: bool,
pub result: Result<Value, (i64, String)>,
}
@@ -41,6 +42,7 @@ impl MockReply {
Self {
events: Vec::new(),
result: Ok(result),
+ suppress_response: false,
}
}
@@ -48,6 +50,7 @@ impl MockReply {
Self {
events: Vec::new(),
result: Err((code, message.to_owned())),
+ suppress_response: false,
}
}
@@ -114,6 +117,9 @@ impl MockCdpServer {
.map(str::to_owned),
};
let reply = handler(&call);
+ if reply.suppress_response {
+ continue;
+ }
for event in reply.events {
let mut frame =
json!({ "method": event.method, "params": event.params });
diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/browser/tools.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/browser/tools.rs
index 7b31ed1a8e..df044019dd 100644
--- a/libs/cua-driver/rust/crates/cua-driver-core/src/browser/tools.rs
+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/browser/tools.rs
@@ -6,10 +6,16 @@
//! `{"status": "ok" | "refused", ...}`.
use std::sync::Arc;
+use std::time::{Duration, Instant};
use async_trait::async_trait;
use serde_json::{json, Value};
+use crate::action_record::{
+ effect_from_value_readback, ActionEffect, ActionEscalation, ActionEvidence,
+ ActionExecutionRecord, ActionTransport, ActualDelivery, EscalationKind, EvidenceKind,
+ RequestedDelivery,
+};
use crate::protocol::{Content, ToolResult};
use crate::tool::{ProtectedResourceOwnership, Tool, ToolDef, ToolRegistry};
use crate::tool_args::ArgsExt;
@@ -45,6 +51,143 @@ pub fn register_browser_tools(engine: &Arc<BrowserEngine>, registry: &mut ToolRe
// ── Shared helpers ───────────────────────────────────────────────────────────
+const DOM_CLICK_READBACK: &str = r#"function() {
+ const bounded = value => typeof value === 'string' ? value.slice(0, 512) : null;
+ const attr = name => bounded(this.getAttribute(name));
+ return {
+ ariaPressed: attr('aria-pressed'),
+ ariaSelected: attr('aria-selected'),
+ ariaChecked: attr('aria-checked'),
+ ariaExpanded: attr('aria-expanded'),
+ ariaCurrent: attr('aria-current'),
+ ariaDisabled: attr('aria-disabled'),
+ disabled: 'disabled' in this ? Boolean(this.disabled) : null,
+ checked: 'checked' in this ? Boolean(this.checked) : null,
+ value: 'value' in this ? bounded(String(this.value)) : null,
+ innerText: bounded(this.innerText),
+ href: this.ownerDocument && this.ownerDocument.location
+ ? bounded(this.ownerDocument.location.href)
+ : null
+ };
+}"#;
+
+const DOM_CLICK_READBACK_DEADLINE: Duration = Duration::from_millis(400);
+const DOM_CLICK_READBACK_INTERVAL: Duration = Duration::from_millis(25);
+
+struct DomClickObservation {
+ effect: ActionEffect,
+ readback_available: bool,
+}
+
+async fn read_dom_click_state(
+ conn: &CdpConnection,
+ cdp: &str,
+ object_id: &str,
+ budget: Duration,
+) -> Option<Value> {
+ tokio::time::timeout(
+ budget,
+ conn.call(
+ Some(cdp),
+ "Runtime.callFunctionOn",
+ json!({
+ "objectId": object_id,
+ "functionDeclaration": DOM_CLICK_READBACK,
+ "returnByValue": true,
+ }),
+ ),
+ )
+ .await
+ .ok()
+ .and_then(Result::ok)
+ .and_then(|reply| reply.pointer("/result/value").cloned())
+ .filter(Value::is_object)
+}
+
+fn dom_click_state_changed(before: &Value, after: &Value) -> bool {
+ before
+ .as_object()
+ .zip(after.as_object())
+ .is_some_and(|(before, after)| before != after)
+}
+
+async fn observe_dom_click(
+ conn: &CdpConnection,
+ cdp: &str,
+ object_id: &str,
+ before: Option<Value>,
+) -> DomClickObservation {
+ let Some(before) = before else {
+ return DomClickObservation {
+ effect: effect_from_value_readback(false, false, false),
+ readback_available: false,
+ };
+ };
+ let started = Instant::now();
+ loop {
+ if started.elapsed() >= DOM_CLICK_READBACK_DEADLINE {
+ return DomClickObservation {
+ effect: ActionEffect::Unverifiable,
+ readback_available: true,
+ };
+ }
+ if let Some(after) = read_dom_click_state(
+ conn,
+ cdp,
+ object_id,
+ DOM_CLICK_READBACK_DEADLINE.saturating_sub(started.elapsed()),
+ )
+ .await
+ {
+ if dom_click_state_changed(&before, &after) {
+ return DomClickObservation {
+ effect: effect_from_value_readback(true, true, true),
+ readback_available: true,
+ };
+ }
+ } else {
+ return DomClickObservation {
+ effect: effect_from_value_readback(false, false, false),
+ readback_available: false,
+ };
+ }
+ tokio::time::sleep(DOM_CLICK_READBACK_INTERVAL).await;
+ }
+}
+
+fn dom_click_action_record(observation: &DomClickObservation) -> ActionExecutionRecord {
+ let mut builder = ActionExecutionRecord::builder(
+ observation.effect,
+ ActionTransport::BrowserCdpRuntimeFunction,
+ RequestedDelivery::NotApplicable,
+ )
+ .actual_delivery(ActualDelivery::Background);
+ if observation.readback_available {
+ builder = builder.evidence(ActionEvidence {
+ kind: EvidenceKind::BrowserReadback,
+ detail: if observation.effect == ActionEffect::Confirmed {
+ "bounded semantic target state changed".to_owned()
+ } else {
+ "bounded semantic target state remained unchanged".to_owned()
+ },
+ });
+ }
+ if observation.effect != ActionEffect::Confirmed {
+ builder = builder.escalation(ActionEscalation {
+ kind: EscalationKind::RetryWithPageAction,
+ detail: Some(if observation.readback_available {
+ "bounded semantic target state remained unchanged; application effect is unverifiable"
+ .to_owned()
+ } else {
+ "semantic target readback was unavailable".to_owned()
+ }),
+ });
+ }
+ builder
+ .build()
+ .expect("DOM click action record must satisfy the stable contract")
+}
+
/// The public caller session id, falling back to the daemon's internal mirror.
fn session_of(args: &Value) -> String {
args.opt_str("session")
@@ -1259,6 +1402,8 @@ impl Tool for BrowserClickTool {
.await;
}
}
+ let before =
+ read_dom_click_state(conn, cdp, &object_id, DOM_CLICK_READBACK_DEADLINE).await;
return match conn
.call(
Some(cdp),
@@ -1270,25 +1415,30 @@ impl Tool for BrowserClickTool {
)
.await
{
- Ok(_) => ToolResult::text(format!(
- "dispatched synthetic DOM click on {} in {tab_id}; application effect not \
- verified (trust-gated controls may ignore untrusted events). Refresh page \
- state and verify the expected postcondition",
- ext_ref.as_deref().unwrap_or("?")
- ))
- .with_structured(json!({
- "status": "ok",
- "effect": "unverifiable",
- "route": "dom_event",
- "target_id": target_id,
- "tab_id": tab_id,
- "ref": ext_ref,
- "frame": frame_kind,
- "escalation": {
- "recommended": "page",
- "reason": "synthetic DOM dispatch cannot prove control activation; refresh page state and verify the expected postcondition",
- },
- })),
+ Ok(_) => {
+ let observation = observe_dom_click(conn, cdp, &object_id, before).await;
+ let text = match observation.effect {
+ ActionEffect::Confirmed => format!(
+ "dispatched synthetic DOM click on {} in {tab_id}; bounded semantic \
+ readback confirmed a target-state change",
+ ext_ref.as_deref().unwrap_or("?")
+ ),
+ _ if observation.readback_available => format!(
+ "dispatched synthetic DOM click on {} in {tab_id}; bounded semantic \
+ target state remained unchanged; application effect is unverifiable. \
+ Refresh page state and verify the expected postcondition",
+ ext_ref.as_deref().unwrap_or("?")
+ ),
+ _ => format!(
+ "dispatched synthetic DOM click on {} in {tab_id}; application effect \
+ not verified because semantic target readback was unavailable. Refresh \
+ page state and verify the expected postcondition",
+ ext_ref.as_deref().unwrap_or("?")
+ ),
+ };
+ let action_record = dom_click_action_record(&observation);
+ ToolResult::text(text).with_action_record(action_record)
+ }
Err(e) => ToolResult::error(format!("DOM click failed: {e}")),
};
}
diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/browser/v2_tests.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/browser/v2_tests.rs
index 14683b10f0..d5f50d2a6e 100644
--- a/libs/cua-driver/rust/crates/cua-driver-core/src/browser/v2_tests.rs
+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/browser/v2_tests.rs
@@ -76,6 +76,12 @@ pub(super) struct FixtureState {
omit_primary_target: bool,
/// Make `Browser.getWindowForTarget` for T1 answer this CDP error.
primary_window_error: Option<(i64, String)>,
+ dom_click_changes_readback: bool,
+ dom_click_readback_available: bool,
+ dom_click_stall_after: Option<usize>,
+ dom_click_generation: u64,
+ dom_click_focused: bool,
+ dom_click_reads_before_failure: Option<usize>,
/// Every incoming CDP call: (sessionId, method, params).
calls: Vec<(Option<String>, String, Value)>,
}
@@ -109,6 +115,12 @@ impl Default for FixtureState {
viewport_css_height: 600.0,
tab_visible: true,
select_outcome: Value::Null,
+ dom_click_changes_readback: true,
+ dom_click_readback_available: true,
+ dom_click_stall_after: None,
+ dom_click_generation: 0,
+ dom_click_focused: false,
+ dom_click_reads_before_failure: None,
calls: Vec::new(),
}
}
@@ -706,7 +718,44 @@ fn fixture_handler(state: SharedState) -> MockHandler {
{
MockReply::ok(json!({ "result": { "value": st.select_outcome.clone() } }))
}
- "Runtime.callFunctionOn" => MockReply::ok(json!({ "result": { "value": true } })),
+ "Runtime.callFunctionOn"
+ if call.params["functionDeclaration"]
+ .as_str()
+ .is_some_and(|function| function.contains("ariaPressed")) =>
+ {
+ if let Some(remaining) = st.dom_click_stall_after.as_mut() {
+ if *remaining == 0 {
+ let mut reply = MockReply::ok(json!({}));
+ reply.suppress_response = true;
+ return reply;
+ }
+ *remaining -= 1;
+ }
+ if let Some(remaining) = st.dom_click_reads_before_failure.as_mut() {
+ if *remaining == 0 {
+ return MockReply::err(-32000, "execution context destroyed");
+ }
+ *remaining -= 1;
+ }
+ if !st.dom_click_readback_available {
+ return MockReply::err(-32000, "semantic click readback unavailable");
+ }
+ assert!(!call.params["functionDeclaration"]
+ .as_str()
+ .unwrap()
+ .contains("activeElement"));
+ let value = json!({
+ "innerText": format!("counter={}", st.dom_click_generation)
+ });
+ MockReply::ok(json!({ "result": { "value": value } }))
+ }
+ "Runtime.callFunctionOn" => {
+ st.dom_click_focused = true;
+ if st.dom_click_changes_readback {
+ st.dom_click_generation += 1;
+ }
+ MockReply::ok(json!({ "result": { "value": true } }))
+ }
other => MockReply::method_not_found(other),
}
})
@@ -1322,10 +1371,12 @@ async fn approved_existing_profile_tools_stay_within_the_reviewed_cdp_surface()
}))
.await;
assert_eq!(
- structured(&clicked)["status"],
- "ok",
- "{}",
- structured(&clicked)
+ clicked
+ .action_record
+ .as_ref()
+ .expect("click action record")
+ .effect,
+ crate::action_record::ActionEffect::Confirmed
);
let typed = BrowserTypeTool::new(f.engine.clone())
.invoke(json!({
@@ -2321,10 +2372,8 @@ async fn browser_visual_feedback_probes_live_tab_visibility_without_activating_i
.await;
assert_eq!(
- structured(&result)["status"],
- "ok",
- "{}",
- structured(&result)
+ result.action_record.as_ref().map(|record| record.effect),
+ Some(crate::action_record::ActionEffect::Confirmed)
);
let visibility = recorded_calls(&f, "Runtime.evaluate");
assert_eq!(visibility.len(), 1, "{visibility:?}");
@@ -2878,34 +2927,25 @@ async fn trusted_click_refuses_when_standalone_background_posture_is_unavailable
"input_route": "dom_event", "session": SESSION
});
let synthetic = BrowserClickTool::new(f.engine.clone())
- .invoke(synthetic_args.clone())
+ .invoke(synthetic_args)
.await;
- assert_eq!(structured(&synthetic)["status"], "ok");
- assert_eq!(structured(&synthetic)["effect"], "unverifiable");
- assert_eq!(structured(&synthetic)["escalation"]["recommended"], "page");
assert!(synthetic.content.iter().any(|content| matches!(
content,
crate::protocol::Content::Text { text, .. }
- if text.contains("application effect not verified")
- && text.contains("trust-gated controls")
+ if text.contains("confirmed a target-state change")
)));
- let public = ActionExecutionRecord::from_legacy(
- "browser_click",
- &synthetic_args,
- structured(&synthetic),
- )
- .expect("browser click action record")
- .public_result()
- .expect("public browser click result");
+ let public = synthetic
+ .action_record
+ .as_ref()
+ .expect("typed browser click action record")
+ .public_result()
+ .expect("public browser click result");
let public = serde_json::to_value(public).expect("serialize public result");
- assert_eq!(public["effect"], "unverifiable", "{public}");
+ assert_eq!(public["effect"], "confirmed", "{public}");
assert_eq!(public["route"], "dom", "{public}");
assert_eq!(public["delivery"]["mode"], "background", "{public}");
- assert_eq!(public["escalation"]["target"], "page", "{public}");
- assert_eq!(
- public["escalation"]["reason"], "effect_unconfirmed",
- "{public}"
- );
+ assert_eq!(public["evidence"][0]["kind"], "value_readback", "{public}");
+ assert!(public["escalation"].is_null(), "{public}");
assert!(public.get("status").is_none(), "{public}");
let dispatches = recorded_calls(&f, "Runtime.callFunctionOn");
let click_sequence = dispatches
@@ -2943,6 +2983,148 @@ async fn trusted_click_refuses_when_standalone_background_posture_is_unavailable
assert!(recorded_calls(&f, "Target.activateTarget").is_empty());
}
+#[tokio::test]
+async fn dom_event_click_remains_unverifiable_when_only_focus_changes() {
+ let f = fixture_with(|state| state.dom_click_changes_readback = false).await;
+ assert!(!f.state.lock().unwrap().dom_click_focused);
+ let (target, tab) = bind(&f).await;
+ let snap = snapshot(&f, &target, &tab).await;
+ let main_ref = ref_of(&snap, "main", "main-btn");
+
+ let result = BrowserClickTool::new(f.engine.clone())
+ .invoke(json!({
+ "target_id": target,
+ "tab_id": tab,
+ "ref": main_ref,
+ "input_route": "dom_event",
+ "session": SESSION,
+ }))
+ .await;
+
+ assert!(result.content.iter().any(|content| matches!(
+ content,
+ Content::Text { text, .. }
+ if text.contains("target state remained unchanged; application effect is unverifiable")
+ )));
+ let public = result
+ .action_record
+ .as_ref()
+ .expect("typed browser click action record")
+ .public_result()
+ .expect("public browser click result");
+ let public = serde_json::to_value(public).expect("serialize public result");
+ assert!(f.state.lock().unwrap().dom_click_focused);
+ assert_eq!(public["effect"], "unverifiable", "{public}");
+ assert_eq!(public["evidence"][0]["kind"], "value_readback", "{public}");
+ assert_eq!(public["escalation"]["target"], "page", "{public}");
+ assert_eq!(
+ public["escalation"]["reason"], "effect_unconfirmed",
+ "{public}"
+ );
+}
+
+#[tokio::test]
+async fn dom_event_click_remains_unverifiable_when_semantic_readback_is_unavailable() {
+ let f = fixture_with(|state| state.dom_click_readback_available = false).await;
+ let (target, tab) = bind(&f).await;
+ let snap = snapshot(&f, &target, &tab).await;
+ let main_ref = ref_of(&snap, "main", "main-btn");
+
+ let result = BrowserClickTool::new(f.engine.clone())
+ .invoke(json!({
+ "target_id": target,
+ "tab_id": tab,
+ "ref": main_ref,
+ "input_route": "dom_event",
+ "session": SESSION,
+ }))
+ .await;
+
+ assert!(result.content.iter().any(|content| matches!(
+ content,
+ Content::Text { text, .. }
+ if text.contains("semantic target readback was unavailable")
+ )));
+ let public = result
+ .action_record
+ .as_ref()
+ .expect("typed browser click action record")
+ .public_result()
+ .expect("public browser click result");
+ let public = serde_json::to_value(public).expect("serialize public result");
+ assert_eq!(public["effect"], "unverifiable", "{public}");
+ assert!(public["evidence"].is_null(), "{public}");
+ assert_eq!(
+ public["escalation"]["reason"], "effect_unconfirmed",
+ "{public}"
+ );
+}
+
+#[tokio::test]
+async fn dom_event_click_bounds_stalled_readbacks() {
+ for reads in [0, 1, 2] {
+ let f = fixture_with(|state| {
+ state.dom_click_stall_after = Some(reads);
+ state.dom_click_changes_readback = false;
+ })
+ .await;
+ let (target, tab) = bind(&f).await;
+ let snap = snapshot(&f, &target, &tab).await;
+ let main_ref = ref_of(&snap, "main", "main-btn");
+ let result = tokio::time::timeout(
+ std::time::Duration::from_secs(2),
+ BrowserClickTool::new(f.engine.clone()).invoke(json!({
+ "target_id": target,
+ "tab_id": tab,
+ "ref": main_ref,
+ "input_route": "dom_event",
+ "session": SESSION,
+ })),
+ )
+ .await
+ .expect("click must not wait for CALL_TIMEOUT");
+ assert!(result.content.iter().any(|content| matches!(
+ content,
+ Content::Text { text, .. }
+ if text.contains("semantic target readback was unavailable")
+ )));
+ let public = result.action_record.unwrap().public_result().unwrap();
+ let public = serde_json::to_value(public).unwrap();
+ assert_eq!(public["effect"], "unverifiable");
+ assert!(public["evidence"].is_null());
+ assert_eq!(public["escalation"]["reason"], "effect_unconfirmed");
+ }
+}
+
+#[tokio::test]
+async fn dom_event_click_is_unverifiable_when_readback_disappears() {
+ let f = fixture_with(|state| {
+ state.dom_click_changes_readback = false;
+ state.dom_click_reads_before_failure = Some(2);
+ })
+ .await;
+ let (target, tab) = bind(&f).await;
+ let snap = snapshot(&f, &target, &tab).await;
+ let main_ref = ref_of(&snap, "main", "main-btn");
+ let result = BrowserClickTool::new(f.engine.clone())
+ .invoke(json!({
+ "target_id": target,
+ "tab_id": tab,
+ "ref": main_ref,
+ "input_route": "dom_event",
+ "session": SESSION,
+ }))
+ .await;
+ let public = result.action_record.unwrap().public_result().unwrap();
+ let public = serde_json::to_value(public).unwrap();
+ assert_eq!(public["effect"], "unverifiable", "{public}");
+ assert!(public["evidence"].is_null(), "{public}");
+ assert_eq!(
+ f.state.lock().unwrap().dom_click_reads_before_failure,
+ Some(0)
+ );
+}
+
#[tokio::test]
async fn main_frame_navigation_invalidates_refs_via_loader_identity() {
let f = fixture().await;
diff --git a/libs/cua-driver/rust/crates/cua-driver-e2e/tests/standalone_browser_behavior_test.rs b/libs/cua-driver/rust/crates/cua-driver-e2e/tests/standalone_browser_behavior_test.rs
index f8b1e240f0..a88bcab775 100644
--- a/libs/cua-driver/rust/crates/cua-driver-e2e/tests/standalone_browser_behavior_test.rs
+++ b/libs/cua-driver/rust/crates/cua-driver-e2e/tests/standalone_browser_behavior_test.rs
@@ -1878,7 +1878,19 @@ fn run_roundtrip(spec: &BrowserSpec) {
"session": session,
}),
);
+ assert_ne!(
+ click.action_effect(),
+ Some("suspected_noop"),
+ "{}",
+ click.raw
+ );
assert_eq!(click.action_effect(), Some("unverifiable"), "{}", click.raw);
+ assert_eq!(
+ click.structured()["evidence"][0]["kind"],
+ "value_readback",
+ "{}",
+ click.raw
+ );
wait_for_text(&fixture.server, "lbl-counter", "counter=1");
let snapshot = fixture.driver.call(
@@ -1955,11 +1967,13 @@ fn run_trust_gated_dom_click(spec: &BrowserSpec) {
click.raw
);
assert!(
- click.text().contains("application effect not verified")
- && click.text().contains("trust-gated controls"),
+ click.text().contains(
+ "target state remained unchanged; application effect is unverifiable"
+ ),
"{}",
click.raw
);
+ assert_eq!(click.structured()["evidence"][0]["kind"], "value_readback");
wait_for_text(
&fixture.server,
"standalone-trust-gated-state",