Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

fix(cua-driver): keep a full budget for each dom_event click readback (#6519)

injaneity21efdf32ff36
diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/browser/mock_cdp.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/browser/mock_cdp.rs
index d144ea4a00..7794e8664d 100644
--- a/libs/cua-driver/rust/crates/cua-driver-core/src/browser/mock_cdp.rs
+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/browser/mock_cdp.rs
@@ -37,6 +37,8 @@ pub(crate) struct MockEvent {
pub(crate) struct MockReply {
pub events: Vec<MockEvent>,
pub suppress_response: bool,
+ /// Hold the reply this long before sending it (a slow but live page).
+ pub delay: Option<std::time::Duration>,
pub result: Result<Value, (i64, String)>,
}
@@ -46,6 +48,7 @@ impl MockReply {
events: Vec::new(),
result: Ok(result),
suppress_response: false,
+ delay: None,
}
}
@@ -55,6 +58,7 @@ impl MockReply {
events: Vec::new(),
result: Ok(json!({})),
suppress_response: true,
+ delay: None,
}
}
@@ -63,6 +67,7 @@ impl MockReply {
events: Vec::new(),
result: Err((code, message.to_owned())),
suppress_response: false,
+ delay: None,
}
}
@@ -142,6 +147,9 @@ impl MockCdpServer {
if reply.suppress_response {
continue;
}
+ if let Some(delay) = reply.delay {
+ tokio::time::sleep(delay).await;
+ }
let mut response = match reply.result {
Ok(result) => json!({ "id": id, "result": result }),
Err((code, message)) => {
diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/browser/tools.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/browser/tools.rs
index 4724a62cb2..e589d3cbf2 100644
--- a/libs/cua-driver/rust/crates/cua-driver-core/src/browser/tools.rs
+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/browser/tools.rs
@@ -139,13 +139,12 @@ async fn observe_dom_click(
readback_available: true,
};
}
- if let Some(after) = read_dom_click_state(
- conn,
- cdp,
- object_id,
- DOM_CLICK_READBACK_DEADLINE.saturating_sub(started.elapsed()),
- )
- .await
+ // The window only decides whether to start another read; each read
+ // keeps a full budget. Clipping an in-flight read to the window's
+ // remainder reported a slow-but-answering page as "readback
+ // unavailable" instead of "state unchanged".
+ if let Some(after) =
+ read_dom_click_state(conn, cdp, object_id, DOM_CLICK_READBACK_DEADLINE).await
{
if dom_click_state_changed(&before, &after) {
return DomClickObservation {
diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/browser/v2_tests.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/browser/v2_tests.rs
index 935a8bf9ed..1b542a4d6e 100644
--- a/libs/cua-driver/rust/crates/cua-driver-core/src/browser/v2_tests.rs
+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/browser/v2_tests.rs
@@ -97,6 +97,8 @@ pub(super) struct FixtureState {
dom_click_generation: u64,
dom_click_focused: bool,
dom_click_reads_before_failure: Option<usize>,
+ /// Answer every post-snapshot semantic readback after this delay.
+ dom_click_readback_delay: Option<std::time::Duration>,
/// Every incoming CDP call: (sessionId, method, params).
calls: Vec<(Option<String>, String, Value)>,
}
@@ -140,6 +142,7 @@ impl Default for FixtureState {
dom_click_generation: 0,
dom_click_focused: false,
dom_click_reads_before_failure: None,
+ dom_click_readback_delay: None,
calls: Vec::new(),
}
}
@@ -783,7 +786,9 @@ fn fixture_handler(state: SharedState) -> MockHandler {
let value = json!({
"innerText": format!("counter={}", st.dom_click_generation)
});
- MockReply::ok(json!({ "result": { "value": value } }))
+ let mut reply = MockReply::ok(json!({ "result": { "value": value } }));
+ reply.delay = st.dom_click_readback_delay;
+ reply
}
"Runtime.callFunctionOn" => {
st.dom_click_focused = true;
@@ -3259,6 +3264,40 @@ async fn dom_event_click_bounds_stalled_readbacks() {
}
}
+#[tokio::test]
+async fn dom_event_click_keeps_readback_when_reads_are_slow() {
+ // Each read answers in 150 ms, so the third post-click read starts with
+ // only ~50 ms of the 400 ms window left. It must still be waited for:
+ // a slow page whose state did not change is "unchanged", not
+ // "readback unavailable".
+ let f = fixture_with(|state| {
+ state.dom_click_changes_readback = false;
+ state.dom_click_readback_delay = Some(std::time::Duration::from_millis(150));
+ })
+ .await;
+ let (target, tab) = bind(&f).await;
+ let snap = snapshot(&f, &target, &tab).await;
+ let main_ref = ref_of(&snap, "main", "main-btn");
+ let result = BrowserClickTool::new(f.engine.clone())
+ .invoke(json!({
+ "target_id": target,
+ "tab_id": tab,
+ "ref": main_ref,
+ "input_route": "dom_event",
+ "session": SESSION,
+ }))
+ .await;
+ assert!(result.content.iter().any(|content| matches!(
+ content,
+ Content::Text { text, .. }
+ if text.contains("target state remained unchanged; application effect is unverifiable")
+ )));
+ let public = result.action_record.unwrap().public_result().unwrap();
+ let public = serde_json::to_value(public).unwrap();
+ assert_eq!(public["effect"], "unverifiable", "{public}");
+ assert_eq!(public["evidence"][0]["kind"], "value_readback", "{public}");
+}
+
#[tokio::test]
async fn dom_event_click_is_unverifiable_when_readback_disappears() {
let f = fixture_with(|state| {