Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

fix(cua-driver): report PostMessage pixel fallback truthfully

Francesco Bonacci4dc537ef34e7
diff --git a/.github/workflows/ci-rust-windows.yml b/.github/workflows/ci-rust-windows.yml
index 376f881678..63117775fb 100644
--- a/.github/workflows/ci-rust-windows.yml
+++ b/.github/workflows/ci-rust-windows.yml
@@ -71,6 +71,9 @@ jobs:
if ($LASTEXITCODE -ne 0) { throw "PostMessage read-back tests failed with exit $LASTEXITCODE" }
cargo test -p platform-windows value_write_readback_tests:: --lib --locked
if ($LASTEXITCODE -ne 0) { throw "ValuePattern read-back tests failed with exit $LASTEXITCODE" }
+ - name: Run pixel click transport regression
+ working-directory: libs/cua-driver/rust
+ run: cargo test -p platform-windows post_message_pixel_click_reports_synthetic_transport --lib --locked
- name: Run named-pipe readiness regression
working-directory: libs/cua-driver/rust
run: cargo test -p cua-driver-testkit stalled_pipe_attempt_is_bounded_and_does_not_block_the_next_probe --lib --locked
diff --git a/libs/cua-driver/rust/crates/cua-driver/tests/harness_web_test.rs b/libs/cua-driver/rust/crates/cua-driver/tests/harness_web_test.rs
index 1be4e7870e..de06bf2537 100644
--- a/libs/cua-driver/rust/crates/cua-driver/tests/harness_web_test.rs
+++ b/libs/cua-driver/rust/crates/cua-driver/tests/harness_web_test.rs
@@ -305,7 +305,7 @@ fn harness_webview_left_click_px_background() {
"webview2",
"left_click",
Targeting::Px,
- DriverRoute::UiaInvoke,
+ DriverRoute::Composite,
);
let point = Cell::new(None);
run_web_case_with_preparation(
@@ -376,13 +376,13 @@ fn harness_webview_left_click_px_background() {
"WebView2 PX background click failed: {}",
click.text()
);
- assert_eq!(
- click.action_route(),
- Some("accessibility"),
- "WebView2 PX background click used an unexpected driver route: {}",
- click.text()
- );
wait_for_journal_text(journal, "lbl-counter", "counter=3");
+ let route = click.action_route();
+ assert!(
+ matches!(route, Some("accessibility" | "synthetic_events")),
+ "WebView2 PX background click used an unsupported driver route {route:?}: {}",
+ click.text(),
+ );
},
);
}
diff --git a/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs b/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs
index 3d24348ed3..9e0f7effd2 100644
--- a/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs
+++ b/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs
@@ -2809,6 +2809,14 @@ impl Tool for LaunchAppTool {
// ── click ─────────────────────────────────────────────────────────────────────
+fn posted_pixel_click_result(pid: u32, click_word: &str) -> ToolResult {
+ ToolResult::text(format!("✅ Posted {click_word} to pid {pid}.")).with_structured(json!({
+ "path": "post_message",
+ "verified": false,
+ "effect": "unverifiable"
+ }))
+}
+
pub struct ClickTool {
state: Arc<ToolState>,
}
@@ -3783,15 +3791,12 @@ impl Tool for ClickTool {
};
}
- // delivery_mode:"background", non-dropped + non-UIA click — e.g. a
- // right-click or multi-click on a plain Win32 window. Chromium is
- // never reached here: its mouse events are flagged silently-dropped
- // above and handled by the injection path, so the only targets that
- // fall through are ones PostMessage delivers to without a foreground
- // swap. (Foreground was handled at the top; the legacy `auto`
- // Chromium-SendInput heuristic was removed in the macOS-alignment
- // pass — escalating to delivery_mode:"foreground" is now the explicit
- // path for any target PostMessage can't reach.)
+ // delivery_mode:"background", non-dropped + non-UIA click. This
+ // includes plain Win32 targets and embedded Chromium when its UIA
+ // provider is temporarily unavailable. Attempt PostMessage without a
+ // foreground swap; the caller must verify the resulting fixture state.
+ // Foreground was handled at the top, and known dropped surfaces use
+ // the targeted-injection path above.
// bitmap pixels -> screen (DWM-frame origin + inset). Use
// post_click_screen so we don't double-ClientToScreen.
@@ -3807,10 +3812,7 @@ impl Tool for ClickTool {
3 => "triple-click",
_ => "click",
};
- ToolResult::text(format!("✅ Posted {click_word} to pid {pid}."))
- .with_structured(
- json!({ "path": "pixel", "verified": false, "effect": "unverifiable" }),
- )
+ posted_pixel_click_result(pid, click_word)
}
Ok(Err(e)) => ToolResult::error(e.to_string()),
Err(e) => ToolResult::error(format!("Task error: {e}")),
@@ -3822,6 +3824,39 @@ impl Tool for ClickTool {
}
}
+#[cfg(test)]
+mod pixel_click_transport_tests {
+ use super::posted_pixel_click_result;
+ use cua_driver_core::action_record::{
+ ActionExecutionRecord, ActionTransport, ActualDelivery, RequestedDelivery,
+ };
+
+ #[test]
+ fn post_message_pixel_click_reports_synthetic_transport() {
+ let result = posted_pixel_click_result(42, "click");
+ let structured = result
+ .structured_content
+ .as_ref()
+ .expect("posted click should expose legacy transport metadata");
+ assert_eq!(structured["path"], "post_message");
+
+ let record = ActionExecutionRecord::from_legacy(
+ "click",
+ &serde_json::json!({ "delivery_mode": "background" }),
+ structured,
+ )
+ .expect("posted click should normalize into the public action contract");
+ assert_eq!(record.transport, ActionTransport::WindowsPostMessage);
+ assert_eq!(record.requested_delivery, RequestedDelivery::Background);
+ assert_eq!(record.actual_delivery, Some(ActualDelivery::Background));
+
+ let public = serde_json::to_value(record.public_result().expect("valid ActionResult"))
+ .expect("serialize ActionResult");
+ assert_eq!(public["route"], "synthetic_events");
+ assert_eq!(public["delivery"]["mode"], "background");
+ }
+}
+
// ── type_text ─────────────────────────────────────────────────────────────────
/// px-focus for the keyboard family (type_text / press_key / hotkey): pixel-click