Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
fix(cua-driver): report PostMessage pixel fallback truthfully
Francesco Bonacci4dc537ef34e7
diff --git a/.github/workflows/ci-rust-windows.yml b/.github/workflows/ci-rust-windows.ymlindex 376f881678..63117775fb 100644--- a/.github/workflows/ci-rust-windows.yml+++ b/.github/workflows/ci-rust-windows.yml@@ -71,6 +71,9 @@ jobs:if ($LASTEXITCODE -ne 0) { throw "PostMessage read-back tests failed with exit $LASTEXITCODE" }cargo test -p platform-windows value_write_readback_tests:: --lib --lockedif ($LASTEXITCODE -ne 0) { throw "ValuePattern read-back tests failed with exit $LASTEXITCODE" }+ - name: Run pixel click transport regression+ working-directory: libs/cua-driver/rust+ run: cargo test -p platform-windows post_message_pixel_click_reports_synthetic_transport --lib --locked- name: Run named-pipe readiness regressionworking-directory: libs/cua-driver/rustrun: cargo test -p cua-driver-testkit stalled_pipe_attempt_is_bounded_and_does_not_block_the_next_probe --lib --lockeddiff --git a/libs/cua-driver/rust/crates/cua-driver/tests/harness_web_test.rs b/libs/cua-driver/rust/crates/cua-driver/tests/harness_web_test.rsindex 1be4e7870e..de06bf2537 100644--- a/libs/cua-driver/rust/crates/cua-driver/tests/harness_web_test.rs+++ b/libs/cua-driver/rust/crates/cua-driver/tests/harness_web_test.rs@@ -305,7 +305,7 @@ fn harness_webview_left_click_px_background() {"webview2","left_click",Targeting::Px,- DriverRoute::UiaInvoke,+ DriverRoute::Composite,);let point = Cell::new(None);run_web_case_with_preparation(@@ -376,13 +376,13 @@ fn harness_webview_left_click_px_background() {"WebView2 PX background click failed: {}",click.text());- assert_eq!(- click.action_route(),- Some("accessibility"),- "WebView2 PX background click used an unexpected driver route: {}",- click.text()- );wait_for_journal_text(journal, "lbl-counter", "counter=3");+ let route = click.action_route();+ assert!(+ matches!(route, Some("accessibility" | "synthetic_events")),+ "WebView2 PX background click used an unsupported driver route {route:?}: {}",+ click.text(),+ );},);}diff --git a/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs b/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rsindex 3d24348ed3..9e0f7effd2 100644--- a/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs+++ b/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs@@ -2809,6 +2809,14 @@ impl Tool for LaunchAppTool {// ── click ─────────────────────────────────────────────────────────────────────+fn posted_pixel_click_result(pid: u32, click_word: &str) -> ToolResult {+ ToolResult::text(format!("✅ Posted {click_word} to pid {pid}.")).with_structured(json!({+ "path": "post_message",+ "verified": false,+ "effect": "unverifiable"+ }))+}+pub struct ClickTool {state: Arc<ToolState>,}@@ -3783,15 +3791,12 @@ impl Tool for ClickTool {};}- // delivery_mode:"background", non-dropped + non-UIA click — e.g. a- // right-click or multi-click on a plain Win32 window. Chromium is- // never reached here: its mouse events are flagged silently-dropped- // above and handled by the injection path, so the only targets that- // fall through are ones PostMessage delivers to without a foreground- // swap. (Foreground was handled at the top; the legacy `auto`- // Chromium-SendInput heuristic was removed in the macOS-alignment- // pass — escalating to delivery_mode:"foreground" is now the explicit- // path for any target PostMessage can't reach.)+ // delivery_mode:"background", non-dropped + non-UIA click. This+ // includes plain Win32 targets and embedded Chromium when its UIA+ // provider is temporarily unavailable. Attempt PostMessage without a+ // foreground swap; the caller must verify the resulting fixture state.+ // Foreground was handled at the top, and known dropped surfaces use+ // the targeted-injection path above.// bitmap pixels -> screen (DWM-frame origin + inset). Use// post_click_screen so we don't double-ClientToScreen.@@ -3807,10 +3812,7 @@ impl Tool for ClickTool {3 => "triple-click",_ => "click",};- ToolResult::text(format!("✅ Posted {click_word} to pid {pid}."))- .with_structured(- json!({ "path": "pixel", "verified": false, "effect": "unverifiable" }),- )+ posted_pixel_click_result(pid, click_word)}Ok(Err(e)) => ToolResult::error(e.to_string()),Err(e) => ToolResult::error(format!("Task error: {e}")),@@ -3822,6 +3824,39 @@ impl Tool for ClickTool {}}+#[cfg(test)]+mod pixel_click_transport_tests {+ use super::posted_pixel_click_result;+ use cua_driver_core::action_record::{+ ActionExecutionRecord, ActionTransport, ActualDelivery, RequestedDelivery,+ };++ #[test]+ fn post_message_pixel_click_reports_synthetic_transport() {+ let result = posted_pixel_click_result(42, "click");+ let structured = result+ .structured_content+ .as_ref()+ .expect("posted click should expose legacy transport metadata");+ assert_eq!(structured["path"], "post_message");++ let record = ActionExecutionRecord::from_legacy(+ "click",+ &serde_json::json!({ "delivery_mode": "background" }),+ structured,+ )+ .expect("posted click should normalize into the public action contract");+ assert_eq!(record.transport, ActionTransport::WindowsPostMessage);+ assert_eq!(record.requested_delivery, RequestedDelivery::Background);+ assert_eq!(record.actual_delivery, Some(ActualDelivery::Background));++ let public = serde_json::to_value(record.public_result().expect("valid ActionResult"))+ .expect("serialize ActionResult");+ assert_eq!(public["route"], "synthetic_events");+ assert_eq!(public["delivery"]["mode"], "background");+ }+}+// ── type_text ─────────────────────────────────────────────────────────────────/// px-focus for the keyboard family (type_text / press_key / hotkey): pixel-click