Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
fix(cua-driver-rs)(platform-windows): close 4 functional gaps from PR #1699 harness (#1705)
Francesco Bonacci49c882f7abb2
diff --git a/libs/cua-driver/rust/crates/cua-driver/tests/harness_web_test.rs b/libs/cua-driver/rust/crates/cua-driver/tests/harness_web_test.rsindex e37f8d4848..eecd7609de 100644--- a/libs/cua-driver/rust/crates/cua-driver/tests/harness_web_test.rs+++ b/libs/cua-driver/rust/crates/cua-driver/tests/harness_web_test.rs@@ -154,20 +154,65 @@ where F: FnOnce(u32, u64, &mut ChildStdin, &mut BufReader<&mut ChildStdout>) {drop(session);}-// ── WebView2 structural ──────────────────────────────────────────────────────+// ── WebView2 structural + page tool ─────────────────────────────────────────#[test]#[ignore]fn harness_webview_window_discoverable() {- // Smoke test: WebView2 harness launches, window appears via list_windows.- // Behavioural page-tool tests are deferred until WebView2 actually opens- // its CDP listener — see the module docstring TODO.run_with_session("webview", webview_exe(), "CuaTestHarness WebView", 9222,|pid, wid, _stdin, _stdout| {println!("✅ harness_webview_window_discoverable: pid={pid} wid={wid}");});}+#[test]+#[ignore]+fn harness_webview_page_tool() {+ // Regression guard for WebView2 CDP exposure via+ // CoreWebView2EnvironmentOptions.AdditionalBrowserArguments.+ // Combined with the `/json` Content-Length fix in mcp-server/src/cdp.rs,+ // the page tool now reaches WebView2's DOM via CDP just like Electron.+ run_with_session("webview", webview_exe(), "CuaTestHarness WebView", 9222,+ |pid, wid, stdin, stdout| {++ let marker_resp = tools_call(stdin, stdout, 30, "page", serde_json::json!({+ "pid": pid as i64, "window_id": wid, "action": "execute_javascript",+ "javascript": "document.querySelector('[data-cua-id=\"page-marker\"]').textContent"+ }));+ let marker = marker_resp.get("result")+ .and_then(|r| r.get("content"))+ .and_then(|c| c.as_array())+ .and_then(|arr| arr.get(0))+ .and_then(|item| item.get("text"))+ .and_then(|t| t.as_str())+ .unwrap_or("");+ assert!(marker.contains("WEB_HARNESS_MARKER_v1"),+ "WebView2 CDP execute_javascript marker fetch: {marker:?}");++ // click_element via DOM selector + counter readback.+ let _ = tools_call(stdin, stdout, 31, "page", serde_json::json!({+ "pid": pid as i64, "window_id": wid, "action": "click_element",+ "selector": "#btn-increment"+ }));+ std::thread::sleep(Duration::from_millis(500));++ let counter_resp = tools_call(stdin, stdout, 32, "page", serde_json::json!({+ "pid": pid as i64, "window_id": wid, "action": "execute_javascript",+ "javascript": "document.getElementById('lbl-counter').textContent"+ }));+ let post = counter_resp.get("result")+ .and_then(|r| r.get("content"))+ .and_then(|c| c.as_array())+ .and_then(|arr| arr.get(0))+ .and_then(|item| item.get("text"))+ .and_then(|t| t.as_str())+ .unwrap_or("");+ assert!(post.contains("counter=1"),+ "WebView2 counter didn't advance via page.click_element: {post:?}");+ println!("✅ harness_webview_page_tool: CDP+execute_javascript+click_element green");+ });+}+// ── Electron structural + page tool ──────────────────────────────────────────#[test]@@ -217,25 +262,24 @@ fn harness_electron_page_tool() {});}-/// Documents a separate gap in page.click_element: its CDP probe runs JS-/// that returns a JSON object (vx/vy/sx/sy/dpr) but the result is wrapped-/// as a CDP `runtime.evaluate.user_gesture` string by the time the page-/// tool's JSON parser sees it — causing "probe JSON missing required-/// field 'vx'". Fix: unwrap the CDP result's `.value` before parsing,-/// or send the probe with `returnByValue:true` in the Runtime.evaluate-/// params. Manually unwrapping the harness's verified JSON string works.+/// Regression guard for the page.click_element double-encode fix.+///+/// Originally the CDP runtime.evaluate response for the probe JS came+/// back as a JSON-encoded string containing the actual `{vx,vy,...}`+/// object. The page tool's `serde_json::from_str(&probe_json).or_else(...)`+/// only fell into the inner-decode branch on a hard parse error, but+/// `from_str` happily parses a JSON-string into a `Value::String`, so the+/// inner-decode branch never ran and `parsed.get("vx")` returned None.+/// Fix: match on Value::String and re-decode explicitly.#[test]#[ignore]-fn harness_electron_click_element_DOCUMENTED_wrapper_bug() {+fn harness_electron_click_element() {run_with_session("electron", electron_exe(), "CuaTestHarness Electron", 9223,|pid, wid, stdin, stdout| {let resp = tools_call(stdin, stdout, 30, "page", serde_json::json!({"pid": pid as i64, "window_id": wid, "action": "click_element","selector": "#btn-increment"}));- // Safe traversal — error responses from `page` can vary in shape- // (sometimes `result.isError: true` + `content[0].text`, sometimes- // `error.message`). Don't blow up on a missing array entry.let text = resp.get("result").and_then(|r| r.get("content")).and_then(|c| c.as_array())@@ -244,12 +288,24 @@ fn harness_electron_click_element_DOCUMENTED_wrapper_bug() {.and_then(|t| t.as_str()).or_else(|| resp.get("error").and_then(|e| e.get("message")).and_then(|m| m.as_str())).unwrap_or("");- // Expect the gap: parser fails on the CDP-wrapped probe response.- let expected_gap = text.contains("probe JSON missing") || text.contains("required field");- assert!(expected_gap,- "Expected click_element probe-wrapper gap. Got: {text:?}. \- If this asserts on a success message, the wrapper-unwrap fix is in — \- flip this assertion to assert success.");- println!("⚠️ harness_electron_click_element_DOCUMENTED_wrapper_bug: probe JSON parse gap present");+ assert!(!text.contains("probe JSON missing") && !text.contains("required field"),+ "click_element probe parse regressed: {text:?}");+ std::thread::sleep(Duration::from_millis(400));++ // Verify the click actually fired in the DOM.+ let counter_resp = tools_call(stdin, stdout, 31, "page", serde_json::json!({+ "pid": pid as i64, "window_id": wid, "action": "execute_javascript",+ "javascript": "document.getElementById('lbl-counter').textContent"+ }));+ let post = counter_resp.get("result")+ .and_then(|r| r.get("content"))+ .and_then(|c| c.as_array())+ .and_then(|arr| arr.get(0))+ .and_then(|item| item.get("text"))+ .and_then(|t| t.as_str())+ .unwrap_or("");+ assert!(post.contains("counter=1"),+ "Counter didn't advance after page.click_element: {post:?}");+ println!("✅ harness_electron_click_element: probe parsed, click fired, counter=1");});}diff --git a/libs/cua-driver/rust/crates/cua-driver/tests/harness_winui3_test.rs b/libs/cua-driver/rust/crates/cua-driver/tests/harness_winui3_test.rsindex 6f3feb1e5a..4f26d1273e 100644--- a/libs/cua-driver/rust/crates/cua-driver/tests/harness_winui3_test.rs+++ b/libs/cua-driver/rust/crates/cua-driver/tests/harness_winui3_test.rs@@ -326,22 +326,36 @@ fn harness_winui3_radio_select() {/// fails on RangeValuePattern-only elements. Real fix: cua-driver should/// try RangeValuePattern.SetValue (coercing the string to a double) when/// ValuePattern isn't found.-/// WinUI3 Slider's parent AutomationId doesn't surface in the flat UIA-/// element list (SliderAutomationPeer's children are the actionable-/// elements, not the slider itself). Still document the gap — fixing-/// the UIA enumeration to expose the slider as an indexed element is-/// orthogonal to this branch's set_value fix.+/// Regression guard for Slider element enumeration + RangeValuePattern+/// set_value. cua-driver's UIA cache now pre-fetches RangeValuePattern,+/// and `detect_cached_actions` reports `set_value` when present — so+/// Slider parents get an `[N]` flat-tree index. The `set_value` tool+/// already falls back to RangeValuePattern, so writing the value works+/// end-to-end against a WinUI3 Slider.#[test]#[ignore]-fn harness_winui3_slider_DOCUMENTED_unreachable() {+fn harness_winui3_slider_set_value() {winui3_with_session(|pid, wid, stdin, stdout| {let snap = tools_call(stdin, stdout, 20, "get_window_state",serde_json::json!({"pid": pid as i64, "window_id": wid, "capture_mode": "ax"}));- let idx_opt = find_idx(snapshot_text(&snap), "sld-value");- assert!(idx_opt.is_none(),- "WinUI3 Slider 'sld-value' now appears in the UIA flat tree — \- cua-driver fixed the slider-element enumeration gap. Update this test.");- println!("⚠️ harness_winui3_slider_DOCUMENTED_unreachable: sld-value not in UIA flat tree");+ let idx = find_idx(snapshot_text(&snap), "sld-value")+ .expect("sld-value should now be in the UIA flat tree after RangeValuePattern detection fix");+ let resp = tools_call(stdin, stdout, 30, "set_value", serde_json::json!({+ "pid": pid as i64, "window_id": wid, "element_index": idx,+ "value": "42"+ }));+ println!("set_value sld-value=42: {}", resp["result"]["content"][0]["text"]);+ std::thread::sleep(Duration::from_millis(400));+ let post = tools_call(stdin, stdout, 31, "get_window_state",+ serde_json::json!({"pid": pid as i64, "window_id": wid, "capture_mode": "ax"}));+ let text = snapshot_text(&post);+ let advanced = text.lines().any(|l|+ l.contains("slider_value=") && !l.contains("slider_value=0\""));+ assert!(advanced,+ "WinUI3 Slider didn't move via RangeValuePattern.SetValue. Lines: {}",+ text.lines().filter(|l| l.contains("slider_value"))+ .collect::<Vec<_>>().join(" / "));+ println!("✅ harness_winui3_slider_set_value: value moved via UIA RangeValuePattern.SetValue");});}diff --git a/libs/cua-driver/rust/crates/cua-driver/tests/harness_wpf_test.rs b/libs/cua-driver/rust/crates/cua-driver/tests/harness_wpf_test.rsindex 7bb1e40e92..3f22376e03 100644--- a/libs/cua-driver/rust/crates/cua-driver/tests/harness_wpf_test.rs+++ b/libs/cua-driver/rust/crates/cua-driver/tests/harness_wpf_test.rs@@ -676,37 +676,55 @@ fn harness_wpf_layered_popup_capture() {#[test]#[ignore]-fn harness_wpf_slider_drag_tool_returns() {- // Coverage note for the drag tool against a WPF Slider thumb.+fn harness_wpf_slider_drag() {+ // Regression guard for the SendInput drag path. PostMessage drag+ // doesn't update GetKeyState, so WPF's Thumb-drag handler (which+ // polls Mouse.LeftButton via GetKeyState) never sees the button+ // held — the thumb stays put. dispatch:"foreground" routes through+ // send_drag_synthesized which goes via the system input queue and+ // DOES update GetKeyState, so the thumb actually tracks.//- // PostMessage WM_LBUTTONDOWN / WM_MOUSEMOVE / WM_LBUTTONUP doesn't- // update the OS keyboard/mouse state visible to GetKeyState. WPF's- // Slider Thumb relies on Mouse.LeftButton (which polls GetKeyState)- // to recognise an in-progress drag — so a PostMessage drag never- // moves a WPF thumb, even when from/to are correctly on the thumb- // in client coords. The companion `harness_wpf_slider_increase_large`- // test covers the slider via UIA Invoke on its internal IncreaseLarge- // sub-button, which is what an agent SHOULD use for slider- // manipulation on a backgrounded window.- //- // We still exercise the drag tool against the slider so its codepath- // (coord translation, dispatch policy, message synthesis) is on the- // critical-path test list — we just don't assert on the value moving.- // TODO: add a SendInput-based drag path so dispatch:"foreground" can- // drive the thumb, then enable a behavioral assertion here.+ // Foreground-lock caveat: SetForegroundWindow can be rejected from+ // non-UIAccess processes during the daemon-process foreground swap.+ // bring_to_front first to make the harness foreground (via+ // AttachThreadInput), then SendInput's own SetForegroundWindow is a+ // no-op success.with_session(|pid, wid, stdin, stdout| {focus_harness(stdin, stdout, pid, wid);+ let pre = snapshot_elements(stdin, stdout, pid, wid);+ assert!(snapshot_text(&pre).contains("slider_value=0"),+ "initial slider_value=0 missing");+let resp = tools_call(stdin, stdout, 30, "drag", serde_json::json!({"pid": pid as i64, "window_id": wid,+ // drag screen-coords path: send_drag_synthesized takes screen+ // coords directly. The harness window is centered at+ // (517, 66) with the slider track at client (50-330, 275);+ // convert to screen via ClientToScreen approximation by+ // offsetting by window position + non-client chrome+ // (title bar + border ~30,8). screen coords here are+ // re-derived in window-local form by the tool's existing+ // ClientToScreen step."from_x": 50.0, "from_y": 275.0,"to_x": 330.0, "to_y": 275.0,- "duration_ms": 600, "steps": 30+ "duration_ms": 700, "steps": 40,+ "dispatch": "foreground"}));let msg = resp["result"]["content"][0]["text"].as_str().unwrap_or("");- println!("drag slider: {msg}");+ println!("drag slider (foreground): {msg}");assert!(msg.starts_with("✅"),"drag tool returned non-success: {msg}");- println!("✅ harness_wpf_slider_drag_tool_returns: PostMessage drag emitted (known no-op vs WPF Thumb)");+ std::thread::sleep(Duration::from_millis(500));++ let post = snapshot_elements(stdin, stdout, pid, wid);+ let text = snapshot_text(&post);+ let advanced = text.lines().any(|l|+ l.contains("slider_value=") && !l.contains("slider_value=0\""));+ assert!(advanced,+ "Slider value did not advance via SendInput drag. Lines: {}",+ text.lines().filter(|l| l.contains("slider_value"))+ .collect::<Vec<_>>().join(" / "));+ println!("✅ harness_wpf_slider_drag: thumb tracked via SendInput drag");});}diff --git a/libs/cua-driver/rust/crates/platform-windows/src/input/mouse.rs b/libs/cua-driver/rust/crates/platform-windows/src/input/mouse.rsindex d42f29430c..8a7cbc2f47 100644--- a/libs/cua-driver/rust/crates/platform-windows/src/input/mouse.rs+++ b/libs/cua-driver/rust/crates/platform-windows/src/input/mouse.rs@@ -377,3 +377,133 @@ pub fn send_click_synthesized(Ok(())}++/// Press-hold-move-release drag via `SendInput`. Companion to+/// [`send_click_synthesized`] for the `drag` tool's `dispatch:"foreground"`+/// path.+///+/// Why a SendInput drag is needed at all: the PostMessage drag path posts+/// `WM_LBUTTONDOWN` + `WM_MOUSEMOVE`s + `WM_LBUTTONUP` to the target HWND.+/// PostMessage does NOT update the per-thread keyboard state that+/// `GetKeyState(VK_LBUTTON)` reads, so frameworks that poll the button-held+/// state during their drag handler (WPF's Thumb.IsDragging logic does this+/// via Mouse.LeftButton, which polls GetKeyState) never observe the button+/// as down and the drag is a no-op. SendInput goes through the system+/// input queue and DOES update GetKeyState, so a WPF Slider thumb actually+/// tracks the drag.+///+/// Same UIAccess constraints as [`send_click_synthesized`] — the+/// `SetForegroundWindow` swap is rejected from non-UIAccess processes+/// when foreground-lock is active; route through `cua-driver-uia.exe`+/// for reliable operation.+pub fn send_drag_synthesized(+ target: u64,+ sx_from: i32, sy_from: i32,+ sx_to: i32, sy_to: i32,+ duration_ms: u64,+ steps: usize,+ button: &str,+) -> Result<()> {+ let target = HWND(target as *mut _);+ if target.0.is_null() {+ bail!("invalid target hwnd");+ }+ if let Some(msg) = crate::input::post_message_blocked_by_uipi(target.0 as u64) {+ bail!(msg);+ }++ let (down_flag, up_flag) = match button {+ "right" => (MOUSEEVENTF_RIGHTDOWN, MOUSEEVENTF_RIGHTUP),+ "middle" => (MOUSEEVENTF_MIDDLEDOWN, MOUSEEVENTF_MIDDLEUP),+ _ => (MOUSEEVENTF_LEFTDOWN, MOUSEEVENTF_LEFTUP),+ };++ let (vd_x, vd_y, vd_w, vd_h) = unsafe {+ (+ GetSystemMetrics(SM_XVIRTUALSCREEN),+ GetSystemMetrics(SM_YVIRTUALSCREEN),+ GetSystemMetrics(SM_CXVIRTUALSCREEN).max(1),+ GetSystemMetrics(SM_CYVIRTUALSCREEN).max(1),+ )+ };+ let norm = |sx: i32, sy: i32| -> (i32, i32) {+ let nx = ((sx - vd_x) as i64 * 65535 / vd_w as i64).clamp(0, 65535) as i32;+ let ny = ((sy - vd_y) as i64 * 65535 / vd_h as i64).clamp(0, 65535) as i32;+ (nx, ny)+ };+ let make_input = |dx: i32, dy: i32, flags| INPUT {+ r#type: INPUT_MOUSE,+ Anonymous: INPUT_0 {+ mi: MOUSEINPUT {+ dx, dy, mouseData: 0,+ dwFlags: flags | MOUSEEVENTF_ABSOLUTE | MOUSEEVENTF_VIRTUALDESK,+ time: 0, dwExtraInfo: 0,+ },+ },+ };++ let steps = steps.max(1);+ let step_delay_ms = if steps > 1 { duration_ms / steps as u64 } else { 0 };++ unsafe {+ let prev_fg = GetForegroundWindow();+ let mut prev_cursor = POINT::default();+ let _ = GetCursorPos(&mut prev_cursor);++ let _ = SetForegroundWindow(target);+ sleep(Duration::from_millis(8));+ let actual_fg = GetForegroundWindow();+ if actual_fg != target {+ bail!(+ "Foreground swap to target HWND {:?} was rejected by Windows \+ (actual foreground is HWND {:?}). Non-UIAccess processes can't \+ reliably change foreground under the foreground-lock. Route the \+ drag through cua-driver-uia.exe.",+ target.0, actual_fg.0+ );+ }++ // 1. Move + press at the start of the drag.+ let (nfx, nfy) = norm(sx_from, sy_from);+ let _ = SetCursorPos(sx_from, sy_from);+ let prelude = [+ make_input(nfx, nfy, MOUSEEVENTF_MOVE),+ make_input(nfx, nfy, down_flag),+ ];+ let sent = SendInput(&prelude, std::mem::size_of::<INPUT>() as i32);+ if sent as usize != prelude.len() {+ let _ = SetCursorPos(prev_cursor.x, prev_cursor.y);+ let _ = SetForegroundWindow(prev_fg);+ bail!("SendInput drag-prelude inserted {sent}/{} events", prelude.len());+ }++ // 2. Interpolate the path. SetCursorPos + MOUSEEVENTF_MOVE in lockstep+ // so both the visible cursor and the system input queue track the+ // same path — WPF's drag-handler watches GetKeyState during each+ // move event.+ for i in 1..=steps {+ let t = i as f64 / steps as f64;+ let x = sx_from + ((sx_to - sx_from) as f64 * t).round() as i32;+ let y = sy_from + ((sy_to - sy_from) as f64 * t).round() as i32;+ let (nx, ny) = norm(x, y);+ let _ = SetCursorPos(x, y);+ let mv = [make_input(nx, ny, MOUSEEVENTF_MOVE)];+ let _ = SendInput(&mv, std::mem::size_of::<INPUT>() as i32);+ if step_delay_ms > 0 {+ sleep(Duration::from_millis(step_delay_ms));+ }+ }++ // 3. Release at the end.+ let (ntx, nty) = norm(sx_to, sy_to);+ let release = [make_input(ntx, nty, up_flag)];+ let _ = SendInput(&release, std::mem::size_of::<INPUT>() as i32);++ // Brief settle, then restore previous state.+ sleep(Duration::from_millis(40));+ let _ = SetCursorPos(prev_cursor.x, prev_cursor.y);+ let _ = SetForegroundWindow(prev_fg);+ }++ Ok(())+}diff --git a/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs b/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rsindex dd26e7caba..7c9d6094b7 100644--- a/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs+++ b/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs@@ -3414,19 +3414,36 @@ impl Tool for DragTool {{return background_unavailable_error(hwnd, EventKind::MouseClick);}- // dispatch:"foreground" — no SendInput-based drag helper yet. The- // PostMessage drag works for the GTK-canvas + plain-Win32 cases we- // care about today; Chromium DOM dragstart will need a real- // SendInput-MOUSE-MOVE sequence, tracked as TODO.+ // dispatch:"foreground" — SendInput-based drag. Required for WPF+ // Slider thumbs (and any framework that polls GetKeyState during+ // its drag handler — PostMessage doesn't update per-thread input+ // state, so those targets see a button-up world during the drag+ // and never start tracking). Subject to the same UIAccess+ // foreground-lock constraint as send_click_synthesized.if dispatch == DispatchMode::Foreground {- return ToolResult::error(- "dispatch:\"foreground\" is not yet implemented for the drag tool. \- Use bring_to_front to activate the target first, then retry with \- dispatch:\"auto\" (PostMessage WM_MOUSEMOVE works against an \- already-foreground window in most cases). TODO: add a \- send_drag_synthesized helper analogous to send_click_synthesized."- .to_string(),- );+ let btn_fg = button.clone();+ let prev_fg_addr = unsafe {+ windows::Win32::UI::WindowsAndMessaging::GetForegroundWindow().0 as usize+ };+ let send_result = tokio::task::spawn_blocking(move || {+ crate::input::mouse::send_drag_synthesized(+ hwnd,+ sx_from, sy_from,+ sx_to, sy_to,+ duration_ms, steps, &btn_fg,+ )+ }).await;+ tokio::spawn(restore_foreground_polling_best_effort(prev_fg_addr, pid));+ let button_suffix = if button == "left" { String::new() } else { format!(" ({} button)", button) };+ return match send_result {+ Ok(Ok(())) => ToolResult::text(format!(+ "✅ Sent drag{button_suffix} via SendInput on pid {raw_pid} \+ from screen ({sx_from},{sy_from}) → ({sx_to},{sy_to}) \+ in {duration_ms}ms / {steps} steps (dispatch:foreground)."+ )),+ Ok(Err(e)) => ToolResult::error(e.to_string()),+ Err(e) => ToolResult::error(format!("Task error: {e}")),+ };}// Pin the agent-cursor overlay above the drag target so the syntheticdiff --git a/libs/cua-driver/rust/crates/platform-windows/src/tools/page.rs b/libs/cua-driver/rust/crates/platform-windows/src/tools/page.rsindex 120d6999d3..0481dd8b44 100644--- a/libs/cua-driver/rust/crates/platform-windows/src/tools/page.rs+++ b/libs/cua-driver/rust/crates/platform-windows/src/tools/page.rs@@ -166,18 +166,30 @@ impl PageBackend for WindowsPageBackend {// The wrapper paths return the result JSON-stringified. The bookmark// wrapper wraps the user expression's return in JSON.stringify(),- // which means the inner string itself is JSON-encoded — we need to- // parse twice to get the actual coord object.- let parsed: serde_json::Value = serde_json::from_str(&probe_json)- .or_else(|_| {- // Bookmark wrapper double-encoded the string — decode the- // outer JSON-string, then parse the inner JSON.- let inner: String = serde_json::from_str(&probe_json)?;- serde_json::from_str::<serde_json::Value>(&inner)- })- .map_err(|e| anyhow::anyhow!(- "click_element: could not parse coord JSON from probe (raw: {probe_raw:?}): {e}"- ))?;+ // and CDP's runtime.evaluate also serialises the JS return as a+ // JSON-string when it's not a primitive — either way the inner+ // string itself is JSON-encoded. We need to parse twice to get+ // the actual coord object.+ //+ // Crucially, the first parse can SUCCEED as a Value::String (when+ // the outer is a quoted JSON-string), so we can't rely on+ // `.or_else(|_| ...)` to trigger the second decode — that branch+ // only fires on a hard parse error. Inspect the parsed value and+ // re-decode when it's a String.+ let parsed: serde_json::Value = {+ let first = serde_json::from_str::<serde_json::Value>(&probe_json)+ .map_err(|e| anyhow::anyhow!(+ "click_element: could not parse coord JSON from probe (raw: {probe_raw:?}): {e}"+ ))?;+ match first {+ serde_json::Value::String(inner) => serde_json::from_str(&inner)+ .map_err(|e| anyhow::anyhow!(+ "click_element: inner JSON parse failed for double-encoded probe \+ response (raw: {probe_raw:?}): {e}"+ ))?,+ other => other,+ }+ };// Required-field validation. The previous version defaulted any// missing field to 0.0, which silently animated the visible cursordiff --git a/libs/cua-driver/rust/crates/platform-windows/src/uia/mod.rs b/libs/cua-driver/rust/crates/platform-windows/src/uia/mod.rsindex 39cb9fb9e9..76e9dacf21 100644--- a/libs/cua-driver/rust/crates/platform-windows/src/uia/mod.rs+++ b/libs/cua-driver/rust/crates/platform-windows/src/uia/mod.rs@@ -19,7 +19,7 @@ use windows::Win32::UI::Accessibility::{UIA_ProcessIdPropertyId, UIA_ValueValuePropertyId,UIA_InvokePatternId, UIA_SelectionItemPatternId,UIA_TogglePatternId, UIA_ExpandCollapsePatternId, UIA_TextPatternId,- UIA_ValuePatternId, UIA_ScrollPatternId,+ UIA_ValuePatternId, UIA_RangeValuePatternId, UIA_ScrollPatternId,TreeScope_Children, TreeScope_Subtree,};@@ -100,6 +100,7 @@ unsafe fn walk_tree_unsafe(hwnd: u64, query: Option<&str>) -> UiaTreeResult {UIA_SelectionItemPatternId,UIA_ExpandCollapsePatternId,UIA_ValuePatternId,+ UIA_RangeValuePatternId,UIA_TextPatternId,UIA_ScrollPatternId,] {@@ -522,6 +523,13 @@ fn detect_cached_actions(element: &IUIAutomationElement, is_enabled: bool) -> Veif element.GetCachedPattern(UIA_ValuePatternId).is_ok() {actions.push("set_value".into());}+ // RangeValuePattern is exposed by Sliders, ProgressBars, and other+ // numeric-range controls. Without this entry the slider parent+ // gets actions=[] → marked non-actionable → no `[N]` index in the+ // flat tree, making the slider unaddressable by AutomationId.+ if element.GetCachedPattern(UIA_RangeValuePatternId).is_ok() {+ actions.push("set_value".into());+ }if element.GetCachedPattern(UIA_TextPatternId).is_ok() {actions.push("text".into());}