Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

fix(cua-driver-rs)(platform-windows): close 4 functional gaps from PR #1699 harness (#1705)

Francesco Bonacci49c882f7abb2
diff --git a/libs/cua-driver/rust/crates/cua-driver/tests/harness_web_test.rs b/libs/cua-driver/rust/crates/cua-driver/tests/harness_web_test.rs
index e37f8d4848..eecd7609de 100644
--- a/libs/cua-driver/rust/crates/cua-driver/tests/harness_web_test.rs
+++ b/libs/cua-driver/rust/crates/cua-driver/tests/harness_web_test.rs
@@ -154,20 +154,65 @@ where F: FnOnce(u32, u64, &mut ChildStdin, &mut BufReader<&mut ChildStdout>) {
drop(session);
}
-// ── WebView2 structural ──────────────────────────────────────────────────────
+// ── WebView2 structural + page tool ─────────────────────────────────────────
#[test]
#[ignore]
fn harness_webview_window_discoverable() {
- // Smoke test: WebView2 harness launches, window appears via list_windows.
- // Behavioural page-tool tests are deferred until WebView2 actually opens
- // its CDP listener — see the module docstring TODO.
run_with_session("webview", webview_exe(), "CuaTestHarness WebView", 9222,
|pid, wid, _stdin, _stdout| {
println!("✅ harness_webview_window_discoverable: pid={pid} wid={wid}");
});
}
+#[test]
+#[ignore]
+fn harness_webview_page_tool() {
+ // Regression guard for WebView2 CDP exposure via
+ // CoreWebView2EnvironmentOptions.AdditionalBrowserArguments.
+ // Combined with the `/json` Content-Length fix in mcp-server/src/cdp.rs,
+ // the page tool now reaches WebView2's DOM via CDP just like Electron.
+ run_with_session("webview", webview_exe(), "CuaTestHarness WebView", 9222,
+ |pid, wid, stdin, stdout| {
+
+ let marker_resp = tools_call(stdin, stdout, 30, "page", serde_json::json!({
+ "pid": pid as i64, "window_id": wid, "action": "execute_javascript",
+ "javascript": "document.querySelector('[data-cua-id=\"page-marker\"]').textContent"
+ }));
+ let marker = marker_resp.get("result")
+ .and_then(|r| r.get("content"))
+ .and_then(|c| c.as_array())
+ .and_then(|arr| arr.get(0))
+ .and_then(|item| item.get("text"))
+ .and_then(|t| t.as_str())
+ .unwrap_or("");
+ assert!(marker.contains("WEB_HARNESS_MARKER_v1"),
+ "WebView2 CDP execute_javascript marker fetch: {marker:?}");
+
+ // click_element via DOM selector + counter readback.
+ let _ = tools_call(stdin, stdout, 31, "page", serde_json::json!({
+ "pid": pid as i64, "window_id": wid, "action": "click_element",
+ "selector": "#btn-increment"
+ }));
+ std::thread::sleep(Duration::from_millis(500));
+
+ let counter_resp = tools_call(stdin, stdout, 32, "page", serde_json::json!({
+ "pid": pid as i64, "window_id": wid, "action": "execute_javascript",
+ "javascript": "document.getElementById('lbl-counter').textContent"
+ }));
+ let post = counter_resp.get("result")
+ .and_then(|r| r.get("content"))
+ .and_then(|c| c.as_array())
+ .and_then(|arr| arr.get(0))
+ .and_then(|item| item.get("text"))
+ .and_then(|t| t.as_str())
+ .unwrap_or("");
+ assert!(post.contains("counter=1"),
+ "WebView2 counter didn't advance via page.click_element: {post:?}");
+ println!("✅ harness_webview_page_tool: CDP+execute_javascript+click_element green");
+ });
+}
+
// ── Electron structural + page tool ──────────────────────────────────────────
#[test]
@@ -217,25 +262,24 @@ fn harness_electron_page_tool() {
});
}
-/// Documents a separate gap in page.click_element: its CDP probe runs JS
-/// that returns a JSON object (vx/vy/sx/sy/dpr) but the result is wrapped
-/// as a CDP `runtime.evaluate.user_gesture` string by the time the page
-/// tool's JSON parser sees it — causing "probe JSON missing required
-/// field 'vx'". Fix: unwrap the CDP result's `.value` before parsing,
-/// or send the probe with `returnByValue:true` in the Runtime.evaluate
-/// params. Manually unwrapping the harness's verified JSON string works.
+/// Regression guard for the page.click_element double-encode fix.
+///
+/// Originally the CDP runtime.evaluate response for the probe JS came
+/// back as a JSON-encoded string containing the actual `{vx,vy,...}`
+/// object. The page tool's `serde_json::from_str(&probe_json).or_else(...)`
+/// only fell into the inner-decode branch on a hard parse error, but
+/// `from_str` happily parses a JSON-string into a `Value::String`, so the
+/// inner-decode branch never ran and `parsed.get("vx")` returned None.
+/// Fix: match on Value::String and re-decode explicitly.
#[test]
#[ignore]
-fn harness_electron_click_element_DOCUMENTED_wrapper_bug() {
+fn harness_electron_click_element() {
run_with_session("electron", electron_exe(), "CuaTestHarness Electron", 9223,
|pid, wid, stdin, stdout| {
let resp = tools_call(stdin, stdout, 30, "page", serde_json::json!({
"pid": pid as i64, "window_id": wid, "action": "click_element",
"selector": "#btn-increment"
}));
- // Safe traversal — error responses from `page` can vary in shape
- // (sometimes `result.isError: true` + `content[0].text`, sometimes
- // `error.message`). Don't blow up on a missing array entry.
let text = resp.get("result")
.and_then(|r| r.get("content"))
.and_then(|c| c.as_array())
@@ -244,12 +288,24 @@ fn harness_electron_click_element_DOCUMENTED_wrapper_bug() {
.and_then(|t| t.as_str())
.or_else(|| resp.get("error").and_then(|e| e.get("message")).and_then(|m| m.as_str()))
.unwrap_or("");
- // Expect the gap: parser fails on the CDP-wrapped probe response.
- let expected_gap = text.contains("probe JSON missing") || text.contains("required field");
- assert!(expected_gap,
- "Expected click_element probe-wrapper gap. Got: {text:?}. \
- If this asserts on a success message, the wrapper-unwrap fix is in — \
- flip this assertion to assert success.");
- println!("⚠️ harness_electron_click_element_DOCUMENTED_wrapper_bug: probe JSON parse gap present");
+ assert!(!text.contains("probe JSON missing") && !text.contains("required field"),
+ "click_element probe parse regressed: {text:?}");
+ std::thread::sleep(Duration::from_millis(400));
+
+ // Verify the click actually fired in the DOM.
+ let counter_resp = tools_call(stdin, stdout, 31, "page", serde_json::json!({
+ "pid": pid as i64, "window_id": wid, "action": "execute_javascript",
+ "javascript": "document.getElementById('lbl-counter').textContent"
+ }));
+ let post = counter_resp.get("result")
+ .and_then(|r| r.get("content"))
+ .and_then(|c| c.as_array())
+ .and_then(|arr| arr.get(0))
+ .and_then(|item| item.get("text"))
+ .and_then(|t| t.as_str())
+ .unwrap_or("");
+ assert!(post.contains("counter=1"),
+ "Counter didn't advance after page.click_element: {post:?}");
+ println!("✅ harness_electron_click_element: probe parsed, click fired, counter=1");
});
}
diff --git a/libs/cua-driver/rust/crates/cua-driver/tests/harness_winui3_test.rs b/libs/cua-driver/rust/crates/cua-driver/tests/harness_winui3_test.rs
index 6f3feb1e5a..4f26d1273e 100644
--- a/libs/cua-driver/rust/crates/cua-driver/tests/harness_winui3_test.rs
+++ b/libs/cua-driver/rust/crates/cua-driver/tests/harness_winui3_test.rs
@@ -326,22 +326,36 @@ fn harness_winui3_radio_select() {
/// fails on RangeValuePattern-only elements. Real fix: cua-driver should
/// try RangeValuePattern.SetValue (coercing the string to a double) when
/// ValuePattern isn't found.
-/// WinUI3 Slider's parent AutomationId doesn't surface in the flat UIA
-/// element list (SliderAutomationPeer's children are the actionable
-/// elements, not the slider itself). Still document the gap — fixing
-/// the UIA enumeration to expose the slider as an indexed element is
-/// orthogonal to this branch's set_value fix.
+/// Regression guard for Slider element enumeration + RangeValuePattern
+/// set_value. cua-driver's UIA cache now pre-fetches RangeValuePattern,
+/// and `detect_cached_actions` reports `set_value` when present — so
+/// Slider parents get an `[N]` flat-tree index. The `set_value` tool
+/// already falls back to RangeValuePattern, so writing the value works
+/// end-to-end against a WinUI3 Slider.
#[test]
#[ignore]
-fn harness_winui3_slider_DOCUMENTED_unreachable() {
+fn harness_winui3_slider_set_value() {
winui3_with_session(|pid, wid, stdin, stdout| {
let snap = tools_call(stdin, stdout, 20, "get_window_state",
serde_json::json!({"pid": pid as i64, "window_id": wid, "capture_mode": "ax"}));
- let idx_opt = find_idx(snapshot_text(&snap), "sld-value");
- assert!(idx_opt.is_none(),
- "WinUI3 Slider 'sld-value' now appears in the UIA flat tree — \
- cua-driver fixed the slider-element enumeration gap. Update this test.");
- println!("⚠️ harness_winui3_slider_DOCUMENTED_unreachable: sld-value not in UIA flat tree");
+ let idx = find_idx(snapshot_text(&snap), "sld-value")
+ .expect("sld-value should now be in the UIA flat tree after RangeValuePattern detection fix");
+ let resp = tools_call(stdin, stdout, 30, "set_value", serde_json::json!({
+ "pid": pid as i64, "window_id": wid, "element_index": idx,
+ "value": "42"
+ }));
+ println!("set_value sld-value=42: {}", resp["result"]["content"][0]["text"]);
+ std::thread::sleep(Duration::from_millis(400));
+ let post = tools_call(stdin, stdout, 31, "get_window_state",
+ serde_json::json!({"pid": pid as i64, "window_id": wid, "capture_mode": "ax"}));
+ let text = snapshot_text(&post);
+ let advanced = text.lines().any(|l|
+ l.contains("slider_value=") && !l.contains("slider_value=0\""));
+ assert!(advanced,
+ "WinUI3 Slider didn't move via RangeValuePattern.SetValue. Lines: {}",
+ text.lines().filter(|l| l.contains("slider_value"))
+ .collect::<Vec<_>>().join(" / "));
+ println!("✅ harness_winui3_slider_set_value: value moved via UIA RangeValuePattern.SetValue");
});
}
diff --git a/libs/cua-driver/rust/crates/cua-driver/tests/harness_wpf_test.rs b/libs/cua-driver/rust/crates/cua-driver/tests/harness_wpf_test.rs
index 7bb1e40e92..3f22376e03 100644
--- a/libs/cua-driver/rust/crates/cua-driver/tests/harness_wpf_test.rs
+++ b/libs/cua-driver/rust/crates/cua-driver/tests/harness_wpf_test.rs
@@ -676,37 +676,55 @@ fn harness_wpf_layered_popup_capture() {
#[test]
#[ignore]
-fn harness_wpf_slider_drag_tool_returns() {
- // Coverage note for the drag tool against a WPF Slider thumb.
+fn harness_wpf_slider_drag() {
+ // Regression guard for the SendInput drag path. PostMessage drag
+ // doesn't update GetKeyState, so WPF's Thumb-drag handler (which
+ // polls Mouse.LeftButton via GetKeyState) never sees the button
+ // held — the thumb stays put. dispatch:"foreground" routes through
+ // send_drag_synthesized which goes via the system input queue and
+ // DOES update GetKeyState, so the thumb actually tracks.
//
- // PostMessage WM_LBUTTONDOWN / WM_MOUSEMOVE / WM_LBUTTONUP doesn't
- // update the OS keyboard/mouse state visible to GetKeyState. WPF's
- // Slider Thumb relies on Mouse.LeftButton (which polls GetKeyState)
- // to recognise an in-progress drag — so a PostMessage drag never
- // moves a WPF thumb, even when from/to are correctly on the thumb
- // in client coords. The companion `harness_wpf_slider_increase_large`
- // test covers the slider via UIA Invoke on its internal IncreaseLarge
- // sub-button, which is what an agent SHOULD use for slider
- // manipulation on a backgrounded window.
- //
- // We still exercise the drag tool against the slider so its codepath
- // (coord translation, dispatch policy, message synthesis) is on the
- // critical-path test list — we just don't assert on the value moving.
- // TODO: add a SendInput-based drag path so dispatch:"foreground" can
- // drive the thumb, then enable a behavioral assertion here.
+ // Foreground-lock caveat: SetForegroundWindow can be rejected from
+ // non-UIAccess processes during the daemon-process foreground swap.
+ // bring_to_front first to make the harness foreground (via
+ // AttachThreadInput), then SendInput's own SetForegroundWindow is a
+ // no-op success.
with_session(|pid, wid, stdin, stdout| {
focus_harness(stdin, stdout, pid, wid);
+ let pre = snapshot_elements(stdin, stdout, pid, wid);
+ assert!(snapshot_text(&pre).contains("slider_value=0"),
+ "initial slider_value=0 missing");
+
let resp = tools_call(stdin, stdout, 30, "drag", serde_json::json!({
"pid": pid as i64, "window_id": wid,
+ // drag screen-coords path: send_drag_synthesized takes screen
+ // coords directly. The harness window is centered at
+ // (517, 66) with the slider track at client (50-330, 275);
+ // convert to screen via ClientToScreen approximation by
+ // offsetting by window position + non-client chrome
+ // (title bar + border ~30,8). screen coords here are
+ // re-derived in window-local form by the tool's existing
+ // ClientToScreen step.
"from_x": 50.0, "from_y": 275.0,
"to_x": 330.0, "to_y": 275.0,
- "duration_ms": 600, "steps": 30
+ "duration_ms": 700, "steps": 40,
+ "dispatch": "foreground"
}));
let msg = resp["result"]["content"][0]["text"].as_str().unwrap_or("");
- println!("drag slider: {msg}");
+ println!("drag slider (foreground): {msg}");
assert!(msg.starts_with("✅"),
"drag tool returned non-success: {msg}");
- println!("✅ harness_wpf_slider_drag_tool_returns: PostMessage drag emitted (known no-op vs WPF Thumb)");
+ std::thread::sleep(Duration::from_millis(500));
+
+ let post = snapshot_elements(stdin, stdout, pid, wid);
+ let text = snapshot_text(&post);
+ let advanced = text.lines().any(|l|
+ l.contains("slider_value=") && !l.contains("slider_value=0\""));
+ assert!(advanced,
+ "Slider value did not advance via SendInput drag. Lines: {}",
+ text.lines().filter(|l| l.contains("slider_value"))
+ .collect::<Vec<_>>().join(" / "));
+ println!("✅ harness_wpf_slider_drag: thumb tracked via SendInput drag");
});
}
diff --git a/libs/cua-driver/rust/crates/platform-windows/src/input/mouse.rs b/libs/cua-driver/rust/crates/platform-windows/src/input/mouse.rs
index d42f29430c..8a7cbc2f47 100644
--- a/libs/cua-driver/rust/crates/platform-windows/src/input/mouse.rs
+++ b/libs/cua-driver/rust/crates/platform-windows/src/input/mouse.rs
@@ -377,3 +377,133 @@ pub fn send_click_synthesized(
Ok(())
}
+
+/// Press-hold-move-release drag via `SendInput`. Companion to
+/// [`send_click_synthesized`] for the `drag` tool's `dispatch:"foreground"`
+/// path.
+///
+/// Why a SendInput drag is needed at all: the PostMessage drag path posts
+/// `WM_LBUTTONDOWN` + `WM_MOUSEMOVE`s + `WM_LBUTTONUP` to the target HWND.
+/// PostMessage does NOT update the per-thread keyboard state that
+/// `GetKeyState(VK_LBUTTON)` reads, so frameworks that poll the button-held
+/// state during their drag handler (WPF's Thumb.IsDragging logic does this
+/// via Mouse.LeftButton, which polls GetKeyState) never observe the button
+/// as down and the drag is a no-op. SendInput goes through the system
+/// input queue and DOES update GetKeyState, so a WPF Slider thumb actually
+/// tracks the drag.
+///
+/// Same UIAccess constraints as [`send_click_synthesized`] — the
+/// `SetForegroundWindow` swap is rejected from non-UIAccess processes
+/// when foreground-lock is active; route through `cua-driver-uia.exe`
+/// for reliable operation.
+pub fn send_drag_synthesized(
+ target: u64,
+ sx_from: i32, sy_from: i32,
+ sx_to: i32, sy_to: i32,
+ duration_ms: u64,
+ steps: usize,
+ button: &str,
+) -> Result<()> {
+ let target = HWND(target as *mut _);
+ if target.0.is_null() {
+ bail!("invalid target hwnd");
+ }
+ if let Some(msg) = crate::input::post_message_blocked_by_uipi(target.0 as u64) {
+ bail!(msg);
+ }
+
+ let (down_flag, up_flag) = match button {
+ "right" => (MOUSEEVENTF_RIGHTDOWN, MOUSEEVENTF_RIGHTUP),
+ "middle" => (MOUSEEVENTF_MIDDLEDOWN, MOUSEEVENTF_MIDDLEUP),
+ _ => (MOUSEEVENTF_LEFTDOWN, MOUSEEVENTF_LEFTUP),
+ };
+
+ let (vd_x, vd_y, vd_w, vd_h) = unsafe {
+ (
+ GetSystemMetrics(SM_XVIRTUALSCREEN),
+ GetSystemMetrics(SM_YVIRTUALSCREEN),
+ GetSystemMetrics(SM_CXVIRTUALSCREEN).max(1),
+ GetSystemMetrics(SM_CYVIRTUALSCREEN).max(1),
+ )
+ };
+ let norm = |sx: i32, sy: i32| -> (i32, i32) {
+ let nx = ((sx - vd_x) as i64 * 65535 / vd_w as i64).clamp(0, 65535) as i32;
+ let ny = ((sy - vd_y) as i64 * 65535 / vd_h as i64).clamp(0, 65535) as i32;
+ (nx, ny)
+ };
+ let make_input = |dx: i32, dy: i32, flags| INPUT {
+ r#type: INPUT_MOUSE,
+ Anonymous: INPUT_0 {
+ mi: MOUSEINPUT {
+ dx, dy, mouseData: 0,
+ dwFlags: flags | MOUSEEVENTF_ABSOLUTE | MOUSEEVENTF_VIRTUALDESK,
+ time: 0, dwExtraInfo: 0,
+ },
+ },
+ };
+
+ let steps = steps.max(1);
+ let step_delay_ms = if steps > 1 { duration_ms / steps as u64 } else { 0 };
+
+ unsafe {
+ let prev_fg = GetForegroundWindow();
+ let mut prev_cursor = POINT::default();
+ let _ = GetCursorPos(&mut prev_cursor);
+
+ let _ = SetForegroundWindow(target);
+ sleep(Duration::from_millis(8));
+ let actual_fg = GetForegroundWindow();
+ if actual_fg != target {
+ bail!(
+ "Foreground swap to target HWND {:?} was rejected by Windows \
+ (actual foreground is HWND {:?}). Non-UIAccess processes can't \
+ reliably change foreground under the foreground-lock. Route the \
+ drag through cua-driver-uia.exe.",
+ target.0, actual_fg.0
+ );
+ }
+
+ // 1. Move + press at the start of the drag.
+ let (nfx, nfy) = norm(sx_from, sy_from);
+ let _ = SetCursorPos(sx_from, sy_from);
+ let prelude = [
+ make_input(nfx, nfy, MOUSEEVENTF_MOVE),
+ make_input(nfx, nfy, down_flag),
+ ];
+ let sent = SendInput(&prelude, std::mem::size_of::<INPUT>() as i32);
+ if sent as usize != prelude.len() {
+ let _ = SetCursorPos(prev_cursor.x, prev_cursor.y);
+ let _ = SetForegroundWindow(prev_fg);
+ bail!("SendInput drag-prelude inserted {sent}/{} events", prelude.len());
+ }
+
+ // 2. Interpolate the path. SetCursorPos + MOUSEEVENTF_MOVE in lockstep
+ // so both the visible cursor and the system input queue track the
+ // same path — WPF's drag-handler watches GetKeyState during each
+ // move event.
+ for i in 1..=steps {
+ let t = i as f64 / steps as f64;
+ let x = sx_from + ((sx_to - sx_from) as f64 * t).round() as i32;
+ let y = sy_from + ((sy_to - sy_from) as f64 * t).round() as i32;
+ let (nx, ny) = norm(x, y);
+ let _ = SetCursorPos(x, y);
+ let mv = [make_input(nx, ny, MOUSEEVENTF_MOVE)];
+ let _ = SendInput(&mv, std::mem::size_of::<INPUT>() as i32);
+ if step_delay_ms > 0 {
+ sleep(Duration::from_millis(step_delay_ms));
+ }
+ }
+
+ // 3. Release at the end.
+ let (ntx, nty) = norm(sx_to, sy_to);
+ let release = [make_input(ntx, nty, up_flag)];
+ let _ = SendInput(&release, std::mem::size_of::<INPUT>() as i32);
+
+ // Brief settle, then restore previous state.
+ sleep(Duration::from_millis(40));
+ let _ = SetCursorPos(prev_cursor.x, prev_cursor.y);
+ let _ = SetForegroundWindow(prev_fg);
+ }
+
+ Ok(())
+}
diff --git a/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs b/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs
index dd26e7caba..7c9d6094b7 100644
--- a/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs
+++ b/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs
@@ -3414,19 +3414,36 @@ impl Tool for DragTool {
{
return background_unavailable_error(hwnd, EventKind::MouseClick);
}
- // dispatch:"foreground" — no SendInput-based drag helper yet. The
- // PostMessage drag works for the GTK-canvas + plain-Win32 cases we
- // care about today; Chromium DOM dragstart will need a real
- // SendInput-MOUSE-MOVE sequence, tracked as TODO.
+ // dispatch:"foreground" — SendInput-based drag. Required for WPF
+ // Slider thumbs (and any framework that polls GetKeyState during
+ // its drag handler — PostMessage doesn't update per-thread input
+ // state, so those targets see a button-up world during the drag
+ // and never start tracking). Subject to the same UIAccess
+ // foreground-lock constraint as send_click_synthesized.
if dispatch == DispatchMode::Foreground {
- return ToolResult::error(
- "dispatch:\"foreground\" is not yet implemented for the drag tool. \
- Use bring_to_front to activate the target first, then retry with \
- dispatch:\"auto\" (PostMessage WM_MOUSEMOVE works against an \
- already-foreground window in most cases). TODO: add a \
- send_drag_synthesized helper analogous to send_click_synthesized."
- .to_string(),
- );
+ let btn_fg = button.clone();
+ let prev_fg_addr = unsafe {
+ windows::Win32::UI::WindowsAndMessaging::GetForegroundWindow().0 as usize
+ };
+ let send_result = tokio::task::spawn_blocking(move || {
+ crate::input::mouse::send_drag_synthesized(
+ hwnd,
+ sx_from, sy_from,
+ sx_to, sy_to,
+ duration_ms, steps, &btn_fg,
+ )
+ }).await;
+ tokio::spawn(restore_foreground_polling_best_effort(prev_fg_addr, pid));
+ let button_suffix = if button == "left" { String::new() } else { format!(" ({} button)", button) };
+ return match send_result {
+ Ok(Ok(())) => ToolResult::text(format!(
+ "✅ Sent drag{button_suffix} via SendInput on pid {raw_pid} \
+ from screen ({sx_from},{sy_from}) → ({sx_to},{sy_to}) \
+ in {duration_ms}ms / {steps} steps (dispatch:foreground)."
+ )),
+ Ok(Err(e)) => ToolResult::error(e.to_string()),
+ Err(e) => ToolResult::error(format!("Task error: {e}")),
+ };
}
// Pin the agent-cursor overlay above the drag target so the synthetic
diff --git a/libs/cua-driver/rust/crates/platform-windows/src/tools/page.rs b/libs/cua-driver/rust/crates/platform-windows/src/tools/page.rs
index 120d6999d3..0481dd8b44 100644
--- a/libs/cua-driver/rust/crates/platform-windows/src/tools/page.rs
+++ b/libs/cua-driver/rust/crates/platform-windows/src/tools/page.rs
@@ -166,18 +166,30 @@ impl PageBackend for WindowsPageBackend {
// The wrapper paths return the result JSON-stringified. The bookmark
// wrapper wraps the user expression's return in JSON.stringify(),
- // which means the inner string itself is JSON-encoded — we need to
- // parse twice to get the actual coord object.
- let parsed: serde_json::Value = serde_json::from_str(&probe_json)
- .or_else(|_| {
- // Bookmark wrapper double-encoded the string — decode the
- // outer JSON-string, then parse the inner JSON.
- let inner: String = serde_json::from_str(&probe_json)?;
- serde_json::from_str::<serde_json::Value>(&inner)
- })
- .map_err(|e| anyhow::anyhow!(
- "click_element: could not parse coord JSON from probe (raw: {probe_raw:?}): {e}"
- ))?;
+ // and CDP's runtime.evaluate also serialises the JS return as a
+ // JSON-string when it's not a primitive — either way the inner
+ // string itself is JSON-encoded. We need to parse twice to get
+ // the actual coord object.
+ //
+ // Crucially, the first parse can SUCCEED as a Value::String (when
+ // the outer is a quoted JSON-string), so we can't rely on
+ // `.or_else(|_| ...)` to trigger the second decode — that branch
+ // only fires on a hard parse error. Inspect the parsed value and
+ // re-decode when it's a String.
+ let parsed: serde_json::Value = {
+ let first = serde_json::from_str::<serde_json::Value>(&probe_json)
+ .map_err(|e| anyhow::anyhow!(
+ "click_element: could not parse coord JSON from probe (raw: {probe_raw:?}): {e}"
+ ))?;
+ match first {
+ serde_json::Value::String(inner) => serde_json::from_str(&inner)
+ .map_err(|e| anyhow::anyhow!(
+ "click_element: inner JSON parse failed for double-encoded probe \
+ response (raw: {probe_raw:?}): {e}"
+ ))?,
+ other => other,
+ }
+ };
// Required-field validation. The previous version defaulted any
// missing field to 0.0, which silently animated the visible cursor
diff --git a/libs/cua-driver/rust/crates/platform-windows/src/uia/mod.rs b/libs/cua-driver/rust/crates/platform-windows/src/uia/mod.rs
index 39cb9fb9e9..76e9dacf21 100644
--- a/libs/cua-driver/rust/crates/platform-windows/src/uia/mod.rs
+++ b/libs/cua-driver/rust/crates/platform-windows/src/uia/mod.rs
@@ -19,7 +19,7 @@ use windows::Win32::UI::Accessibility::{
UIA_ProcessIdPropertyId, UIA_ValueValuePropertyId,
UIA_InvokePatternId, UIA_SelectionItemPatternId,
UIA_TogglePatternId, UIA_ExpandCollapsePatternId, UIA_TextPatternId,
- UIA_ValuePatternId, UIA_ScrollPatternId,
+ UIA_ValuePatternId, UIA_RangeValuePatternId, UIA_ScrollPatternId,
TreeScope_Children, TreeScope_Subtree,
};
@@ -100,6 +100,7 @@ unsafe fn walk_tree_unsafe(hwnd: u64, query: Option<&str>) -> UiaTreeResult {
UIA_SelectionItemPatternId,
UIA_ExpandCollapsePatternId,
UIA_ValuePatternId,
+ UIA_RangeValuePatternId,
UIA_TextPatternId,
UIA_ScrollPatternId,
] {
@@ -522,6 +523,13 @@ fn detect_cached_actions(element: &IUIAutomationElement, is_enabled: bool) -> Ve
if element.GetCachedPattern(UIA_ValuePatternId).is_ok() {
actions.push("set_value".into());
}
+ // RangeValuePattern is exposed by Sliders, ProgressBars, and other
+ // numeric-range controls. Without this entry the slider parent
+ // gets actions=[] → marked non-actionable → no `[N]` index in the
+ // flat tree, making the slider unaddressable by AutomationId.
+ if element.GetCachedPattern(UIA_RangeValuePatternId).is_ok() {
+ actions.push("set_value".into());
+ }
if element.GetCachedPattern(UIA_TextPatternId).is_ok() {
actions.push("text".into());
}