Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

feat(cua-driver-rs)(harness): WebView + Electron + background-modality + capture-mode coverage (#1699)

Francesco Bonaccid7981d199113
diff --git a/libs/cua-driver/rust/Cargo.lock b/libs/cua-driver/rust/Cargo.lock
index 8a6a7c6c8b..abed552ec8 100644
--- a/libs/cua-driver/rust/Cargo.lock
+++ b/libs/cua-driver/rust/Cargo.lock
@@ -266,6 +266,7 @@ dependencies = [
"ureq",
"uuid",
"wait-timeout",
+ "windows 0.61.3",
]
[[package]]
@@ -430,7 +431,7 @@ checksum = "98de4bbd547a563b716d8dfa9aad1cb19bfab00f4fa09a6a4ed21dbcf44ce9c4"
name = "focus-monitor-win"
version = "0.2.18"
dependencies = [
- "windows",
+ "windows 0.58.0",
]
[[package]]
@@ -562,7 +563,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1bd49230192a3797a9a4d6abe9b3eed6f7fa4c8a8a4947977c6f80025f92cbd8"
dependencies = [
"rustix",
- "windows-link",
+ "windows-link 0.2.1",
]
[[package]]
@@ -1123,7 +1124,7 @@ dependencies = [
"libc",
"redox_syscall",
"smallvec",
- "windows-link",
+ "windows-link 0.2.1",
]
[[package]]
@@ -1216,7 +1217,7 @@ dependencies = [
"tiny-skia",
"tokio",
"tracing",
- "windows",
+ "windows 0.58.0",
]
[[package]]
@@ -2288,23 +2289,69 @@ version = "0.58.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dd04d41d93c4992d421894c18c8b43496aa748dd4c081bac0dc93eb0489272b6"
dependencies = [
- "windows-core",
+ "windows-core 0.58.0",
"windows-targets",
]
+[[package]]
+name = "windows"
+version = "0.61.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9babd3a767a4c1aef6900409f85f5d53ce2544ccdfaa86dad48c91782c6d6893"
+dependencies = [
+ "windows-collections",
+ "windows-core 0.61.2",
+ "windows-future",
+ "windows-link 0.1.3",
+ "windows-numerics",
+]
+
+[[package]]
+name = "windows-collections"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3beeceb5e5cfd9eb1d76b381630e82c4241ccd0d27f1a39ed41b2760b255c5e8"
+dependencies = [
+ "windows-core 0.61.2",
+]
+
[[package]]
name = "windows-core"
version = "0.58.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6ba6d44ec8c2591c134257ce647b7ea6b20335bf6379a27dac5f1641fcf59f99"
dependencies = [
- "windows-implement",
- "windows-interface",
- "windows-result",
- "windows-strings",
+ "windows-implement 0.58.0",
+ "windows-interface 0.58.0",
+ "windows-result 0.2.0",
+ "windows-strings 0.1.0",
"windows-targets",
]
+[[package]]
+name = "windows-core"
+version = "0.61.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c0fdd3ddb90610c7638aa2b3a3ab2904fb9e5cdbecc643ddb3647212781c4ae3"
+dependencies = [
+ "windows-implement 0.60.2",
+ "windows-interface 0.59.3",
+ "windows-link 0.1.3",
+ "windows-result 0.3.4",
+ "windows-strings 0.4.2",
+]
+
+[[package]]
+name = "windows-future"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fc6a41e98427b19fe4b73c550f060b59fa592d7d686537eebf9385621bfbad8e"
+dependencies = [
+ "windows-core 0.61.2",
+ "windows-link 0.1.3",
+ "windows-threading",
+]
+
[[package]]
name = "windows-implement"
version = "0.58.0"
@@ -2316,6 +2363,17 @@ dependencies = [
"syn",
]
+[[package]]
+name = "windows-implement"
+version = "0.60.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
[[package]]
name = "windows-interface"
version = "0.58.0"
@@ -2327,12 +2385,39 @@ dependencies = [
"syn",
]
+[[package]]
+name = "windows-interface"
+version = "0.59.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "windows-link"
+version = "0.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5e6ad25900d524eaabdbbb96d20b4311e1e7ae1699af4fb28c17ae66c80d798a"
+
[[package]]
name = "windows-link"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5"
+[[package]]
+name = "windows-numerics"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9150af68066c4c5c07ddc0ce30421554771e528bde427614c61038bc2c92c2b1"
+dependencies = [
+ "windows-core 0.61.2",
+ "windows-link 0.1.3",
+]
+
[[package]]
name = "windows-result"
version = "0.2.0"
@@ -2342,16 +2427,34 @@ dependencies = [
"windows-targets",
]
+[[package]]
+name = "windows-result"
+version = "0.3.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "56f42bd332cc6c8eac5af113fc0c1fd6a8fd2aa08a0119358686e5160d0586c6"
+dependencies = [
+ "windows-link 0.1.3",
+]
+
[[package]]
name = "windows-strings"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4cd9b125c486025df0eabcb585e62173c6c9eddcec5d117d3b6e8c30e2ee4d10"
dependencies = [
- "windows-result",
+ "windows-result 0.2.0",
"windows-targets",
]
+[[package]]
+name = "windows-strings"
+version = "0.4.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "56e6c93f3a0c3b36176cb1327a4958a0353d5d166c2a35cb268ace15e91d3b57"
+dependencies = [
+ "windows-link 0.1.3",
+]
+
[[package]]
name = "windows-sys"
version = "0.52.0"
@@ -2367,7 +2470,7 @@ version = "0.61.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc"
dependencies = [
- "windows-link",
+ "windows-link 0.2.1",
]
[[package]]
@@ -2386,6 +2489,15 @@ dependencies = [
"windows_x86_64_msvc",
]
+[[package]]
+name = "windows-threading"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b66463ad2e0ea3bbf808b7f1d371311c80e115c0b71d60efc142cafbcfb057a6"
+dependencies = [
+ "windows-link 0.1.3",
+]
+
[[package]]
name = "windows_aarch64_gnullvm"
version = "0.52.6"
diff --git a/libs/cua-driver/rust/crates/cua-driver/Cargo.toml b/libs/cua-driver/rust/crates/cua-driver/Cargo.toml
index e4d58eb91a..f165e1c9d6 100644
--- a/libs/cua-driver/rust/crates/cua-driver/Cargo.toml
+++ b/libs/cua-driver/rust/crates/cua-driver/Cargo.toml
@@ -68,3 +68,4 @@ tempfile = "3"
[target.'cfg(target_os = "windows")'.dev-dependencies]
platform-windows = { path = "../platform-windows" }
+windows = { version = "0.61", features = ["Win32_UI_WindowsAndMessaging", "Win32_Foundation"] }
diff --git a/libs/cua-driver/rust/crates/cua-driver/tests/harness_bg_modality_test.rs b/libs/cua-driver/rust/crates/cua-driver/tests/harness_bg_modality_test.rs
new file mode 100644
index 0000000000..d91c0408b8
--- /dev/null
+++ b/libs/cua-driver/rust/crates/cua-driver/tests/harness_bg_modality_test.rs
@@ -0,0 +1,498 @@
+//! Background-modality + capture-mode tests for the CuaTestHarness.Wpf
+//! harness.
+//!
+//! These verify the **core cua-driver promise**: background automation
+//! must not steal foreground from the user's active window. The harness
+//! window is at z+0 when shown (whatever WPF gives it), the
+//! focus-monitor-win sentinel is then activated to z+0 (displacing the
+//! harness to z+1), and cua-driver actions targeting the harness must
+//! NOT make the harness regain foreground.
+//!
+//! Sentinel: `focus-monitor-win` (already part of the workspace; built by
+//! `cargo build`). It writes `focus_monitor_losses.txt` to %TEMP% — the
+//! count of times its window lost activation. We snapshot before / after
+//! each action and assert delta == 0.
+//!
+//! Also covers **capture_mode ax** (UIA-only, no screenshot) and
+//! **capture_mode vision** (screenshot-only, no UIA tree). The default
+//! `som` covers both.
+
+#![cfg(target_os = "windows")]
+
+use std::io::{BufRead, BufReader, Write};
+use std::path::PathBuf;
+use std::process::{Child, ChildStdin, ChildStdout, Command, Stdio};
+use std::time::Duration;
+
+// ── paths ────────────────────────────────────────────────────────────────────
+
+fn workspace_root() -> PathBuf {
+ let manifest = std::env::var("CARGO_MANIFEST_DIR").expect("CARGO_MANIFEST_DIR");
+ PathBuf::from(manifest).parent().unwrap().parent().unwrap().to_owned()
+}
+fn driver_binary() -> PathBuf { workspace_root().join("target/debug/cua-driver.exe") }
+fn focus_monitor_binary() -> PathBuf { workspace_root().join("target/debug/focus-monitor-win.exe") }
+fn harness_wpf_exe() -> PathBuf {
+ if let Ok(p) = std::env::var("HARNESS_WPF_EXE") {
+ let pb = PathBuf::from(p);
+ if pb.exists() { return pb; }
+ }
+ workspace_root().join("test-apps/harness-wpf/CuaTestHarness.Wpf.exe")
+}
+
+fn loss_file() -> PathBuf { std::env::temp_dir().join("focus_monitor_losses.txt") }
+fn gain_file() -> PathBuf { std::env::temp_dir().join("focus_monitor_gains.txt") }
+fn key_loss_file() -> PathBuf { std::env::temp_dir().join("focus_monitor_key_losses.txt") }
+fn key_gain_file() -> PathBuf { std::env::temp_dir().join("focus_monitor_key_gains.txt") }
+fn focus_pid_file() -> PathBuf { std::env::temp_dir().join("focus_monitor_pid.txt") }
+fn focus_hwnd_file() -> PathBuf { std::env::temp_dir().join("focus_monitor_hwnd.txt") }
+
+fn read_count(p: &std::path::Path) -> u32 {
+ // Surface read / parse errors instead of swallowing them as 0 — a silent
+ // 0 would mask actual focus-steal regressions by making delta computation
+ // look "no change" when really the sentinel file is unreadable.
+ let raw = std::fs::read_to_string(p)
+ .unwrap_or_else(|e| panic!("failed reading sentinel counter {p:?}: {e}"));
+ raw.trim()
+ .parse::<u32>()
+ .unwrap_or_else(|e| panic!("failed parsing sentinel counter {p:?} as u32: {e}"))
+}
+
+// ── JSON-RPC ─────────────────────────────────────────────────────────────────
+
+fn send(stdin: &mut ChildStdin, req: serde_json::Value) {
+ writeln!(stdin, "{}", serde_json::to_string(&req).unwrap()).unwrap();
+}
+fn recv(stdout: &mut BufReader<&mut ChildStdout>) -> serde_json::Value {
+ let mut line = String::new();
+ stdout.read_line(&mut line).expect("read");
+ serde_json::from_str(line.trim()).expect("json")
+}
+fn init(stdin: &mut ChildStdin, stdout: &mut BufReader<&mut ChildStdout>) {
+ send(stdin, serde_json::json!({"jsonrpc":"2.0","id":1,"method":"initialize","params":{}}));
+ let _ = recv(stdout);
+}
+fn call(stdin: &mut ChildStdin, stdout: &mut BufReader<&mut ChildStdout>,
+ id: u32, name: &str, args: serde_json::Value) -> serde_json::Value {
+ send(stdin, serde_json::json!({
+ "jsonrpc":"2.0","id":id,"method":"tools/call",
+ "params":{"name":name,"arguments":args}
+ }));
+ recv(stdout)
+}
+
+fn snapshot_text(s: &serde_json::Value) -> &str {
+ s["result"]["content"][0]["text"].as_str().unwrap_or("")
+}
+fn find_idx_by_aid(s: &serde_json::Value, aid: &str) -> Option<u64> {
+ let needle = format!("id={aid}");
+ for line in snapshot_text(s).lines() {
+ if !line.contains(&needle) { continue; }
+ let st = line.find('[')? + 1;
+ let en = line[st..].find(']')? + st;
+ return line[st..en].trim().parse().ok();
+ }
+ None
+}
+
+// ── shared fixture ───────────────────────────────────────────────────────────
+
+struct BgFixture {
+ harness: Child,
+ fm: Child,
+ driver: Child,
+ driver_stdin: ChildStdin,
+ driver_stdout: ChildStdout,
+ harness_pid: u32,
+ harness_wid: u64,
+}
+
+impl Drop for BgFixture {
+ fn drop(&mut self) {
+ let _ = self.driver.kill();
+ let _ = self.driver.wait();
+ let _ = self.harness.kill();
+ let _ = self.harness.wait();
+ let _ = self.fm.kill();
+ let _ = self.fm.wait();
+ std::thread::sleep(Duration::from_millis(300));
+ }
+}
+
+/// Launch sequence:
+/// 1. WPF harness (becomes foreground briefly on its own activation)
+/// 2. focus-monitor-win sentinel — its OnLoad SetForegroundWindow displaces
+/// the harness; sentinel is now z+0, harness z+1.
+/// 3. cua-driver child
+/// 4. Reset losses.txt to 0 (sentinel may have logged its own startup activate)
+fn setup() -> Option<BgFixture> {
+ let driver_bin = driver_binary();
+ if !driver_bin.exists() {
+ eprintln!("cua-driver.exe not built — skipping"); return None;
+ }
+ let fm_bin = focus_monitor_binary();
+ if !fm_bin.exists() {
+ eprintln!("focus-monitor-win.exe not built — skipping"); return None;
+ }
+ let h_exe = harness_wpf_exe();
+ if !h_exe.exists() {
+ eprintln!("harness WPF exe not built — skipping"); return None;
+ }
+
+ // Reset sentinel files so we start from a known baseline.
+ let _ = std::fs::write(loss_file(), "0");
+ let _ = std::fs::write(gain_file(), "0");
+ let _ = std::fs::write(key_loss_file(), "0");
+ let _ = std::fs::write(key_gain_file(), "0");
+ let _ = std::fs::remove_file(focus_pid_file());
+ let _ = std::fs::remove_file(focus_hwnd_file());
+
+ let mut harness = Command::new(&h_exe)
+ .stdout(Stdio::null()).stderr(Stdio::null())
+ .spawn().ok()?;
+ let harness_pid = harness.id();
+ std::thread::sleep(Duration::from_secs(1));
+
+ // Launch sentinel. focus-monitor-win activates its own window which
+ // displaces the harness.
+ let mut fm = match Command::new(&fm_bin)
+ .stdout(Stdio::null()).stderr(Stdio::null())
+ .spawn()
+ {
+ Ok(c) => c,
+ Err(e) => {
+ eprintln!("focus-monitor-win spawn failed: {e}");
+ // Reap the harness we already spawned so it doesn't leak.
+ let _ = harness.kill();
+ let _ = harness.wait();
+ return None;
+ }
+ };
+ // Wait for sentinel to publish its pid+hwnd files (so we know it's
+ // up and has claimed foreground). On timeout we MUST reap both child
+ // processes so they don't poison subsequent tests.
+ let deadline = std::time::Instant::now() + Duration::from_secs(10);
+ loop {
+ let pid_ok = std::fs::read_to_string(focus_pid_file()).ok()
+ .and_then(|s| s.trim().parse::<u32>().ok()).unwrap_or(0) != 0;
+ let hwnd_ok = std::fs::read_to_string(focus_hwnd_file()).ok()
+ .and_then(|s| s.trim().parse::<u64>().ok()).unwrap_or(0) != 0;
+ if pid_ok && hwnd_ok { break; }
+ if std::time::Instant::now() > deadline {
+ eprintln!("focus-monitor sentinel never published pid/hwnd files");
+ let _ = harness.kill();
+ let _ = harness.wait();
+ let _ = fm.kill();
+ let _ = fm.wait();
+ return None;
+ }
+ std::thread::sleep(Duration::from_millis(100));
+ }
+ std::thread::sleep(Duration::from_millis(400)); // sentinel-active settle
+
+ // Reset losses again now that sentinel has settled. Any prior loss
+ // counts (e.g. from harness coming up after the sentinel did) shouldn't
+ // count against the test.
+ let _ = std::fs::write(loss_file(), "0");
+ let _ = std::fs::write(gain_file(), "0");
+ let _ = std::fs::write(key_loss_file(), "0");
+ let _ = std::fs::write(key_gain_file(), "0");
+
+ let mut driver = match Command::new(&driver_bin)
+ .stdin(Stdio::piped()).stdout(Stdio::piped()).stderr(Stdio::null())
+ .spawn()
+ {
+ Ok(d) => d,
+ Err(e) => {
+ eprintln!("cua-driver spawn failed: {e}");
+ let _ = harness.kill(); let _ = harness.wait();
+ let _ = fm.kill(); let _ = fm.wait();
+ return None;
+ }
+ };
+ let mut driver_stdin = driver.stdin.take().unwrap();
+ let driver_stdout_raw = driver.stdout.take().unwrap();
+ let mut driver_stdout = driver_stdout_raw;
+ {
+ let mut stdout = BufReader::new(&mut driver_stdout);
+ init(&mut driver_stdin, &mut stdout);
+ let resp = call(&mut driver_stdin, &mut stdout, 10, "list_windows",
+ serde_json::json!({"pid": harness_pid as i64}));
+ let wid = resp["result"]["structuredContent"]["windows"].as_array()
+ .and_then(|a| a.iter().find_map(|w| {
+ if w["pid"].as_u64() == Some(harness_pid as u64)
+ && w["title"].as_str().map(|t| t.contains("CuaTestHarness WPF")).unwrap_or(false)
+ { w["window_id"].as_u64() } else { None }
+ }))
+ .expect("harness window not found");
+ drop(stdout);
+ return Some(BgFixture {
+ harness, fm, driver, driver_stdin, driver_stdout,
+ harness_pid, harness_wid: wid,
+ });
+ }
+}
+
+fn with_session<F, R>(fx: &mut BgFixture, f: F) -> R
+where F: FnOnce(&mut ChildStdin, &mut BufReader<&mut ChildStdout>, u32, u64) -> R {
+ let mut stdout = BufReader::new(&mut fx.driver_stdout);
+ f(&mut fx.driver_stdin, &mut stdout, fx.harness_pid, fx.harness_wid)
+}
+
+/// Strict mode: action must not generate ANY sentinel-loss event.
+fn assert_no_focus_steal<F>(label: &str, f: F)
+where F: FnOnce() {
+ let before_act = read_count(&loss_file());
+ let before_key = read_count(&key_loss_file());
+ f();
+ std::thread::sleep(Duration::from_millis(400));
+ let after_act = read_count(&loss_file());
+ let after_key = read_count(&key_loss_file());
+ let d_act = after_act.saturating_sub(before_act);
+ let d_key = after_key.saturating_sub(before_key);
+ assert_eq!(d_act, 0,
+ "{label}: sentinel act_losses went {before_act} -> {after_act} (delta={d_act}). \
+ cua-driver's background action stole foreground from the user.");
+ assert_eq!(d_key, 0,
+ "{label}: sentinel key_losses went {before_key} -> {after_key} (delta={d_key}).");
+ println!("✅ {label}: no focus steal (act_losses=0, key_losses=0)");
+}
+
+/// Relaxed mode: read GetForegroundWindow() after the action and assert
+/// it's still the sentinel HWND. Tolerates a transient blip during the
+/// action (which UIA Invoke against WPF Buttons creates unavoidably —
+/// the target's handler calls Focus() before cua-driver gets control).
+/// The cua-driver fg_bypass restores foreground after, and this check
+/// asserts that restoration actually worked.
+fn assert_foreground_restored<F>(label: &str, f: F)
+where F: FnOnce() {
+ let sentinel_hwnd: u64 = std::fs::read_to_string(focus_hwnd_file())
+ .ok().and_then(|s| s.trim().parse().ok()).unwrap_or(0);
+ assert!(sentinel_hwnd != 0, "{label}: sentinel hwnd unknown (file missing)");
+
+ let before_loss = read_count(&loss_file());
+ f();
+ std::thread::sleep(Duration::from_millis(500));
+ let after_loss = read_count(&loss_file());
+ let d_loss = after_loss.saturating_sub(before_loss);
+
+ // Read current foreground via Win32 directly.
+ let now_fg: u64 = unsafe {
+ let h = windows::Win32::UI::WindowsAndMessaging::GetForegroundWindow();
+ h.0 as u64
+ };
+ assert_eq!(now_fg, sentinel_hwnd,
+ "{label}: GetForegroundWindow={now_fg:#x} but sentinel hwnd={sentinel_hwnd:#x}. \
+ Foreground was NOT restored to the user's window after the action.");
+ if d_loss == 0 {
+ println!("✅ {label}: no focus blip at all (losses=0)");
+ } else {
+ println!("✅ {label}: foreground restored after {d_loss} blip(s) — sentinel HWND={sentinel_hwnd:#x} matches GetForegroundWindow");
+ }
+}
+
+// ── BACKGROUND MODALITY: cua-driver must keep harness at z+1 ────────────────
+
+#[test]
+#[ignore]
+fn bg_modality_get_window_state_no_focus_steal() {
+ let mut fx = match setup() { Some(f) => f, None => return };
+ with_session(&mut fx, |stdin, stdout, pid, wid| {
+ assert_no_focus_steal("get_window_state(som)", || {
+ let _ = call(stdin, stdout, 30, "get_window_state",
+ serde_json::json!({"pid": pid as i64, "window_id": wid, "capture_mode": "som"}));
+ });
+ });
+}
+
+/// Documents the cua-driver focus-steal gap for UIA Invoke on a WPF
+/// Button. Root cause: WPF's ButtonBase.OnClick handler synchronously
+/// calls `UIElement.Focus()` which routes through `SetForegroundWindow`
+/// and is NOT gated by the EnableWindow(false) bypass used for UWP
+/// hosts. The daemon CAN'T restore foreground reliably either, because
+/// non-UIAccess processes are subject to the foreground-lock.
+///
+/// **Mitigation**: route UIA activations through `cua-driver-uia.exe`
+/// (UIAccess-manifested worker). With UIAccess, the worker can both
+/// suppress the self-foreground and restore the user's foreground if
+/// it leaked through.
+///
+/// This test ASSERTS the gap currently exists. When cua-driver gains
+/// the UIAccess worker path, flip this assertion to `delta == 0` and
+/// rename to `..._no_focus_steal`.
+#[test]
+#[ignore]
+fn bg_modality_uia_invoke_click_DOCUMENTED_steals_focus() {
+ let mut fx = match setup() { Some(f) => f, None => return };
+ with_session(&mut fx, |stdin, stdout, pid, wid| {
+ let _ = call(stdin, stdout, 29, "set_agent_cursor_enabled",
+ serde_json::json!({"enabled": false}));
+ std::thread::sleep(Duration::from_millis(200));
+
+ let snap = call(stdin, stdout, 30, "get_window_state",
+ serde_json::json!({"pid": pid as i64, "window_id": wid, "capture_mode": "ax"}));
+ let idx = find_idx_by_aid(&snap, "btn-increment").expect("btn-increment");
+
+ let before = read_count(&loss_file());
+ let _ = call(stdin, stdout, 31, "click",
+ serde_json::json!({"pid": pid as i64, "window_id": wid, "element_index": idx}));
+ std::thread::sleep(Duration::from_millis(500));
+ let after = read_count(&loss_file());
+ let delta = after.saturating_sub(before);
+ assert!(delta >= 1,
+ "Expected the documented focus-steal gap (delta>=1). Got delta={delta}. \
+ If this now passes, cua-driver has fixed UIA Invoke focus-steal — \
+ flip this assertion to delta==0 and update the docstring.");
+ println!("⚠️ bg_modality_uia_invoke_click: gap confirmed (delta={delta}). \
+ Mitigation = route UIA via cua-driver-uia.exe (UIAccess worker).");
+ });
+}
+
+/// Companion gap: UIA ValuePattern.SetValue on WPF TextBox.
+/// Same root cause and mitigation path as
+/// bg_modality_uia_invoke_click_DOCUMENTED_steals_focus.
+#[test]
+#[ignore]
+fn bg_modality_set_value_DOCUMENTED_steals_focus() {
+ let mut fx = match setup() { Some(f) => f, None => return };
+ with_session(&mut fx, |stdin, stdout, pid, wid| {
+ let _ = call(stdin, stdout, 29, "set_agent_cursor_enabled",
+ serde_json::json!({"enabled": false}));
+ std::thread::sleep(Duration::from_millis(200));
+
+ let snap = call(stdin, stdout, 30, "get_window_state",
+ serde_json::json!({"pid": pid as i64, "window_id": wid, "capture_mode": "ax"}));
+ let idx = find_idx_by_aid(&snap, "txt-input").expect("txt-input");
+
+ let before = read_count(&loss_file());
+ let _ = call(stdin, stdout, 31, "set_value",
+ serde_json::json!({
+ "pid": pid as i64, "window_id": wid, "element_index": idx,
+ "value": "via-uia-no-focus-steal"
+ }));
+ std::thread::sleep(Duration::from_millis(500));
+ let after = read_count(&loss_file());
+ let delta = after.saturating_sub(before);
+ assert!(delta >= 1,
+ "Expected the documented SetValue focus-steal gap. delta={delta}.");
+ println!("⚠️ bg_modality_set_value: gap confirmed (delta={delta})");
+ });
+}
+
+#[test]
+#[ignore]
+fn bg_modality_press_key_no_focus_steal() {
+ let mut fx = match setup() { Some(f) => f, None => return };
+ with_session(&mut fx, |stdin, stdout, pid, wid| {
+ // F5 fires the harness accelerator (KeyBinding) via PostMessage
+ // WM_KEYDOWN — background path, no foreground swap.
+ assert_no_focus_steal("press_key(f5, PostMessage)", || {
+ let _ = call(stdin, stdout, 30, "press_key",
+ serde_json::json!({"pid": pid as i64, "window_id": wid, "key": "f5"}));
+ });
+ });
+}
+
+#[test]
+#[ignore]
+fn bg_modality_scroll_no_focus_steal() {
+ let mut fx = match setup() { Some(f) => f, None => return };
+ with_session(&mut fx, |stdin, stdout, pid, wid| {
+ assert_no_focus_steal("scroll(down, PostMessage WM_VSCROLL)", || {
+ let _ = call(stdin, stdout, 30, "scroll",
+ serde_json::json!({
+ "pid": pid as i64, "window_id": wid,
+ "direction": "down", "by": "line", "amount": 3
+ }));
+ });
+ });
+}
+
+// ── CAPTURE MODE: ax + vision modalities ────────────────────────────────────
+
+#[test]
+#[ignore]
+fn capture_mode_ax_returns_tree_only() {
+ let mut fx = match setup() { Some(f) => f, None => return };
+ with_session(&mut fx, |stdin, stdout, pid, wid| {
+ let resp = call(stdin, stdout, 30, "get_window_state",
+ serde_json::json!({"pid": pid as i64, "window_id": wid, "capture_mode": "ax"}));
+ // ax mode: tree_markdown present, no image content array entry.
+ let text = snapshot_text(&resp);
+ assert!(text.contains("id=btn-increment"),
+ "capture_mode=ax tree missing btn-increment AID");
+ let has_image = resp["result"]["content"].as_array()
+ .map(|a| a.iter().any(|c| c["type"].as_str() == Some("image")))
+ .unwrap_or(false);
+ assert!(!has_image,
+ "capture_mode=ax should NOT return image content (got one anyway)");
+ println!("✅ capture_mode_ax_returns_tree_only: tree present, no image");
+
+ // And ax must not steal focus.
+ assert_no_focus_steal("get_window_state(ax)", || {
+ let _ = call(stdin, stdout, 31, "get_window_state",
+ serde_json::json!({"pid": pid as i64, "window_id": wid, "capture_mode": "ax"}));
+ });
+ });
+}
+
+#[test]
+#[ignore]
+fn capture_mode_vision_returns_image_only() {
+ let mut fx = match setup() { Some(f) => f, None => return };
+ with_session(&mut fx, |stdin, stdout, pid, wid| {
+ let resp = call(stdin, stdout, 30, "get_window_state",
+ serde_json::json!({"pid": pid as i64, "window_id": wid, "capture_mode": "vision"}));
+ // vision mode: image content present, no tree markdown.
+ let has_image = resp["result"]["content"].as_array()
+ .map(|a| a.iter().any(|c| c["type"].as_str() == Some("image")))
+ .unwrap_or(false);
+ assert!(has_image, "capture_mode=vision should return image content");
+
+ let text_first = snapshot_text(&resp);
+ // Tree markdown's hallmark is lines starting with `- [N] ` for elements.
+ // In vision mode, those should be absent.
+ let has_tree_markers = text_first.lines()
+ .any(|l| l.trim_start().starts_with("- [") && l.contains(']'));
+ assert!(!has_tree_markers,
+ "capture_mode=vision should not return UIA tree markdown");
+ println!("✅ capture_mode_vision_returns_image_only: image present, no tree");
+
+ assert_no_focus_steal("get_window_state(vision)", || {
+ let _ = call(stdin, stdout, 31, "get_window_state",
+ serde_json::json!({"pid": pid as i64, "window_id": wid, "capture_mode": "vision"}));
+ });
+ });
+}
+
+#[test]
+#[ignore]
+fn capture_mode_ax_and_vision_invoke_roundtrip() {
+ // Cross-modality round-trip: ax to find element, vision to confirm
+ // the screenshot reflects the post-action state. Mirrors how an agent
+ // alternates between symbolic and pixel views during a task.
+ let mut fx = match setup() { Some(f) => f, None => return };
+ with_session(&mut fx, |stdin, stdout, pid, wid| {
+ let snap_ax = call(stdin, stdout, 30, "get_window_state",
+ serde_json::json!({"pid": pid as i64, "window_id": wid, "capture_mode": "ax"}));
+ let idx = find_idx_by_aid(&snap_ax, "btn-increment").expect("btn-increment");
+
+ let _ = call(stdin, stdout, 31, "click",
+ serde_json::json!({"pid": pid as i64, "window_id": wid, "element_index": idx}));
+ std::thread::sleep(Duration::from_millis(300));
+
+ let snap_vision = call(stdin, stdout, 32, "get_window_state",
+ serde_json::json!({"pid": pid as i64, "window_id": wid, "capture_mode": "vision"}));
+ let has_image = snap_vision["result"]["content"].as_array()
+ .map(|a| a.iter().any(|c| c["type"].as_str() == Some("image")))
+ .unwrap_or(false);
+ assert!(has_image, "vision snapshot didn't return an image");
+
+ // And confirm counter advanced via a follow-up ax snapshot.
+ let snap_ax2 = call(stdin, stdout, 33, "get_window_state",
+ serde_json::json!({"pid": pid as i64, "window_id": wid, "capture_mode": "ax"}));
+ assert!(snapshot_text(&snap_ax2).contains("counter=1"),
+ "counter didn't advance after UIA Invoke");
+ println!("✅ capture_mode_ax_and_vision_invoke_roundtrip: ax→invoke→vision+ax green");
+ });
+}
diff --git a/libs/cua-driver/rust/crates/cua-driver/tests/harness_web_test.rs b/libs/cua-driver/rust/crates/cua-driver/tests/harness_web_test.rs
new file mode 100644
index 0000000000..e37f8d4848
--- /dev/null
+++ b/libs/cua-driver/rust/crates/cua-driver/tests/harness_web_test.rs
@@ -0,0 +1,255 @@
+//! Integration tests against the CuaTestHarness.WebView (WPF + WebView2)
+//! and CuaTestHarness.Electron hosts. Both load the same
+//! `test-harness/shared-web/index.html`, so the same `page` tool flows
+//! are exercised against two Chromium-based hosts.
+//!
+//! Run via:
+//! cargo test --test harness_web_test -- --ignored --nocapture
+//!
+//! ## Known cua-driver gaps these tests document
+//!
+//! - **CDP `/json` HTTP read uses `read_to_end`** — `mcp-server/src/cdp.rs`
+//! sends `Connection: close` and then calls `stream.read_to_end()`, but
+//! Chromium's CDP HTTP server ignores `Connection: close` and keeps the
+//! socket alive, so `read_to_end` hangs until the 10 s discovery timeout.
+//! Confirmed against Electron 31 on port 9223 (verified manually via
+//! curl: instant 200, JSON body present). Fix: parse `Content-Length`
+//! and `read_exact` that many bytes, or honour `Transfer-Encoding:
+//! chunked`. Tracked in this test as a structural assertion (window
+//! discoverable) rather than a behavioural one (page tool round-trip).
+//!
+//! - **WebView2 `--remote-debugging-port` ignored** — passing
+//! `AdditionalBrowserArguments = "--remote-debugging-port=9222"` via
+//! `CoreWebView2EnvironmentOptions` does not open a CDP listener on the
+//! WebView2 helper processes. WebView2 may be filtering the flag.
+//! Tracked here as a TODO for the harness rather than a cua-driver
+//! issue (since this is a WebView2 configuration concern).
+
+#![cfg(target_os = "windows")]
+
+use std::io::{BufRead, BufReader, Write};
+use std::path::PathBuf;
+use std::process::{Child, ChildStdin, ChildStdout, Command, Stdio};
+use std::time::Duration;
+
+// ── workspace paths ──────────────────────────────────────────────────────────
+
+fn workspace_root() -> PathBuf {
+ let manifest = std::env::var("CARGO_MANIFEST_DIR").expect("CARGO_MANIFEST_DIR");
+ PathBuf::from(manifest).parent().unwrap().parent().unwrap().to_owned()
+}
+fn driver_binary() -> PathBuf { workspace_root().join("target/debug/cua-driver.exe") }
+
+fn webview_exe() -> PathBuf {
+ if let Ok(p) = std::env::var("HARNESS_WEBVIEW_EXE") {
+ let pb = PathBuf::from(p);
+ if pb.exists() { return pb; }
+ }
+ workspace_root().join("test-apps/harness-webview/CuaTestHarness.WebView.exe")
+}
+fn electron_exe() -> PathBuf {
+ if let Ok(p) = std::env::var("HARNESS_ELECTRON_EXE") {
+ let pb = PathBuf::from(p);
+ if pb.exists() { return pb; }
+ }
+ workspace_root().join("test-apps/harness-electron/CuaTestHarness.Electron.exe")
+}
+
+// ── JSON-RPC plumbing ────────────────────────────────────────────────────────
+
+fn send(stdin: &mut ChildStdin, req: serde_json::Value) {
+ writeln!(stdin, "{}", serde_json::to_string(&req).unwrap()).unwrap();
+}
+fn recv(stdout: &mut BufReader<&mut ChildStdout>) -> serde_json::Value {
+ let mut line = String::new();
+ stdout.read_line(&mut line).expect("read");
+ serde_json::from_str(line.trim()).expect("json")
+}
+fn init(stdin: &mut ChildStdin, stdout: &mut BufReader<&mut ChildStdout>) {
+ send(stdin, serde_json::json!({"jsonrpc":"2.0","id":1,"method":"initialize","params":{}}));
+ let _ = recv(stdout);
+}
+fn tools_call(stdin: &mut ChildStdin, stdout: &mut BufReader<&mut ChildStdout>,
+ id: u32, name: &str, args: serde_json::Value) -> serde_json::Value {
+ send(stdin, serde_json::json!({
+ "jsonrpc":"2.0","id":id,"method":"tools/call",
+ "params":{"name":name,"arguments":args}
+ }));
+ recv(stdout)
+}
+
+fn find_window_by_title(stdin: &mut ChildStdin, stdout: &mut BufReader<&mut ChildStdout>,
+ pid: u32, title_substr: &str) -> Option<u64> {
+ let deadline = std::time::Instant::now() + Duration::from_secs(20);
+ let mut id = 10u32;
+ loop {
+ let resp = tools_call(stdin, stdout, id, "list_windows", serde_json::json!({"pid": pid as i64}));
+ id = id.wrapping_add(1);
+ if let Some(wins) = resp["result"]["structuredContent"]["windows"].as_array() {
+ for w in wins {
+ if w["pid"].as_u64() != Some(pid as u64) { continue; }
+ if w["title"].as_str().unwrap_or("").contains(title_substr) {
+ if let Some(wid) = w["window_id"].as_u64() { return Some(wid); }
+ }
+ }
+ }
+ if std::time::Instant::now() >= deadline { return None; }
+ std::thread::sleep(Duration::from_millis(200));
+ }
+}
+
+// ── shared session helper ────────────────────────────────────────────────────
+
+/// Launch the harness exe + a cua-driver child with `CUA_DRIVER_CDP_PORT`
+/// pointing at the harness's CDP endpoint. Polls list_windows until the
+/// host's window appears.
+struct WebSession {
+ _app: Child,
+ driver: Child,
+}
+
+impl Drop for WebSession {
+ fn drop(&mut self) {
+ let _ = self.driver.kill();
+ let _ = self.driver.wait();
+ let _ = self._app.kill();
+ let _ = self._app.wait();
+ // settle so the next test's launch doesn't see leftover windows
+ std::thread::sleep(Duration::from_millis(500));
+ }
+}
+
+fn run_with_session<F>(label: &str, host_exe: PathBuf, title_substr: &str, cdp_port: u16, f: F)
+where F: FnOnce(u32, u64, &mut ChildStdin, &mut BufReader<&mut ChildStdout>) {
+ if !driver_binary().exists() {
+ eprintln!("cua-driver.exe not built — run `cargo build` first"); return;
+ }
+ if !host_exe.exists() {
+ eprintln!("{label} host exe not found at {host_exe:?} — run test-harness/build.ps1"); return;
+ }
+ // Set the CDP port the host should use so the daemon can find it.
+ let env_var = if label == "webview" { "CUA_WEBVIEW_CDP_PORT" } else { "CUA_ELECTRON_CDP_PORT" };
+ let app = Command::new(&host_exe)
+ .env(env_var, cdp_port.to_string())
+ .stdout(Stdio::null()).stderr(Stdio::null())
+ .spawn().expect("spawn host");
+ let pid = app.id();
+ println!("{label} pid={pid} cdp_port={cdp_port}");
+ std::thread::sleep(Duration::from_secs(2)); // small cold-start for runtime spin-up
+
+ let mut driver = Command::new(driver_binary())
+ .env("CUA_DRIVER_CDP_PORT", cdp_port.to_string())
+ .stdin(Stdio::piped()).stdout(Stdio::piped()).stderr(Stdio::null())
+ .spawn().expect("spawn cua-driver");
+ let mut stdin = driver.stdin.take().unwrap();
+ let mut raw_stdout = driver.stdout.take().unwrap();
+ let mut stdout = BufReader::new(&mut raw_stdout);
+ init(&mut stdin, &mut stdout);
+
+ let wid = find_window_by_title(&mut stdin, &mut stdout, pid, title_substr)
+ .unwrap_or_else(|| panic!("{label} window with title containing {title_substr:?} not found"));
+
+ let session = WebSession { _app: app, driver };
+ f(pid, wid, &mut stdin, &mut stdout);
+ drop(session);
+}
+
+// ── WebView2 structural ──────────────────────────────────────────────────────
+
+#[test]
+#[ignore]
+fn harness_webview_window_discoverable() {
+ // Smoke test: WebView2 harness launches, window appears via list_windows.
+ // Behavioural page-tool tests are deferred until WebView2 actually opens
+ // its CDP listener — see the module docstring TODO.
+ run_with_session("webview", webview_exe(), "CuaTestHarness WebView", 9222,
+ |pid, wid, _stdin, _stdout| {
+ println!("✅ harness_webview_window_discoverable: pid={pid} wid={wid}");
+ });
+}
+
+// ── Electron structural + page tool ──────────────────────────────────────────
+
+#[test]
+#[ignore]
+fn harness_electron_window_discoverable() {
+ run_with_session("electron", electron_exe(), "CuaTestHarness Electron", 9223,
+ |pid, wid, _stdin, _stdout| {
+ println!("✅ harness_electron_window_discoverable: pid={pid} wid={wid}");
+ });
+}
+
+#[test]
+#[ignore]
+fn harness_electron_page_tool() {
+ // Regression guard for the CDP /json discovery fix (parse
+ // Content-Length / Transfer-Encoding instead of read_to_end).
+ // cua-driver's page tool now reaches Electron's CDP successfully.
+ run_with_session("electron", electron_exe(), "CuaTestHarness Electron", 9223,
+ |pid, wid, stdin, stdout| {
+
+ // 1. execute_javascript via CDP.
+ let marker_resp = tools_call(stdin, stdout, 30, "page", serde_json::json!({
+ "pid": pid as i64, "window_id": wid, "action": "execute_javascript",
+ "javascript": "document.querySelector('[data-cua-id=\"page-marker\"]').textContent"
+ }));
+ let marker = marker_resp["result"]["content"][0]["text"].as_str().unwrap_or("");
+ assert!(marker.contains("WEB_HARNESS_MARKER_v1"),
+ "Electron CDP execute_javascript marker fetch: {marker:?}");
+
+ // 2. Increment counter via direct execute_javascript (the
+ // click_element path has a separate probe-JSON-parsing gap
+ // documented below — track separately).
+ let _ = tools_call(stdin, stdout, 31, "page", serde_json::json!({
+ "pid": pid as i64, "window_id": wid, "action": "execute_javascript",
+ "javascript": "document.getElementById('btn-increment').click()"
+ }));
+ std::thread::sleep(Duration::from_millis(300));
+
+ let counter_resp = tools_call(stdin, stdout, 32, "page", serde_json::json!({
+ "pid": pid as i64, "window_id": wid, "action": "execute_javascript",
+ "javascript": "document.getElementById('lbl-counter').textContent"
+ }));
+ let post = counter_resp["result"]["content"][0]["text"].as_str().unwrap_or("");
+ assert!(post.contains("counter=1"),
+ "Electron counter did not advance via execute_javascript: {post:?}");
+ println!("✅ harness_electron_page_tool: CDP+execute_javascript green");
+ });
+}
+
+/// Documents a separate gap in page.click_element: its CDP probe runs JS
+/// that returns a JSON object (vx/vy/sx/sy/dpr) but the result is wrapped
+/// as a CDP `runtime.evaluate.user_gesture` string by the time the page
+/// tool's JSON parser sees it — causing "probe JSON missing required
+/// field 'vx'". Fix: unwrap the CDP result's `.value` before parsing,
+/// or send the probe with `returnByValue:true` in the Runtime.evaluate
+/// params. Manually unwrapping the harness's verified JSON string works.
+#[test]
+#[ignore]
+fn harness_electron_click_element_DOCUMENTED_wrapper_bug() {
+ run_with_session("electron", electron_exe(), "CuaTestHarness Electron", 9223,
+ |pid, wid, stdin, stdout| {
+ let resp = tools_call(stdin, stdout, 30, "page", serde_json::json!({
+ "pid": pid as i64, "window_id": wid, "action": "click_element",
+ "selector": "#btn-increment"
+ }));
+ // Safe traversal — error responses from `page` can vary in shape
+ // (sometimes `result.isError: true` + `content[0].text`, sometimes
+ // `error.message`). Don't blow up on a missing array entry.
+ let text = resp.get("result")
+ .and_then(|r| r.get("content"))
+ .and_then(|c| c.as_array())
+ .and_then(|arr| arr.get(0))
+ .and_then(|item| item.get("text"))
+ .and_then(|t| t.as_str())
+ .or_else(|| resp.get("error").and_then(|e| e.get("message")).and_then(|m| m.as_str()))
+ .unwrap_or("");
+ // Expect the gap: parser fails on the CDP-wrapped probe response.
+ let expected_gap = text.contains("probe JSON missing") || text.contains("required field");
+ assert!(expected_gap,
+ "Expected click_element probe-wrapper gap. Got: {text:?}. \
+ If this asserts on a success message, the wrapper-unwrap fix is in — \
+ flip this assertion to assert success.");
+ println!("⚠️ harness_electron_click_element_DOCUMENTED_wrapper_bug: probe JSON parse gap present");
+ });
+}
diff --git a/libs/cua-driver/rust/crates/cua-driver/tests/harness_winui3_test.rs b/libs/cua-driver/rust/crates/cua-driver/tests/harness_winui3_test.rs
index d9548de873..6f3feb1e5a 100644
--- a/libs/cua-driver/rust/crates/cua-driver/tests/harness_winui3_test.rs
+++ b/libs/cua-driver/rust/crates/cua-driver/tests/harness_winui3_test.rs
@@ -254,3 +254,125 @@ fn harness_winui3_xaml_popup_open() {
child.kill().ok();
}
+
+// ── Session helper for the additional control tests ──────────────────────────
+
+fn winui3_with_session<F>(f: F)
+where F: FnOnce(u32, u64, &mut ChildStdin, &mut BufReader<&mut ChildStdout>) {
+ let driver = driver_binary();
+ if !driver.exists() { eprintln!("cua-driver.exe not built"); return; }
+ let harness = match Harness::launch() { Some(h) => h, None => return };
+ let mut child = Command::new(&driver)
+ .stdin(Stdio::piped()).stdout(Stdio::piped()).stderr(Stdio::null())
+ .spawn().expect("spawn cua-driver");
+ let mut stdin = child.stdin.take().unwrap();
+ let mut raw_stdout = child.stdout.take().unwrap();
+ let mut stdout = BufReader::new(&mut raw_stdout);
+ init(&mut stdin, &mut stdout);
+ let (wid, _) = find_harness_window(&mut stdin, &mut stdout, harness.pid, "CuaTestHarness WinUI3")
+ .expect("WinUI3 main window");
+ f(harness.pid, wid, &mut stdin, &mut stdout);
+ drop(stdout);
+ drop(stdin);
+ child.kill().ok();
+}
+
+/// Regression guard for the click → TogglePattern dispatch fix.
+/// cua-driver `click` now tries Invoke → Toggle → SelectionItem →
+/// ExpandCollapse before falling through to PostMessage, so WinUI3
+/// CheckBox toggles correctly via UIA without needing dispatch:foreground.
+#[test]
+#[ignore]
+fn harness_winui3_checkbox_toggle() {
+ winui3_with_session(|pid, wid, stdin, stdout| {
+ let snap = tools_call(stdin, stdout, 20, "get_window_state",
+ serde_json::json!({"pid": pid as i64, "window_id": wid, "capture_mode": "ax"}));
+ let idx = find_idx(snapshot_text(&snap), "chk-agreed").expect("chk-agreed");
+ let _ = tools_call(stdin, stdout, 30, "click", serde_json::json!({
+ "pid": pid as i64, "window_id": wid, "element_index": idx
+ }));
+ std::thread::sleep(Duration::from_millis(400));
+ let post = tools_call(stdin, stdout, 31, "get_window_state",
+ serde_json::json!({"pid": pid as i64, "window_id": wid, "capture_mode": "ax"}));
+ assert!(snapshot_text(&post).contains("agreed=True"),
+ "WinUI3 CheckBox didn't toggle: TogglePattern dispatch may have regressed.");
+ println!("✅ harness_winui3_checkbox_toggle: agreed=True via UIA Toggle");
+ });
+}
+
+/// Regression guard for SelectionItem.Select dispatch on RadioButton.
+#[test]
+#[ignore]
+fn harness_winui3_radio_select() {
+ winui3_with_session(|pid, wid, stdin, stdout| {
+ let snap = tools_call(stdin, stdout, 20, "get_window_state",
+ serde_json::json!({"pid": pid as i64, "window_id": wid, "capture_mode": "ax"}));
+ let idx = find_idx(snapshot_text(&snap), "rdo-high").expect("rdo-high");
+ let _ = tools_call(stdin, stdout, 30, "click", serde_json::json!({
+ "pid": pid as i64, "window_id": wid, "element_index": idx
+ }));
+ std::thread::sleep(Duration::from_millis(400));
+ let post = tools_call(stdin, stdout, 31, "get_window_state",
+ serde_json::json!({"pid": pid as i64, "window_id": wid, "capture_mode": "ax"}));
+ assert!(snapshot_text(&post).contains("prio=High"),
+ "WinUI3 radio didn't select High via SelectionItem.Select.");
+ println!("✅ harness_winui3_radio_select: prio=High via UIA SelectionItem");
+ });
+}
+
+/// Documents cua-driver gap: WinUI3 Slider implements
+/// `RangeValuePattern`, not `ValuePattern`. cua-driver's `set_value` tool
+/// queries `ValuePatternId` specifically (impl_.rs:2640), so it silently
+/// fails on RangeValuePattern-only elements. Real fix: cua-driver should
+/// try RangeValuePattern.SetValue (coercing the string to a double) when
+/// ValuePattern isn't found.
+/// WinUI3 Slider's parent AutomationId doesn't surface in the flat UIA
+/// element list (SliderAutomationPeer's children are the actionable
+/// elements, not the slider itself). Still document the gap — fixing
+/// the UIA enumeration to expose the slider as an indexed element is
+/// orthogonal to this branch's set_value fix.
+#[test]
+#[ignore]
+fn harness_winui3_slider_DOCUMENTED_unreachable() {
+ winui3_with_session(|pid, wid, stdin, stdout| {
+ let snap = tools_call(stdin, stdout, 20, "get_window_state",
+ serde_json::json!({"pid": pid as i64, "window_id": wid, "capture_mode": "ax"}));
+ let idx_opt = find_idx(snapshot_text(&snap), "sld-value");
+ assert!(idx_opt.is_none(),
+ "WinUI3 Slider 'sld-value' now appears in the UIA flat tree — \
+ cua-driver fixed the slider-element enumeration gap. Update this test.");
+ println!("⚠️ harness_winui3_slider_DOCUMENTED_unreachable: sld-value not in UIA flat tree");
+ });
+}
+
+/// Regression guard for ExpandCollapse.Expand + SelectionItem.Select on
+/// WinUI3 ComboBox.
+#[test]
+#[ignore]
+fn harness_winui3_combo_select() {
+ winui3_with_session(|pid, wid, stdin, stdout| {
+ let snap = tools_call(stdin, stdout, 20, "get_window_state",
+ serde_json::json!({"pid": pid as i64, "window_id": wid, "capture_mode": "ax"}));
+ let combo_idx = find_idx(snapshot_text(&snap), "cbo-color").expect("cbo-color");
+ // Expand the dropdown via ExpandCollapse.
+ let _ = tools_call(stdin, stdout, 30, "click", serde_json::json!({
+ "pid": pid as i64, "window_id": wid, "element_index": combo_idx
+ }));
+ std::thread::sleep(Duration::from_millis(400));
+ // Re-snapshot — items materialize after expand.
+ let snap2 = tools_call(stdin, stdout, 31, "get_window_state",
+ serde_json::json!({"pid": pid as i64, "window_id": wid, "capture_mode": "ax"}));
+ let item_idx = find_idx(snapshot_text(&snap2), "cbo-item-orange")
+ .expect("cbo-item-orange after expand");
+ // Select the item via SelectionItem.Select.
+ let _ = tools_call(stdin, stdout, 32, "click", serde_json::json!({
+ "pid": pid as i64, "window_id": wid, "element_index": item_idx
+ }));
+ std::thread::sleep(Duration::from_millis(400));
+ let post = tools_call(stdin, stdout, 33, "get_window_state",
+ serde_json::json!({"pid": pid as i64, "window_id": wid, "capture_mode": "ax"}));
+ assert!(snapshot_text(&post).contains("color=orange"),
+ "WinUI3 combo didn't switch to orange via ExpandCollapse + SelectionItem.Select.");
+ println!("✅ harness_winui3_combo_select: color=orange via UIA Expand + Select");
+ });
+}
diff --git a/libs/cua-driver/rust/crates/cua-driver/tests/harness_wpf_test.rs b/libs/cua-driver/rust/crates/cua-driver/tests/harness_wpf_test.rs
index df4aafb5a8..7bb1e40e92 100644
--- a/libs/cua-driver/rust/crates/cua-driver/tests/harness_wpf_test.rs
+++ b/libs/cua-driver/rust/crates/cua-driver/tests/harness_wpf_test.rs
@@ -671,3 +671,209 @@ fn harness_wpf_layered_popup_capture() {
rgb.width(), rgb.height());
});
}
+
+// ── slider / checkable / combo / list / menu coverage ────────────────────────
+
+#[test]
+#[ignore]
+fn harness_wpf_slider_drag_tool_returns() {
+ // Coverage note for the drag tool against a WPF Slider thumb.
+ //
+ // PostMessage WM_LBUTTONDOWN / WM_MOUSEMOVE / WM_LBUTTONUP doesn't
+ // update the OS keyboard/mouse state visible to GetKeyState. WPF's
+ // Slider Thumb relies on Mouse.LeftButton (which polls GetKeyState)
+ // to recognise an in-progress drag — so a PostMessage drag never
+ // moves a WPF thumb, even when from/to are correctly on the thumb
+ // in client coords. The companion `harness_wpf_slider_increase_large`
+ // test covers the slider via UIA Invoke on its internal IncreaseLarge
+ // sub-button, which is what an agent SHOULD use for slider
+ // manipulation on a backgrounded window.
+ //
+ // We still exercise the drag tool against the slider so its codepath
+ // (coord translation, dispatch policy, message synthesis) is on the
+ // critical-path test list — we just don't assert on the value moving.
+ // TODO: add a SendInput-based drag path so dispatch:"foreground" can
+ // drive the thumb, then enable a behavioral assertion here.
+ with_session(|pid, wid, stdin, stdout| {
+ focus_harness(stdin, stdout, pid, wid);
+ let resp = tools_call(stdin, stdout, 30, "drag", serde_json::json!({
+ "pid": pid as i64, "window_id": wid,
+ "from_x": 50.0, "from_y": 275.0,
+ "to_x": 330.0, "to_y": 275.0,
+ "duration_ms": 600, "steps": 30
+ }));
+ let msg = resp["result"]["content"][0]["text"].as_str().unwrap_or("");
+ println!("drag slider: {msg}");
+ assert!(msg.starts_with("✅"),
+ "drag tool returned non-success: {msg}");
+ println!("✅ harness_wpf_slider_drag_tool_returns: PostMessage drag emitted (known no-op vs WPF Thumb)");
+ });
+}
+
+#[test]
+#[ignore]
+fn harness_wpf_slider_increase_large() {
+ // Companion to slider_drag — exercises UIA Invoke on the Slider's
+ // internal IncreaseLarge "page-up" button. Doesn't depend on screen
+ // coords, so it's the more robust slider integration test.
+ with_session(|pid, wid, stdin, stdout| {
+ focus_harness(stdin, stdout, pid, wid);
+ let snap = snapshot_elements(stdin, stdout, pid, wid);
+ let idx = find_element_index_by_aid(&snap, "IncreaseLarge")
+ .expect("slider IncreaseLarge button not in snapshot");
+ for i in 0..3 {
+ let resp = tools_call(stdin, stdout, 30 + i, "click", serde_json::json!({
+ "pid": pid as i64, "window_id": wid, "element_index": idx
+ }));
+ println!("invoke IncreaseLarge #{i}: {}", resp["result"]["content"][0]["text"]);
+ std::thread::sleep(Duration::from_millis(150));
+ }
+ std::thread::sleep(Duration::from_millis(300));
+ let post = snapshot_elements(stdin, stdout, pid, wid);
+ let text = snapshot_text(&post);
+ let advanced = text.lines().any(|l|
+ l.contains("slider_value=") && !l.contains("slider_value=0\""));
+ assert!(advanced, "slider IncreaseLarge invokes did not advance value. Lines: {}",
+ text.lines().filter(|l| l.contains("slider_value")).collect::<Vec<_>>().join(" / "));
+ println!("✅ harness_wpf_slider_increase_large: advanced via UIA Invoke");
+ });
+}
+
+#[test]
+#[ignore]
+fn harness_wpf_checkbox_toggle() {
+ with_session(|pid, wid, stdin, stdout| {
+ focus_harness(stdin, stdout, pid, wid);
+ let snap = snapshot_elements(stdin, stdout, pid, wid);
+ let idx = find_element_index_by_aid(&snap, "chk-agreed")
+ .expect("chk-agreed missing");
+ // CheckBox exposes UIA TogglePattern (actions=[toggle]), not Invoke.
+ // cua-driver's click tool tries UIA Invoke first; for elements that
+ // don't support it the PostMessage fallback path runs. Use
+ // dispatch:"foreground" to land a SendInput click that WPF
+ // recognises as a real user click and processes through Toggle.
+ let resp = tools_call(stdin, stdout, 30, "click", serde_json::json!({
+ "pid": pid as i64, "window_id": wid, "element_index": idx,
+ "dispatch": "foreground"
+ }));
+ println!("click chk-agreed: {}", resp["result"]["content"][0]["text"]);
+ std::thread::sleep(Duration::from_millis(400));
+ let post = snapshot_elements(stdin, stdout, pid, wid);
+ assert!(snapshot_text(&post).contains("agreed=True"),
+ "checkbox didn't toggle: {}",
+ snapshot_text(&post).lines().filter(|l| l.contains("agreed=")).collect::<Vec<_>>().join(" / "));
+ println!("✅ harness_wpf_checkbox_toggle: agreed=True");
+ });
+}
+
+#[test]
+#[ignore]
+fn harness_wpf_radio_select() {
+ with_session(|pid, wid, stdin, stdout| {
+ focus_harness(stdin, stdout, pid, wid);
+ let snap = snapshot_elements(stdin, stdout, pid, wid);
+ let idx = find_element_index_by_aid(&snap, "rdo-high")
+ .expect("rdo-high missing");
+ // RadioButton exposes SelectionItem pattern (actions=[select]).
+ // Same dispatch:foreground rationale as the checkbox test.
+ let _ = tools_call(stdin, stdout, 30, "click", serde_json::json!({
+ "pid": pid as i64, "window_id": wid, "element_index": idx,
+ "dispatch": "foreground"
+ }));
+ std::thread::sleep(Duration::from_millis(400));
+ let post = snapshot_elements(stdin, stdout, pid, wid);
+ assert!(snapshot_text(&post).contains("prio=High"),
+ "radio didn't switch to High");
+ println!("✅ harness_wpf_radio_select: prio=High");
+ });
+}
+
+#[test]
+#[ignore]
+fn harness_wpf_combo_select() {
+ with_session(|pid, wid, stdin, stdout| {
+ focus_harness(stdin, stdout, pid, wid);
+ let snap = snapshot_elements(stdin, stdout, pid, wid);
+ let combo_idx = find_element_index_by_aid(&snap, "cbo-color")
+ .expect("cbo-color missing");
+ // WPF ComboBox UIA peer surfaces ExpandCollapsePattern (actions=[expand])
+ // but not ValuePattern — set_value at the parent is a no-op. Standard
+ // recipe: invoke the combo to expand the dropdown, re-snapshot so the
+ // item AIDs land in the element cache, then click the target item.
+ let _ = tools_call(stdin, stdout, 30, "click", serde_json::json!({
+ "pid": pid as i64, "window_id": wid, "element_index": combo_idx,
+ "dispatch": "foreground"
+ }));
+ std::thread::sleep(Duration::from_millis(500));
+
+ let snap2 = snapshot_elements(stdin, stdout, pid, wid);
+ let item_idx = find_element_index_by_aid(&snap2, "cbo-item-orange")
+ .expect("cbo-item-orange missing after expand");
+ let _ = tools_call(stdin, stdout, 31, "click", serde_json::json!({
+ "pid": pid as i64, "window_id": wid, "element_index": item_idx,
+ "dispatch": "foreground"
+ }));
+ std::thread::sleep(Duration::from_millis(500));
+
+ let post = snapshot_elements(stdin, stdout, pid, wid);
+ assert!(snapshot_text(&post).contains("color=orange"),
+ "combo didn't switch to orange: {}",
+ snapshot_text(&post).lines().filter(|l| l.contains("color=")).collect::<Vec<_>>().join(" / "));
+ println!("✅ harness_wpf_combo_select: color=orange");
+ });
+}
+
+#[test]
+#[ignore]
+fn harness_wpf_listbox_select() {
+ with_session(|pid, wid, stdin, stdout| {
+ focus_harness(stdin, stdout, pid, wid);
+ let snap = snapshot_elements(stdin, stdout, pid, wid);
+ let idx = find_element_index_by_aid(&snap, "lst-cherry")
+ .expect("lst-cherry missing");
+ let _ = tools_call(stdin, stdout, 30, "click", serde_json::json!({
+ "pid": pid as i64, "window_id": wid, "element_index": idx,
+ "dispatch": "foreground"
+ }));
+ std::thread::sleep(Duration::from_millis(400));
+ let post = snapshot_elements(stdin, stdout, pid, wid);
+ assert!(snapshot_text(&post).contains("selected=cherry"),
+ "list didn't select cherry: {}",
+ snapshot_text(&post).lines().filter(|l| l.contains("selected=")).collect::<Vec<_>>().join(" / "));
+ println!("✅ harness_wpf_listbox_select: selected=cherry");
+ });
+}
+
+#[test]
+#[ignore]
+fn harness_wpf_menu_invoke() {
+ with_session(|pid, wid, stdin, stdout| {
+ focus_harness(stdin, stdout, pid, wid);
+ // Expand File menu first (UIA expand pattern on MenuItem)
+ let snap = snapshot_elements(stdin, stdout, pid, wid);
+ let file_idx = find_element_index_by_aid(&snap, "menu-file")
+ .expect("menu-file missing");
+ let _ = tools_call(stdin, stdout, 30, "click", serde_json::json!({
+ "pid": pid as i64, "window_id": wid, "element_index": file_idx,
+ "dispatch": "foreground"
+ }));
+ std::thread::sleep(Duration::from_millis(400));
+
+ // Re-snapshot so menu-file-new is in the cache (it materialized
+ // when the menu expanded).
+ let snap2 = snapshot_elements(stdin, stdout, pid, wid);
+ let new_idx = find_element_index_by_aid(&snap2, "menu-file-new")
+ .expect("menu-file-new missing after expand");
+ let _ = tools_call(stdin, stdout, 31, "click", serde_json::json!({
+ "pid": pid as i64, "window_id": wid, "element_index": new_idx,
+ "dispatch": "foreground"
+ }));
+ std::thread::sleep(Duration::from_millis(500));
+
+ let post = snapshot_elements(stdin, stdout, pid, wid);
+ assert!(snapshot_text(&post).contains("menu_action=file_new"),
+ "File>New didn't invoke: {}",
+ snapshot_text(&post).lines().filter(|l| l.contains("menu_action=")).collect::<Vec<_>>().join(" / "));
+ println!("✅ harness_wpf_menu_invoke: menu_action=file_new");
+ });
+}
diff --git a/libs/cua-driver/rust/crates/focus-monitor-win/src/main.rs b/libs/cua-driver/rust/crates/focus-monitor-win/src/main.rs
index 1e55bdbd12..389ab809bf 100644
--- a/libs/cua-driver/rust/crates/focus-monitor-win/src/main.rs
+++ b/libs/cua-driver/rust/crates/focus-monitor-win/src/main.rs
@@ -32,10 +32,14 @@ mod win {
// ── global loss counters ─────────────────────────────────────────────────
static ACTIVATE_LOSSES: AtomicU32 = AtomicU32::new(0);
+ static ACTIVATE_GAINS: AtomicU32 = AtomicU32::new(0);
static KEY_LOSSES: AtomicU32 = AtomicU32::new(0);
+ static KEY_GAINS: AtomicU32 = AtomicU32::new(0);
fn loss_file() -> std::path::PathBuf { loss_path("focus_monitor_losses.txt") }
+ fn gain_file() -> std::path::PathBuf { loss_path("focus_monitor_gains.txt") }
fn key_loss_file() -> std::path::PathBuf { loss_path("focus_monitor_key_losses.txt") }
+ fn key_gain_file() -> std::path::PathBuf { loss_path("focus_monitor_key_gains.txt") }
fn loss_path(name: &str) -> std::path::PathBuf {
let mut p = std::env::temp_dir();
@@ -56,26 +60,35 @@ mod win {
) -> LRESULT {
match msg {
WM_ACTIVATE => {
- // WA_INACTIVE == 0 in the low word of wParam
+ // WA_INACTIVE == 0 in the low word of wParam; WA_ACTIVE == 1, WA_CLICKACTIVE == 2
if (wparam.0 & 0xFFFF) == 0 {
let n = ACTIVATE_LOSSES.fetch_add(1, Ordering::SeqCst) + 1;
write_count(&loss_file(), n);
- // Repaint to show updated count
- let _ = InvalidateRect(hwnd, None, true);
+ } else {
+ let n = ACTIVATE_GAINS.fetch_add(1, Ordering::SeqCst) + 1;
+ write_count(&gain_file(), n);
}
+ let _ = InvalidateRect(hwnd, None, true);
}
WM_KILLFOCUS => {
let n = KEY_LOSSES.fetch_add(1, Ordering::SeqCst) + 1;
write_count(&key_loss_file(), n);
let _ = InvalidateRect(hwnd, None, true);
}
+ WM_SETFOCUS => {
+ let n = KEY_GAINS.fetch_add(1, Ordering::SeqCst) + 1;
+ write_count(&key_gain_file(), n);
+ let _ = InvalidateRect(hwnd, None, true);
+ }
WM_PAINT => {
let mut ps = PAINTSTRUCT::default();
let hdc = BeginPaint(hwnd, &mut ps);
- let act = ACTIVATE_LOSSES.load(Ordering::SeqCst);
- let key = KEY_LOSSES.load(Ordering::SeqCst);
+ let act_l = ACTIVATE_LOSSES.load(Ordering::SeqCst);
+ let act_g = ACTIVATE_GAINS.load(Ordering::SeqCst);
+ let key_l = KEY_LOSSES.load(Ordering::SeqCst);
+ let key_g = KEY_GAINS.load(Ordering::SeqCst);
let text = wide(&format!(
- "act_losses: {act} key_losses: {key} (should stay 0)"
+ "act: {act_l}L / {act_g}G key: {key_l}L / {key_g}G (should stay net 0)"
));
TextOutW(hdc, 10, 10, &text);
EndPaint(hwnd, &ps);
@@ -116,9 +129,11 @@ mod win {
ShowWindow(hwnd, SW_SHOWNORMAL);
UpdateWindow(hwnd).ok();
- // Write initial zeros so tests can read even before any loss.
+ // Write initial zeros so tests can read even before any event.
write_count(&loss_file(), 0);
+ write_count(&gain_file(), 0);
write_count(&key_loss_file(), 0);
+ write_count(&key_gain_file(), 0);
// Signal the test harness via temp files (avoids pipe-blocking issues
// when stdout is captured by the test runner in sandbox environments).
diff --git a/libs/cua-driver/rust/crates/mcp-server/src/cdp.rs b/libs/cua-driver/rust/crates/mcp-server/src/cdp.rs
index 579861e544..e02b33713b 100644
--- a/libs/cua-driver/rust/crates/mcp-server/src/cdp.rs
+++ b/libs/cua-driver/rust/crates/mcp-server/src/cdp.rs
@@ -103,28 +103,72 @@ fn format_cdp_result(response: &Value) -> String {
// ── HTTP page discovery ───────────────────────────────────────────────────
async fn cdp_list_pages(port: u16) -> anyhow::Result<Vec<Value>> {
- use tokio::io::{AsyncReadExt, AsyncWriteExt};
+ use tokio::io::{AsyncBufReadExt, AsyncReadExt, AsyncWriteExt, BufReader};
let mut stream = tokio::net::TcpStream::connect(("127.0.0.1", port))
.await
.map_err(|e| anyhow::anyhow!("Cannot connect to CDP on port {port}: {e}"))?;
+ // `Connection: close` would be ideal but Chromium's CDP HTTP server
+ // ignores it and keeps the socket alive — a `read_to_end` then hangs
+ // forever. Parse Content-Length / Transfer-Encoding: chunked from the
+ // response headers and read exactly that many body bytes, leaving the
+ // socket open (it's torn down when we drop the stream).
let req = format!(
"GET /json HTTP/1.1\r\nHost: 127.0.0.1:{port}\r\nConnection: close\r\n\r\n"
);
stream.write_all(req.as_bytes()).await?;
- let mut buf = Vec::new();
- stream.read_to_end(&mut buf).await?;
+ let mut reader = BufReader::new(stream);
+ let mut header_line = String::new();
+ let mut content_length: Option<usize> = None;
+ let mut chunked = false;
+ loop {
+ header_line.clear();
+ let n = reader.read_line(&mut header_line).await?;
+ if n == 0 { anyhow::bail!("CDP /json: EOF in headers"); }
+ if header_line == "\r\n" { break; }
+ let lower = header_line.to_ascii_lowercase();
+ if let Some(v) = lower.strip_prefix("content-length:") {
+ content_length = v.trim().parse::<usize>().ok();
+ }
+ if lower.starts_with("transfer-encoding:") && lower.contains("chunked") {
+ chunked = true;
+ }
+ }
+
+ let body_bytes: Vec<u8> = if let Some(n) = content_length {
+ let mut buf = vec![0u8; n];
+ reader.read_exact(&mut buf).await?;
+ buf
+ } else if chunked {
+ let mut out = Vec::new();
+ loop {
+ let mut size_line = String::new();
+ reader.read_line(&mut size_line).await?;
+ let size = usize::from_str_radix(size_line.trim_end(), 16)
+ .map_err(|_| anyhow::anyhow!("CDP /json: bad chunk size {size_line:?}"))?;
+ if size == 0 { break; }
+ let mut chunk = vec![0u8; size];
+ reader.read_exact(&mut chunk).await?;
+ out.extend_from_slice(&chunk);
+ let mut crlf = [0u8; 2];
+ reader.read_exact(&mut crlf).await?;
+ }
+ out
+ } else {
+ // No Content-Length and not chunked — fall back to read-until-EOF.
+ // Server will close eventually if it actually honoured our
+ // Connection: close header.
+ let mut buf = Vec::new();
+ reader.read_to_end(&mut buf).await?;
+ buf
+ };
- let text = std::str::from_utf8(&buf)
+ let body = std::str::from_utf8(&body_bytes)
.map_err(|_| anyhow::anyhow!("CDP /json response is not valid UTF-8"))?;
- let body_start = text
- .find("\r\n\r\n")
- .ok_or_else(|| anyhow::anyhow!("Invalid HTTP response from CDP"))?
- + 4;
- let all: Value = serde_json::from_str(&text[body_start..])
+ let all: Value = serde_json::from_str(body)
.map_err(|e| anyhow::anyhow!("CDP /json parse error: {e}"))?;
let pages = all
diff --git a/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs b/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs
index 2b33291463..dd26e7caba 100644
--- a/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs
+++ b/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs
@@ -1788,40 +1788,85 @@ impl Tool for ClickTool {
if use_uia_invoke {
if let Some(ptr) = state_clone.element_cache.get_element_ptr(pid, hwnd, idx) {
use windows::Win32::UI::Accessibility::{
- IUIAutomationElement, IUIAutomationInvokePattern, UIA_InvokePatternId,
+ IUIAutomationElement, IUIAutomationInvokePattern,
+ IUIAutomationTogglePattern, IUIAutomationSelectionItemPattern,
+ IUIAutomationExpandCollapsePattern,
+ UIA_InvokePatternId, UIA_TogglePatternId,
+ UIA_SelectionItemPatternId, UIA_ExpandCollapsePatternId,
};
use windows::core::Interface;
- // Reconstruct without consuming the cache's AddRef;
- // forget after we extract the pattern so Drop doesn't
- // double-Release the cached pointer.
let elem: IUIAutomationElement =
unsafe { IUIAutomationElement::from_raw(ptr as *mut _) };
+
+ // Try patterns in order of click-semantics specificity:
+ // 1. Invoke - buttons / hyperlinks (canonical)
+ // 2. Toggle - checkboxes (no Invoke)
+ // 3. SelectionItem - radio buttons, listbox items
+ // 4. ExpandCollapse - combo boxes, tree nodes
+ // Each call is wrapped in the UWP foreground-steal bypass.
+ //
+ // Why this order: PR #1699's WinUI3 control parity tests
+ // surfaced these pattern-dispatch gaps. Without these
+ // fallthroughs, cua-driver couldn't drive WinUI3
+ // CheckBox / RadioButton / ComboBox because PostMessage
+ // doesn't reach their handlers (CoreInput dispatcher).
let invoke_result = unsafe { elem.GetCurrentPattern(UIA_InvokePatternId) };
- std::mem::forget(elem);
if let Ok(pattern) = invoke_result {
if let Ok(inv) = pattern.cast::<IUIAutomationInvokePattern>() {
- // UWP foreground-steal bypass: wrap Invoke so
- // XAML hosts don't self-foreground and steal
- // user focus. See crate::uia::fg_bypass.
- let invoke_outcome = crate::uia::fg_bypass::run_with_uwp_bypass(
- hwnd as isize,
- || unsafe { inv.Invoke() },
- );
- match invoke_outcome {
+ let outcome = crate::uia::fg_bypass::run_with_uwp_bypass(
+ hwnd as isize, || unsafe { inv.Invoke() });
+ match outcome {
Ok(()) => {
+ std::mem::forget(elem);
return Ok(format!(
"✅ Performed UIA Invoke on [{idx}] (screen ({cx},{cy}))."
));
}
- Err(e) => {
- tracing::debug!(
- target: "click",
- "UIA Invoke failed for [{idx}]: {e}; falling back to PostMessage"
- );
- }
+ Err(e) => tracing::debug!(target: "click",
+ "UIA Invoke on [{idx}]: {e}, trying Toggle"),
}
}
}
+ let toggle_result = unsafe { elem.GetCurrentPattern(UIA_TogglePatternId) };
+ if let Ok(pattern) = toggle_result {
+ if let Ok(tg) = pattern.cast::<IUIAutomationTogglePattern>() {
+ let outcome = crate::uia::fg_bypass::run_with_uwp_bypass(
+ hwnd as isize, || unsafe { tg.Toggle() });
+ if outcome.is_ok() {
+ std::mem::forget(elem);
+ return Ok(format!(
+ "✅ Performed UIA Toggle on [{idx}] (screen ({cx},{cy}))."
+ ));
+ }
+ }
+ }
+ let sel_result = unsafe { elem.GetCurrentPattern(UIA_SelectionItemPatternId) };
+ if let Ok(pattern) = sel_result {
+ if let Ok(si) = pattern.cast::<IUIAutomationSelectionItemPattern>() {
+ let outcome = crate::uia::fg_bypass::run_with_uwp_bypass(
+ hwnd as isize, || unsafe { si.Select() });
+ if outcome.is_ok() {
+ std::mem::forget(elem);
+ return Ok(format!(
+ "✅ Performed UIA SelectionItem.Select on [{idx}] (screen ({cx},{cy}))."
+ ));
+ }
+ }
+ }
+ let exp_result = unsafe { elem.GetCurrentPattern(UIA_ExpandCollapsePatternId) };
+ if let Ok(pattern) = exp_result {
+ if let Ok(ec) = pattern.cast::<IUIAutomationExpandCollapsePattern>() {
+ let outcome = crate::uia::fg_bypass::run_with_uwp_bypass(
+ hwnd as isize, || unsafe { ec.Expand() });
+ if outcome.is_ok() {
+ std::mem::forget(elem);
+ return Ok(format!(
+ "✅ Performed UIA ExpandCollapse.Expand on [{idx}] (screen ({cx},{cy}))."
+ ));
+ }
+ }
+ }
+ std::mem::forget(elem);
}
}
// PostMessage fallback (legacy Win32 + non-Invokable elements).
@@ -2631,22 +2676,51 @@ impl Tool for SetValueTool {
};
let state = self.state.clone();
- let result = tokio::task::spawn_blocking(move || -> anyhow::Result<()> {
+ let result = tokio::task::spawn_blocking(move || -> anyhow::Result<String> {
let ptr = state.element_cache.get_element_ptr(pid, hwnd, idx)
.ok_or_else(|| anyhow::anyhow!("Element {idx} not in cache."))?;
use windows::Win32::UI::Accessibility::{IUIAutomationElement, IUIAutomationValuePattern, UIA_ValuePatternId};
use windows::core::{Interface, BSTR};
let elem: IUIAutomationElement = unsafe { IUIAutomationElement::from_raw(ptr as *mut _) };
- let pattern = unsafe { elem.GetCurrentPattern(UIA_ValuePatternId)? };
+ // Try ValuePattern first (text inputs, editable combos, etc).
+ if let Ok(pattern) = unsafe { elem.GetCurrentPattern(UIA_ValuePatternId) } {
+ if let Ok(vp) = pattern.cast::<IUIAutomationValuePattern>() {
+ if unsafe { vp.SetValue(&BSTR::from(value.as_str())) }.is_ok() {
+ std::mem::forget(elem);
+ return Ok("ValuePattern".to_string());
+ }
+ }
+ }
+ // Fall through to RangeValuePattern for Sliders / ProgressBars /
+ // numeric ranges. RangeValue.SetValue takes a double, so coerce
+ // the string. Documented gap-closer (PR #1699 harness exposed).
+ use windows::Win32::UI::Accessibility::{
+ IUIAutomationRangeValuePattern, UIA_RangeValuePatternId,
+ };
+ if let Ok(pattern) = unsafe { elem.GetCurrentPattern(UIA_RangeValuePatternId) } {
+ if let Ok(rv) = pattern.cast::<IUIAutomationRangeValuePattern>() {
+ let parsed: f64 = value.parse().map_err(|_| anyhow::anyhow!(
+ "set_value: target element exposes RangeValuePattern (Slider / \
+ ProgressBar / numeric range). `value` must be a parseable f64; \
+ got {value:?}."
+ ))?;
+ unsafe { rv.SetValue(parsed)? };
+ std::mem::forget(elem);
+ return Ok("RangeValuePattern".to_string());
+ }
+ }
std::mem::forget(elem);
- let vp: IUIAutomationValuePattern = pattern.cast()?;
- unsafe { vp.SetValue(&BSTR::from(value.as_str()))? };
- Ok(())
+ anyhow::bail!(
+ "set_value: element [{idx}] does not implement ValuePattern or \
+ RangeValuePattern. For controls with TogglePattern (CheckBox) or \
+ SelectionItemPattern (RadioButton / ComboBoxItem), use the `click` \
+ tool instead."
+ );
}).await;
match result {
- // Match Swift's text format 1:1 (default AXValue-write path).
- // UIA role/title placeholder pending element-cache enrichment.
- Ok(Ok(())) => ToolResult::text(format!("✅ Set AXValue on [{idx}] (UIA ValuePattern).")),
+ Ok(Ok(pattern_name)) => ToolResult::text(format!(
+ "✅ Set AXValue on [{idx}] (UIA {pattern_name})."
+ )),
Ok(Err(e)) => ToolResult::error(e.to_string()),
Err(e) => ToolResult::error(format!("Task error: {e}")),
}
diff --git a/libs/cua-driver/rust/crates/platform-windows/src/uia/fg_bypass.rs b/libs/cua-driver/rust/crates/platform-windows/src/uia/fg_bypass.rs
index a757d02dcf..2c8561d087 100644
--- a/libs/cua-driver/rust/crates/platform-windows/src/uia/fg_bypass.rs
+++ b/libs/cua-driver/rust/crates/platform-windows/src/uia/fg_bypass.rs
@@ -65,10 +65,18 @@ impl Drop for DisabledHwndGuard {
/// UIA element. When it identifies as an XAML host (`is_xaml_host_hwnd`), the
/// HWND is disabled for the duration of `action`. For non-XAML / classic
/// Win32 hosts the closure runs unmodified — empirically those don't
-/// self-foreground.
+/// self-foreground via the input-queue path that EnableWindow gates.
///
-/// The guard restores the previous enabled-state at Drop, so a panic in
-/// `action` still leaves the window in a usable state.
+/// **Known limitation, WPF Buttons / TextBoxes:** WPF's automation peers
+/// call `UIElement.Focus()` synchronously during the UIA Invoke /
+/// ValuePattern.SetValue handler. `Focus()` routes through
+/// `SetForegroundWindow`, which is NOT gated by EnableWindow. The bypass
+/// has no effect there, and the daemon WILL transiently steal foreground
+/// from the user. Restoring foreground from a non-UIAccess process is
+/// blocked by the foreground-lock, so the only mitigation is to spawn
+/// cua-driver-uia.exe (UIAccess-manifested worker) and route UIA
+/// activations through it. See PR #1699 bg-modality tests for the
+/// regression guards.
pub fn run_with_uwp_bypass<T>(host_hwnd: isize, action: impl FnOnce() -> T) -> T {
let _guard = make_guard(host_hwnd);
action()
diff --git a/libs/cua-driver/rust/sandbox/run-tests-in-sandbox.ps1 b/libs/cua-driver/rust/sandbox/run-tests-in-sandbox.ps1
index 5c088f3a36..1a7be139f8 100644
--- a/libs/cua-driver/rust/sandbox/run-tests-in-sandbox.ps1
+++ b/libs/cua-driver/rust/sandbox/run-tests-in-sandbox.ps1
@@ -87,6 +87,14 @@ try {
Write-Host "`n[BUILD] cargo test --no-run (harness_winui3_test)..." -ForegroundColor Yellow
cargo test --test harness_winui3_test --no-run
if ($LASTEXITCODE -ne 0) { throw "cargo test --no-run (harness_winui3_test) failed" }
+
+ Write-Host "`n[BUILD] cargo test --no-run (harness_web_test)..." -ForegroundColor Yellow
+ cargo test --test harness_web_test --no-run
+ if ($LASTEXITCODE -ne 0) { throw "cargo test --no-run (harness_web_test) failed" }
+
+ Write-Host "`n[BUILD] cargo test --no-run (harness_bg_modality_test)..." -ForegroundColor Yellow
+ cargo test --test harness_bg_modality_test --no-run
+ if ($LASTEXITCODE -ne 0) { throw "cargo test --no-run (harness_bg_modality_test) failed" }
} finally { Pop-Location }
# ── 1.5. Build the .NET test-harness if dotnet is on PATH ────────────────────
diff --git a/libs/cua-driver/rust/sandbox/sandbox-runner.ps1 b/libs/cua-driver/rust/sandbox/sandbox-runner.ps1
index 50571f6df4..08b15db307 100644
--- a/libs/cua-driver/rust/sandbox/sandbox-runner.ps1
+++ b/libs/cua-driver/rust/sandbox/sandbox-runner.ps1
@@ -44,16 +44,20 @@ Log "cua-driver : $driverExe"
# Run mcp_protocol_test first, then ux_guard_test (UX guard needs a real
# desktop session and spawns visible windows, so it runs second).
$testSuites = @(
- @{ Pattern = "mcp_protocol_test-*.exe"; Label = "mcp_protocol_test" },
- @{ Pattern = "ux_guard_test-*.exe"; Label = "ux_guard_test" },
- @{ Pattern = "harness_wpf_test-*.exe"; Label = "harness_wpf_test"; Extra = @("--ignored") },
- @{ Pattern = "harness_winui3_test-*.exe"; Label = "harness_winui3_test"; Extra = @("--ignored") }
+ @{ Pattern = "mcp_protocol_test-*.exe"; Label = "mcp_protocol_test" },
+ @{ Pattern = "ux_guard_test-*.exe"; Label = "ux_guard_test" },
+ @{ Pattern = "harness_wpf_test-*.exe"; Label = "harness_wpf_test"; Extra = @("--ignored") },
+ @{ Pattern = "harness_winui3_test-*.exe"; Label = "harness_winui3_test"; Extra = @("--ignored") },
+ @{ Pattern = "harness_web_test-*.exe"; Label = "harness_web_test"; Extra = @("--ignored") },
+ @{ Pattern = "harness_bg_modality_test-*.exe";Label = "harness_bg_modality_test";Extra = @("--ignored") }
)
# ── stage harness binaries to %TEMP% (same Zone-3 ShellExecute workaround) ──
$harnessRoots = @(
- @{ Src = "C:\cua-driver-rs\test-apps\harness-wpf"; EnvVar = "HARNESS_WPF_EXE"; Exe = "CuaTestHarness.Wpf.exe" },
- @{ Src = "C:\cua-driver-rs\test-apps\harness-winui3"; EnvVar = "HARNESS_WINUI3_EXE"; Exe = "CuaTestHarness.WinUI3.exe" }
+ @{ Src = "C:\cua-driver-rs\test-apps\harness-wpf"; EnvVar = "HARNESS_WPF_EXE"; Exe = "CuaTestHarness.Wpf.exe" },
+ @{ Src = "C:\cua-driver-rs\test-apps\harness-winui3"; EnvVar = "HARNESS_WINUI3_EXE"; Exe = "CuaTestHarness.WinUI3.exe" },
+ @{ Src = "C:\cua-driver-rs\test-apps\harness-webview"; EnvVar = "HARNESS_WEBVIEW_EXE"; Exe = "CuaTestHarness.WebView.exe" },
+ @{ Src = "C:\cua-driver-rs\test-apps\harness-electron"; EnvVar = "HARNESS_ELECTRON_EXE"; Exe = "CuaTestHarness.Electron.exe" }
)
foreach ($h in $harnessRoots) {
if (-not (Test-Path $h.Src)) {
diff --git a/libs/cua-driver/rust/test-apps/.gitignore b/libs/cua-driver/rust/test-apps/.gitignore
index c1bae11177..e3ba2f39e6 100644
--- a/libs/cua-driver/rust/test-apps/.gitignore
+++ b/libs/cua-driver/rust/test-apps/.gitignore
@@ -13,3 +13,5 @@
# (several thousand files, ~150 MB). Always rebuilt locally.
harness-wpf/
harness-winui3/
+harness-webview/
+harness-electron/
diff --git a/libs/cua-driver/test-harness/CuaTestHarness.Electron/.gitignore b/libs/cua-driver/test-harness/CuaTestHarness.Electron/.gitignore
new file mode 100644
index 0000000000..d8b0fe4524
--- /dev/null
+++ b/libs/cua-driver/test-harness/CuaTestHarness.Electron/.gitignore
@@ -0,0 +1,4 @@
+node_modules/
+dist/
+web/
+package-lock.json
diff --git a/libs/cua-driver/test-harness/CuaTestHarness.Electron/build.ps1 b/libs/cua-driver/test-harness/CuaTestHarness.Electron/build.ps1
new file mode 100644
index 0000000000..4b7e406169
--- /dev/null
+++ b/libs/cua-driver/test-harness/CuaTestHarness.Electron/build.ps1
@@ -0,0 +1,55 @@
+# build.ps1 - stage the Electron test harness for cua-driver tests.
+#
+# electron-builder portable mode needs admin (symlink privilege) for the
+# winCodeSign cache extraction. We instead stage a flat folder containing
+# the electron runtime + our app resources, with electron.exe renamed to
+# CuaTestHarness.Electron.exe so tests get a deterministic exe name.
+#
+# Output: ../rust/test-apps/harness-electron/
+
+Set-StrictMode -Version Latest
+$ErrorActionPreference = "Stop"
+
+$elecDir = Split-Path -Parent $MyInvocation.MyCommand.Definition
+$harnessDir = Split-Path -Parent $elecDir
+$cuaDriverDir = Split-Path -Parent $harnessDir
+$testAppsDir = Join-Path $cuaDriverDir "rust\test-apps"
+$outDir = Join-Path $testAppsDir "harness-electron"
+
+if (-not (Get-Command npm -ErrorAction SilentlyContinue)) {
+ Write-Host "[ERROR] npm not on PATH. Install Node.js first." -ForegroundColor Red
+ exit 1
+}
+
+Push-Location $elecDir
+try {
+ $webDir = Join-Path $elecDir "web"
+ if (-not (Test-Path $webDir)) { New-Item -ItemType Directory $webDir | Out-Null }
+ Copy-Item (Join-Path $harnessDir "shared-web\*") $webDir -Recurse -Force
+
+ if (-not (Test-Path "node_modules\electron\dist\electron.exe")) {
+ Write-Host "[INSTALL] npm install (first run)..." -ForegroundColor Yellow
+ npm install --silent
+ if ($LASTEXITCODE -ne 0) { throw "npm install failed" }
+ }
+
+ $electronSrc = Join-Path $elecDir "node_modules\electron\dist"
+ if (-not (Test-Path (Join-Path $electronSrc "electron.exe"))) {
+ throw "electron.exe not found under $electronSrc - npm install incomplete"
+ }
+
+ Write-Host "[STAGE] Copying electron runtime + app to $outDir..." -ForegroundColor Cyan
+ if (Test-Path $outDir) { Remove-Item $outDir -Recurse -Force }
+ Copy-Item $electronSrc $outDir -Recurse -Force
+
+ $appDir = Join-Path $outDir "resources\app"
+ if (-not (Test-Path $appDir)) { New-Item -ItemType Directory $appDir -Force | Out-Null }
+ Copy-Item (Join-Path $elecDir "main.js") $appDir -Force
+ Copy-Item (Join-Path $elecDir "package.json") $appDir -Force
+ Copy-Item $webDir (Join-Path $appDir "web") -Recurse -Force
+
+ Rename-Item (Join-Path $outDir "electron.exe") "CuaTestHarness.Electron.exe" -Force
+ Write-Host "[OK] Staged: $outDir\CuaTestHarness.Electron.exe" -ForegroundColor Green
+} finally {
+ Pop-Location
+}
diff --git a/libs/cua-driver/test-harness/CuaTestHarness.Electron/main.js b/libs/cua-driver/test-harness/CuaTestHarness.Electron/main.js
new file mode 100644
index 0000000000..89ad75543c
--- /dev/null
+++ b/libs/cua-driver/test-harness/CuaTestHarness.Electron/main.js
@@ -0,0 +1,73 @@
+// CuaTestHarness.Electron — minimal Electron host loading the shared
+// index.html that CuaTestHarness.WebView also loads. cua-driver's `page`
+// tool routes through CDP when --remote-debugging-port is set, so we
+// expose one here on a configurable port.
+
+const { app, BrowserWindow } = require('electron');
+const path = require('path');
+
+// Validate CUA_ELECTRON_CDP_PORT before forwarding to Chromium —
+// remote-debugging-port=0 means "pick an ephemeral port" which would
+// break our fixed-port expectation in the harness tests, and a
+// non-numeric value silently disables CDP.
+const rawCdpPort = process.env.CUA_ELECTRON_CDP_PORT ?? '9223';
+const cdpPortNum = Number(rawCdpPort);
+if (!Number.isInteger(cdpPortNum) || cdpPortNum < 1 || cdpPortNum > 65535) {
+ throw new Error(
+ `Invalid CUA_ELECTRON_CDP_PORT: "${rawCdpPort}". Expected an integer in 1-65535.`
+ );
+}
+const CDP_PORT = String(cdpPortNum);
+app.commandLine.appendSwitch('remote-debugging-port', CDP_PORT);
+
+let mainWindow;
+
+function createWindow() {
+ const fixedTitle = `CuaTestHarness Electron [cdp=${CDP_PORT}]`;
+ mainWindow = new BrowserWindow({
+ width: 940,
+ height: 780,
+ title: fixedTitle,
+ autoHideMenuBar: true,
+ webPreferences: {
+ nodeIntegration: false,
+ contextIsolation: true,
+ },
+ });
+
+ // Override the page's <title> with our deterministic harness title so
+ // cua-driver tests can find the window by substring match. Without this,
+ // Electron syncs window.title to document.title which would be
+ // 'cua-driver Web Harness' (the page's title).
+ mainWindow.on('page-title-updated', e => e.preventDefault());
+ mainWindow.setTitle(fixedTitle);
+
+ mainWindow
+ .loadFile(path.join(__dirname, 'web', 'index.html'))
+ .then(() => {
+ // Re-set after page-load — Electron syncs window.title to
+ // document.title once the load finishes, which would override
+ // our fixedTitle and break the harness-window-discovery test.
+ if (mainWindow && !mainWindow.isDestroyed()) {
+ mainWindow.setTitle(fixedTitle);
+ }
+ })
+ .catch(err => {
+ // Fail deterministically rather than leaving the harness window
+ // up with no content — the integration tests would then time out
+ // waiting for the DOM markers to render.
+ console.error('Failed to load harness web/index.html:', err);
+ app.exit(1);
+ });
+}
+
+app.whenReady().then(() => {
+ createWindow();
+ app.on('activate', () => {
+ if (BrowserWindow.getAllWindows().length === 0) createWindow();
+ });
+});
+
+app.on('window-all-closed', () => {
+ if (process.platform !== 'darwin') app.quit();
+});
diff --git a/libs/cua-driver/test-harness/CuaTestHarness.Electron/package.json b/libs/cua-driver/test-harness/CuaTestHarness.Electron/package.json
new file mode 100644
index 0000000000..afe7cd99e1
--- /dev/null
+++ b/libs/cua-driver/test-harness/CuaTestHarness.Electron/package.json
@@ -0,0 +1,37 @@
+{
+ "name": "cua-test-harness-electron",
+ "version": "1.0.0",
+ "description": "cua-driver Electron test harness — loads the same shared-web/index.html as CuaTestHarness.WebView",
+ "private": true,
+ "main": "main.js",
+ "scripts": {
+ "start": "electron .",
+ "build": "electron-builder --win portable --x64"
+ },
+ "devDependencies": {
+ "electron": "31.6.0",
+ "electron-builder": "25.0.5"
+ },
+ "build": {
+ "appId": "com.trycua.harness.electron",
+ "productName": "CuaTestHarness.Electron",
+ "directories": {
+ "output": "dist"
+ },
+ "files": [
+ "main.js",
+ "preload.js",
+ "web/**/*",
+ "package.json"
+ ],
+ "win": {
+ "target": [{
+ "target": "portable",
+ "arch": ["x64"]
+ }]
+ },
+ "portable": {
+ "artifactName": "CuaTestHarness.Electron.exe"
+ }
+ }
+}
diff --git a/libs/cua-driver/test-harness/CuaTestHarness.WebView/.gitignore b/libs/cua-driver/test-harness/CuaTestHarness.WebView/.gitignore
new file mode 100644
index 0000000000..10f960612a
--- /dev/null
+++ b/libs/cua-driver/test-harness/CuaTestHarness.WebView/.gitignore
@@ -0,0 +1,4 @@
+bin/
+obj/
+*.user
+.vs/
diff --git a/libs/cua-driver/test-harness/CuaTestHarness.WebView/App.xaml b/libs/cua-driver/test-harness/CuaTestHarness.WebView/App.xaml
new file mode 100644
index 0000000000..ff3bb5603a
--- /dev/null
+++ b/libs/cua-driver/test-harness/CuaTestHarness.WebView/App.xaml
@@ -0,0 +1,6 @@
+<Application x:Class="CuaTestHarness.WebView.App"
+ xmlns="http://schemas.microsoft.com/winfx/2006/xaml/presentation"
+ xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml"
+ StartupUri="MainWindow.xaml">
+ <Application.Resources/>
+</Application>
diff --git a/libs/cua-driver/test-harness/CuaTestHarness.WebView/App.xaml.cs b/libs/cua-driver/test-harness/CuaTestHarness.WebView/App.xaml.cs
new file mode 100644
index 0000000000..0cc2c96e34
--- /dev/null
+++ b/libs/cua-driver/test-harness/CuaTestHarness.WebView/App.xaml.cs
@@ -0,0 +1,7 @@
+using System.Windows;
+
+namespace CuaTestHarness.WebView;
+
+public partial class App : Application
+{
+}
diff --git a/libs/cua-driver/test-harness/CuaTestHarness.WebView/CuaTestHarness.WebView.csproj b/libs/cua-driver/test-harness/CuaTestHarness.WebView/CuaTestHarness.WebView.csproj
new file mode 100644
index 0000000000..09d73533f5
--- /dev/null
+++ b/libs/cua-driver/test-harness/CuaTestHarness.WebView/CuaTestHarness.WebView.csproj
@@ -0,0 +1,28 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+ <PropertyGroup>
+ <OutputType>WinExe</OutputType>
+ <TargetFramework>net8.0-windows</TargetFramework>
+ <UseWPF>true</UseWPF>
+ <Nullable>enable</Nullable>
+ <LangVersion>latest</LangVersion>
+ <ApplicationManifest>app.manifest</ApplicationManifest>
+ <AssemblyName>CuaTestHarness.WebView</AssemblyName>
+ <RootNamespace>CuaTestHarness.WebView</RootNamespace>
+ <PublishSingleFile>false</PublishSingleFile>
+ <SelfContained>true</SelfContained>
+ <RuntimeIdentifier>win-x64</RuntimeIdentifier>
+ <Platforms>x64</Platforms>
+ </PropertyGroup>
+
+ <ItemGroup>
+ <PackageReference Include="Microsoft.Web.WebView2" Version="1.0.2792.45" />
+ </ItemGroup>
+
+ <ItemGroup>
+ <None Include="..\shared-web\index.html" Link="web\index.html">
+ <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+ </None>
+ </ItemGroup>
+
+</Project>
diff --git a/libs/cua-driver/test-harness/CuaTestHarness.WebView/MainWindow.xaml b/libs/cua-driver/test-harness/CuaTestHarness.WebView/MainWindow.xaml
new file mode 100644
index 0000000000..27025fbfb3
--- /dev/null
+++ b/libs/cua-driver/test-harness/CuaTestHarness.WebView/MainWindow.xaml
@@ -0,0 +1,27 @@
+<Window x:Class="CuaTestHarness.WebView.MainWindow"
+ xmlns="http://schemas.microsoft.com/winfx/2006/xaml/presentation"
+ xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml"
+ xmlns:wv2="clr-namespace:Microsoft.Web.WebView2.Wpf;assembly=Microsoft.Web.WebView2.Wpf"
+ Title="CuaTestHarness WebView"
+ AutomationProperties.AutomationId="wnd-main"
+ Width="940" Height="780"
+ WindowStartupLocation="CenterScreen">
+ <DockPanel LastChildFill="True">
+ <Border DockPanel.Dock="Top" Background="#F0F0F0" Padding="10,6">
+ <StackPanel Orientation="Horizontal">
+ <TextBlock Text="WebView2 host — same DOM as the Electron harness ("
+ VerticalAlignment="Center"/>
+ <TextBlock x:Name="LblPageUrl"
+ AutomationProperties.AutomationId="lbl-page-url"
+ FontFamily="Consolas" Text="(loading)" VerticalAlignment="Center"/>
+ <TextBlock Text=")" VerticalAlignment="Center"/>
+ <Button x:Name="BtnExit"
+ AutomationProperties.AutomationId="btn-exit"
+ Content="Exit" Width="80" Margin="20,0,0,0"
+ Click="OnExitClick"/>
+ </StackPanel>
+ </Border>
+ <wv2:WebView2 x:Name="Wv"
+ AutomationProperties.AutomationId="wv-host"/>
+ </DockPanel>
+</Window>
diff --git a/libs/cua-driver/test-harness/CuaTestHarness.WebView/MainWindow.xaml.cs b/libs/cua-driver/test-harness/CuaTestHarness.WebView/MainWindow.xaml.cs
new file mode 100644
index 0000000000..22a8049495
--- /dev/null
+++ b/libs/cua-driver/test-harness/CuaTestHarness.WebView/MainWindow.xaml.cs
@@ -0,0 +1,69 @@
+using System;
+using System.IO;
+using System.Windows;
+using Microsoft.Web.WebView2.Core;
+
+namespace CuaTestHarness.WebView;
+
+public partial class MainWindow : Window
+{
+ public MainWindow()
+ {
+ InitializeComponent();
+ Loaded += OnLoaded;
+ }
+
+ private async void OnLoaded(object sender, RoutedEventArgs e)
+ {
+ try
+ {
+ var userData = Path.Combine(Path.GetTempPath(), "CuaTestHarness.WebView.UserData");
+ Directory.CreateDirectory(userData);
+
+ // Read the CDP port from CUA_WEBVIEW_CDP_PORT (default 9222).
+ // cua-driver's `page` tool routes JS execution through CDP when
+ // `--remote-debugging-port` is exposed; this is the analogue of
+ // launching Chrome with --remote-debugging-port for the same
+ // path. Setting it on WebView2 is essential for testing the
+ // page tool against this host.
+ // Validate the CDP port — Chromium accepts --remote-debugging-port=0
+ // (means "pick an ephemeral port") but the harness tests expect a
+ // fixed port and don't read DevToolsActivePort. Reject 0 + any
+ // non-numeric input here so failures are diagnosed at startup
+ // rather than as opaque "CDP discovery timed out" later.
+ var portStr = Environment.GetEnvironmentVariable("CUA_WEBVIEW_CDP_PORT") ?? "9222";
+ if (!ushort.TryParse(portStr, out var cdpPort) || cdpPort == 0)
+ {
+ throw new InvalidOperationException(
+ $"Invalid CUA_WEBVIEW_CDP_PORT: '{portStr}'. Expected an integer in 1-65535.");
+ }
+ var opts = new CoreWebView2EnvironmentOptions
+ {
+ AdditionalBrowserArguments = $"--remote-debugging-port={cdpPort}",
+ };
+ var env = await CoreWebView2Environment.CreateAsync(userDataFolder: userData, options: opts);
+ await Wv.EnsureCoreWebView2Async(env);
+
+ var htmlPath = Path.Combine(AppContext.BaseDirectory, "web", "index.html");
+ if (!File.Exists(htmlPath))
+ {
+ throw new FileNotFoundException(
+ "Harness web entry point not found. The csproj copies " +
+ "../shared-web/index.html to bin/.../web/index.html via the " +
+ "<None Include=...> + Link rule; check that the rule fired.",
+ htmlPath);
+ }
+ var fileUri = new Uri(htmlPath).AbsoluteUri;
+ Wv.Source = new Uri(fileUri);
+ LblPageUrl.Text = fileUri;
+ Title = $"CuaTestHarness WebView [cdp={cdpPort}]";
+ }
+ catch (Exception ex)
+ {
+ MessageBox.Show($"WebView2 init failed: {ex.Message}", "harness", MessageBoxButton.OK, MessageBoxImage.Error);
+ throw;
+ }
+ }
+
+ private void OnExitClick(object sender, RoutedEventArgs e) => Application.Current.Shutdown(0);
+}
diff --git a/libs/cua-driver/test-harness/CuaTestHarness.WebView/app.manifest b/libs/cua-driver/test-harness/CuaTestHarness.WebView/app.manifest
new file mode 100644
index 0000000000..bf395cf4f1
--- /dev/null
+++ b/libs/cua-driver/test-harness/CuaTestHarness.WebView/app.manifest
@@ -0,0 +1,23 @@
+<?xml version="1.0" encoding="utf-8"?>
+<assembly manifestVersion="1.0" xmlns="urn:schemas-microsoft-com:asm.v1">
+ <assemblyIdentity version="1.0.0.0" name="CuaTestHarness.WebView"/>
+ <trustInfo xmlns="urn:schemas-microsoft-com:asm.v2">
+ <security>
+ <requestedPrivileges xmlns="urn:schemas-microsoft-com:asm.v3">
+ <requestedExecutionLevel level="asInvoker" uiAccess="false"/>
+ </requestedPrivileges>
+ </security>
+ </trustInfo>
+ <compatibility xmlns="urn:schemas-microsoft-com:compatibility.v1">
+ <application>
+ <supportedOS Id="{8e0f7a12-bfb3-4fe8-b9a5-48fd50a15a9a}"/>
+ <supportedOS Id="{1f676c76-80e1-4239-95bb-83d0f6d0da78}"/>
+ </application>
+ </compatibility>
+ <application xmlns="urn:schemas-microsoft-com:asm.v3">
+ <windowsSettings>
+ <dpiAware xmlns="http://schemas.microsoft.com/SMI/2005/WindowsSettings">true/pm</dpiAware>
+ <dpiAwareness xmlns="http://schemas.microsoft.com/SMI/2016/WindowsSettings">permonitorv2</dpiAwareness>
+ </windowsSettings>
+ </application>
+</assembly>
diff --git a/libs/cua-driver/test-harness/CuaTestHarness.WinUI3/MainWindow.xaml b/libs/cua-driver/test-harness/CuaTestHarness.WinUI3/MainWindow.xaml
index 1d90e9d82c..9f60b71551 100644
--- a/libs/cua-driver/test-harness/CuaTestHarness.WinUI3/MainWindow.xaml
+++ b/libs/cua-driver/test-harness/CuaTestHarness.WinUI3/MainWindow.xaml
@@ -5,20 +5,11 @@
xmlns:auto="using:Microsoft.UI.Xaml.Automation"
Title="CuaTestHarness WinUI3">
- <Grid Padding="16" RowSpacing="12">
- <Grid.RowDefinitions>
- <RowDefinition Height="Auto"/>
- <RowDefinition Height="Auto"/>
- <RowDefinition Height="Auto"/>
- <RowDefinition Height="Auto"/>
- <RowDefinition Height="Auto"/>
- <RowDefinition Height="Auto"/>
- <RowDefinition Height="*"/>
- <RowDefinition Height="Auto"/>
- </Grid.RowDefinitions>
+ <ScrollViewer>
+ <StackPanel Padding="16" Spacing="10">
<!-- Counter -->
- <StackPanel Grid.Row="0" Orientation="Horizontal" Spacing="8">
+ <StackPanel Orientation="Horizontal" Spacing="8">
<Button x:Name="BtnIncrement"
auto:AutomationProperties.AutomationId="btn-increment"
Content="Increment" Width="100"
@@ -34,14 +25,12 @@
Text="counter=0"/>
</StackPanel>
- <!-- Text body -->
- <TextBlock Grid.Row="1"
- auto:AutomationProperties.AutomationId="txt-body"
+ <TextBlock auto:AutomationProperties.AutomationId="txt-body"
FontFamily="Consolas" TextWrapping="Wrap"
Text="HARNESS_TEXT_MARKER_v1 — WinUI3 text body for get_window_state."/>
<!-- TextBox + mirror -->
- <StackPanel Grid.Row="2" Orientation="Vertical" Spacing="4">
+ <StackPanel Orientation="Vertical" Spacing="4">
<TextBox x:Name="TxtInput"
auto:AutomationProperties.AutomationId="txt-input"
Width="300" HorizontalAlignment="Left"
@@ -52,9 +41,64 @@
Text="mirror="/>
</StackPanel>
+ <!-- Slider -->
+ <StackPanel Spacing="4">
+ <Slider x:Name="SldValue"
+ auto:AutomationProperties.AutomationId="sld-value"
+ Minimum="0" Maximum="100" Value="0"
+ Width="320" HorizontalAlignment="Left"
+ ValueChanged="OnSliderChanged"/>
+ <TextBlock x:Name="LblSliderValue"
+ auto:AutomationProperties.AutomationId="lbl-slider-value"
+ FontFamily="Consolas"
+ Text="slider_value=0"/>
+ </StackPanel>
+
+ <!-- CheckBox + RadioButton group -->
+ <StackPanel Spacing="4">
+ <CheckBox x:Name="ChkAgreed"
+ auto:AutomationProperties.AutomationId="chk-agreed"
+ Content="I agree"
+ Checked="OnChkChanged" Unchecked="OnChkChanged"/>
+ <StackPanel Orientation="Horizontal" Spacing="12">
+ <RadioButton x:Name="RdoLow" Content="Low"
+ auto:AutomationProperties.AutomationId="rdo-low"
+ GroupName="prio" IsChecked="True"
+ Checked="OnRadioChanged"/>
+ <RadioButton x:Name="RdoMed" Content="Medium"
+ auto:AutomationProperties.AutomationId="rdo-med"
+ GroupName="prio"
+ Checked="OnRadioChanged"/>
+ <RadioButton x:Name="RdoHigh" Content="High"
+ auto:AutomationProperties.AutomationId="rdo-high"
+ GroupName="prio"
+ Checked="OnRadioChanged"/>
+ </StackPanel>
+ <TextBlock x:Name="LblChkState"
+ auto:AutomationProperties.AutomationId="lbl-chk-state"
+ FontFamily="Consolas"
+ Text="agreed=False, prio=Low"/>
+ </StackPanel>
+
+ <!-- ComboBox -->
+ <StackPanel Spacing="4">
+ <ComboBox x:Name="CboColor"
+ auto:AutomationProperties.AutomationId="cbo-color"
+ Width="180" HorizontalAlignment="Left"
+ SelectionChanged="OnComboChanged">
+ <ComboBoxItem Content="red" auto:AutomationProperties.AutomationId="cbo-item-red"/>
+ <ComboBoxItem Content="green" auto:AutomationProperties.AutomationId="cbo-item-green" IsSelected="True"/>
+ <ComboBoxItem Content="blue" auto:AutomationProperties.AutomationId="cbo-item-blue"/>
+ <ComboBoxItem Content="orange" auto:AutomationProperties.AutomationId="cbo-item-orange"/>
+ </ComboBox>
+ <TextBlock x:Name="LblComboValue"
+ auto:AutomationProperties.AutomationId="lbl-combo-value"
+ FontFamily="Consolas"
+ Text="color=green"/>
+ </StackPanel>
+
<!-- CommandBarFlyout trigger -->
- <Button Grid.Row="3"
- x:Name="BtnOpenFlyout"
+ <Button x:Name="BtnOpenFlyout"
auto:AutomationProperties.AutomationId="btn-open-flyout"
Content="Open CommandBarFlyout"
Width="220" HorizontalAlignment="Left"
@@ -70,14 +114,13 @@
</Button>
<!-- XAML Popup trigger -->
- <Button Grid.Row="4"
- x:Name="BtnOpenPopup"
+ <Button x:Name="BtnOpenPopup"
auto:AutomationProperties.AutomationId="btn-open-popup"
Content="Open XAML Popup"
Width="220" HorizontalAlignment="Left"
Click="OnOpenPopupClick"/>
- <Popup x:Name="PopXaml" Grid.Row="5"
+ <Popup x:Name="PopXaml"
HorizontalOffset="40" VerticalOffset="40">
<Border Background="#202020" Padding="16" CornerRadius="6"
BorderBrush="#FFFFFF" BorderThickness="1">
@@ -93,10 +136,11 @@
</Popup>
<!-- Exit -->
- <Button Grid.Row="7"
- x:Name="BtnExit"
+ <Button x:Name="BtnExit"
auto:AutomationProperties.AutomationId="btn-exit"
Content="Exit" Width="80" HorizontalAlignment="Right"
Click="OnExitClick"/>
- </Grid>
+
+ </StackPanel>
+ </ScrollViewer>
</Window>
diff --git a/libs/cua-driver/test-harness/CuaTestHarness.WinUI3/MainWindow.xaml.cs b/libs/cua-driver/test-harness/CuaTestHarness.WinUI3/MainWindow.xaml.cs
index 1f25ff23ef..bb48fdca8c 100644
--- a/libs/cua-driver/test-harness/CuaTestHarness.WinUI3/MainWindow.xaml.cs
+++ b/libs/cua-driver/test-harness/CuaTestHarness.WinUI3/MainWindow.xaml.cs
@@ -55,4 +55,31 @@ public sealed partial class MainWindow : Window
{
LblInputMirror.Text = $"mirror={TxtInput.Text}";
}
+
+ private void OnSliderChanged(object sender, Microsoft.UI.Xaml.Controls.Primitives.RangeBaseValueChangedEventArgs e)
+ {
+ if (LblSliderValue is null) return;
+ LblSliderValue.Text = $"slider_value={(int)e.NewValue}";
+ }
+
+ private void UpdateChkState()
+ {
+ if (ChkAgreed is null || RdoLow is null || LblChkState is null) return;
+ var prio = RdoLow.IsChecked == true ? "Low"
+ : RdoMed?.IsChecked == true ? "Medium"
+ : RdoHigh?.IsChecked == true ? "High"
+ : "?";
+ LblChkState.Text = $"agreed={ChkAgreed.IsChecked == true}, prio={prio}";
+ }
+ private void OnChkChanged(object sender, RoutedEventArgs e) => UpdateChkState();
+ private void OnRadioChanged(object sender, RoutedEventArgs e) => UpdateChkState();
+
+ private void OnComboChanged(object sender, SelectionChangedEventArgs e)
+ {
+ if (LblComboValue is null) return;
+ if (CboColor?.SelectedItem is ComboBoxItem item)
+ {
+ LblComboValue.Text = $"color={item.Content}";
+ }
+ }
}
diff --git a/libs/cua-driver/test-harness/CuaTestHarness.Wpf/MainWindow.xaml b/libs/cua-driver/test-harness/CuaTestHarness.Wpf/MainWindow.xaml
index 7c9b9fade4..10e65205ba 100644
--- a/libs/cua-driver/test-harness/CuaTestHarness.Wpf/MainWindow.xaml
+++ b/libs/cua-driver/test-harness/CuaTestHarness.Wpf/MainWindow.xaml
@@ -5,7 +5,7 @@
xmlns:auto="clr-namespace:System.Windows.Automation;assembly=PresentationCore"
Title="CuaTestHarness WPF"
AutomationProperties.AutomationId="wnd-main"
- Width="900" Height="700"
+ Width="900" Height="900"
WindowStartupLocation="CenterScreen">
<Window.InputBindings>
<!-- F5 (no modifiers) — chosen for automated testing. WPF's
@@ -80,6 +80,135 @@
</StackPanel>
</GroupBox>
+ <!-- Slider (drives `drag` tool — thumb at left end, label
+ tracks Value. ValuePattern also surfaced so the cua-driver
+ `set_value` tool can be exercised too.) -->
+ <GroupBox Header="slider" Padding="10" Margin="0,0,0,12">
+ <StackPanel>
+ <Slider x:Name="SldValue"
+ AutomationProperties.AutomationId="sld-value"
+ Minimum="0" Maximum="100" Value="0"
+ Width="320" HorizontalAlignment="Left"
+ IsSnapToTickEnabled="False"
+ ValueChanged="OnSliderChanged"/>
+ <TextBlock x:Name="LblSliderValue"
+ AutomationProperties.AutomationId="lbl-slider-value"
+ Margin="0,6,0,0" FontFamily="Consolas"
+ Text="slider_value=0"/>
+ </StackPanel>
+ </GroupBox>
+
+ <!-- CheckBox + RadioButton group -->
+ <GroupBox Header="checkable_controls" Padding="10" Margin="0,0,0,12">
+ <StackPanel>
+ <CheckBox x:Name="ChkAgreed"
+ AutomationProperties.AutomationId="chk-agreed"
+ Content="I agree" Margin="0,0,0,4"
+ Checked="OnChkChanged" Unchecked="OnChkChanged"/>
+ <StackPanel Orientation="Horizontal">
+ <RadioButton x:Name="RdoLow"
+ AutomationProperties.AutomationId="rdo-low"
+ GroupName="prio" Content="Low" IsChecked="True"
+ Margin="0,0,12,0" Checked="OnRadioChanged"/>
+ <RadioButton x:Name="RdoMed"
+ AutomationProperties.AutomationId="rdo-med"
+ GroupName="prio" Content="Medium"
+ Margin="0,0,12,0" Checked="OnRadioChanged"/>
+ <RadioButton x:Name="RdoHigh"
+ AutomationProperties.AutomationId="rdo-high"
+ GroupName="prio" Content="High"
+ Checked="OnRadioChanged"/>
+ </StackPanel>
+ <TextBlock x:Name="LblChkState"
+ AutomationProperties.AutomationId="lbl-chk-state"
+ Margin="0,6,0,0" FontFamily="Consolas"
+ Text="agreed=False, prio=Low"/>
+ </StackPanel>
+ </GroupBox>
+
+ <!-- ComboBox -->
+ <GroupBox Header="combo_box" Padding="10" Margin="0,0,0,12">
+ <StackPanel>
+ <ComboBox x:Name="CboColor"
+ AutomationProperties.AutomationId="cbo-color"
+ Width="180" HorizontalAlignment="Left"
+ SelectionChanged="OnComboChanged">
+ <ComboBoxItem Content="red" AutomationProperties.AutomationId="cbo-item-red"/>
+ <ComboBoxItem Content="green" AutomationProperties.AutomationId="cbo-item-green" IsSelected="True"/>
+ <ComboBoxItem Content="blue" AutomationProperties.AutomationId="cbo-item-blue"/>
+ <ComboBoxItem Content="orange" AutomationProperties.AutomationId="cbo-item-orange"/>
+ </ComboBox>
+ <TextBlock x:Name="LblComboValue"
+ AutomationProperties.AutomationId="lbl-combo-value"
+ Margin="0,6,0,0" FontFamily="Consolas"
+ Text="color=green"/>
+ </StackPanel>
+ </GroupBox>
+
+ <!-- ListBox with selection -->
+ <GroupBox Header="list_box" Padding="10" Margin="0,0,0,12">
+ <StackPanel>
+ <ListBox x:Name="LstItems"
+ AutomationProperties.AutomationId="lst-items"
+ Width="220" Height="80" HorizontalAlignment="Left"
+ SelectionChanged="OnListChanged">
+ <ListBoxItem Content="apple" AutomationProperties.AutomationId="lst-apple"/>
+ <ListBoxItem Content="banana" AutomationProperties.AutomationId="lst-banana"/>
+ <ListBoxItem Content="cherry" AutomationProperties.AutomationId="lst-cherry"/>
+ <ListBoxItem Content="date" AutomationProperties.AutomationId="lst-date"/>
+ </ListBox>
+ <TextBlock x:Name="LblListValue"
+ AutomationProperties.AutomationId="lbl-list-value"
+ Margin="0,6,0,0" FontFamily="Consolas"
+ Text="selected=none"/>
+ </StackPanel>
+ </GroupBox>
+
+ <!-- Menu + ContextMenu (a Button with attached ContextMenu) -->
+ <GroupBox Header="menus" Padding="10" Margin="0,0,0,12">
+ <StackPanel>
+ <Menu Background="Transparent" HorizontalAlignment="Left">
+ <MenuItem Header="_File"
+ AutomationProperties.AutomationId="menu-file">
+ <MenuItem Header="_New"
+ AutomationProperties.AutomationId="menu-file-new"
+ Click="OnMenuFileNew"/>
+ <MenuItem Header="_Open"
+ AutomationProperties.AutomationId="menu-file-open"
+ Click="OnMenuFileOpen"/>
+ </MenuItem>
+ <MenuItem Header="_Edit"
+ AutomationProperties.AutomationId="menu-edit">
+ <MenuItem Header="_Copy"
+ AutomationProperties.AutomationId="menu-edit-copy"
+ Click="OnMenuEditCopy"/>
+ </MenuItem>
+ </Menu>
+ <Button x:Name="BtnContextTarget"
+ AutomationProperties.AutomationId="btn-context-target"
+ Content="Right-click for context menu"
+ Width="260" Margin="0,8,0,0" HorizontalAlignment="Left">
+ <Button.ContextMenu>
+ <ContextMenu>
+ <MenuItem Header="_Cut"
+ AutomationProperties.AutomationId="ctx-cut"
+ Click="OnCtxAction"/>
+ <MenuItem Header="C_opy"
+ AutomationProperties.AutomationId="ctx-copy"
+ Click="OnCtxAction"/>
+ <MenuItem Header="_Paste"
+ AutomationProperties.AutomationId="ctx-paste"
+ Click="OnCtxAction"/>
+ </ContextMenu>
+ </Button.ContextMenu>
+ </Button>
+ <TextBlock x:Name="LblMenuAction"
+ AutomationProperties.AutomationId="lbl-menu-action"
+ Margin="0,6,0,0" FontFamily="Consolas"
+ Text="menu_action=none"/>
+ </StackPanel>
+ </GroupBox>
+
<!-- Counter scenario -->
<GroupBox Header="counter" Padding="10" Margin="0,0,0,12">
<StackPanel Orientation="Horizontal" VerticalAlignment="Center">
diff --git a/libs/cua-driver/test-harness/CuaTestHarness.Wpf/MainWindow.xaml.cs b/libs/cua-driver/test-harness/CuaTestHarness.Wpf/MainWindow.xaml.cs
index fa4e698e8d..6664135fb6 100644
--- a/libs/cua-driver/test-harness/CuaTestHarness.Wpf/MainWindow.xaml.cs
+++ b/libs/cua-driver/test-harness/CuaTestHarness.Wpf/MainWindow.xaml.cs
@@ -179,4 +179,55 @@ public partial class MainWindow : Window
LblScrollOffset.Text = $"scroll_offset={(int)sv.VerticalOffset}";
}
}
+
+ private void OnSliderChanged(object sender, RoutedPropertyChangedEventArgs<double> e)
+ {
+ LblSliderValue.Text = $"slider_value={(int)e.NewValue}";
+ }
+
+ private void UpdateChkState()
+ {
+ // XAML init fires Checked on the IsChecked="True" RadioButton before
+ // sibling named controls are wired up — guard against the partial
+ // construction.
+ if (ChkAgreed is null || RdoLow is null || LblChkState is null) return;
+ var prio = RdoLow.IsChecked == true ? "Low"
+ : RdoMed?.IsChecked == true ? "Medium"
+ : RdoHigh?.IsChecked == true ? "High"
+ : "?";
+ LblChkState.Text = $"agreed={ChkAgreed.IsChecked == true}, prio={prio}";
+ }
+ private void OnChkChanged(object sender, RoutedEventArgs e) => UpdateChkState();
+ private void OnRadioChanged(object sender, RoutedEventArgs e) => UpdateChkState();
+
+ private void OnComboChanged(object sender, SelectionChangedEventArgs e)
+ {
+ if (LblComboValue is null) return;
+ if (CboColor?.SelectedItem is ComboBoxItem item)
+ {
+ LblComboValue.Text = $"color={item.Content}";
+ }
+ }
+
+ private void OnListChanged(object sender, SelectionChangedEventArgs e)
+ {
+ if (LblListValue is null) return;
+ if (LstItems?.SelectedItem is ListBoxItem item)
+ {
+ LblListValue.Text = $"selected={item.Content}";
+ }
+ }
+
+ private void OnMenuFileNew(object sender, RoutedEventArgs e) => LblMenuAction.Text = "menu_action=file_new";
+ private void OnMenuFileOpen(object sender, RoutedEventArgs e) => LblMenuAction.Text = "menu_action=file_open";
+ private void OnMenuEditCopy(object sender, RoutedEventArgs e) => LblMenuAction.Text = "menu_action=edit_copy";
+
+ private void OnCtxAction(object sender, RoutedEventArgs e)
+ {
+ if (sender is MenuItem mi)
+ {
+ var label = mi.Header?.ToString()?.Replace("_", "").ToLowerInvariant() ?? "?";
+ LblMenuAction.Text = $"menu_action=ctx_{label}";
+ }
+ }
}
diff --git a/libs/cua-driver/test-harness/CuaTestHarness.sln b/libs/cua-driver/test-harness/CuaTestHarness.sln
index 7240a8d213..366302b890 100644
--- a/libs/cua-driver/test-harness/CuaTestHarness.sln
+++ b/libs/cua-driver/test-harness/CuaTestHarness.sln
@@ -6,6 +6,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "CuaTestHarness.Wpf", "CuaTe
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "CuaTestHarness.WinUI3", "CuaTestHarness.WinUI3\CuaTestHarness.WinUI3.csproj", "{B2C3D4E5-F6A7-489B-CDEF-012345678901}"
EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "CuaTestHarness.WebView", "CuaTestHarness.WebView\CuaTestHarness.WebView.csproj", "{C3D4E5F6-A7B8-49CD-EF01-234567890123}"
+EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|x64 = Debug|x64
@@ -20,5 +22,9 @@ Global
{B2C3D4E5-F6A7-489B-CDEF-012345678901}.Debug|x64.Build.0 = Debug|x64
{B2C3D4E5-F6A7-489B-CDEF-012345678901}.Release|x64.ActiveCfg = Release|x64
{B2C3D4E5-F6A7-489B-CDEF-012345678901}.Release|x64.Build.0 = Release|x64
+ {C3D4E5F6-A7B8-49CD-EF01-234567890123}.Debug|x64.ActiveCfg = Debug|x64
+ {C3D4E5F6-A7B8-49CD-EF01-234567890123}.Debug|x64.Build.0 = Debug|x64
+ {C3D4E5F6-A7B8-49CD-EF01-234567890123}.Release|x64.ActiveCfg = Release|x64
+ {C3D4E5F6-A7B8-49CD-EF01-234567890123}.Release|x64.Build.0 = Release|x64
EndGlobalSection
EndGlobal
diff --git a/libs/cua-driver/test-harness/build.ps1 b/libs/cua-driver/test-harness/build.ps1
index 2fa43a8acb..766d52677f 100644
--- a/libs/cua-driver/test-harness/build.ps1
+++ b/libs/cua-driver/test-harness/build.ps1
@@ -11,7 +11,7 @@
# Requires: .NET 8 SDK on PATH.
param(
- [ValidateSet("none","wpf","winui3")]
+ [ValidateSet("none","wpf","winui3","webview","electron")]
[string]$Skip = "none"
)
@@ -63,6 +63,34 @@ if ($Skip -ne "winui3") {
Write-Host "[SKIP] WinUI3 project not present yet - skipping." -ForegroundColor Yellow
}
}
+if ($Skip -ne "webview") {
+ $webProj = Join-Path $harnessDir "CuaTestHarness.WebView\CuaTestHarness.WebView.csproj"
+ if (Test-Path $webProj) {
+ Publish-Project $webProj "harness-webview"
+ } else {
+ Write-Host "[SKIP] WebView project not present yet - skipping." -ForegroundColor Yellow
+ }
+}
+if ($Skip -ne "electron") {
+ $elecBuild = Join-Path $harnessDir "CuaTestHarness.Electron\build.ps1"
+ if (Test-Path $elecBuild) {
+ Write-Host ""
+ Write-Host "[BUILD] CuaTestHarness.Electron -> $testAppsDir\harness-electron\" -ForegroundColor Cyan
+ # Electron build sets $ErrorActionPreference=Stop internally and
+ # throws on npm install / publish failure. Wrap so a throw degrades
+ # to a warning rather than aborting the whole harness build.
+ try {
+ & $elecBuild
+ if ($LASTEXITCODE -ne 0) {
+ Write-Host "[WARN] Electron build failed (exit $LASTEXITCODE)" -ForegroundColor Yellow
+ }
+ } catch {
+ Write-Host "[WARN] Electron build errored: $($_.Exception.Message)" -ForegroundColor Yellow
+ }
+ } else {
+ Write-Host "[SKIP] Electron project not present yet - skipping." -ForegroundColor Yellow
+ }
+}
Write-Host ""
Write-Host "[DONE] Test harness build complete." -ForegroundColor Green
diff --git a/libs/cua-driver/test-harness/scenarios/scenarios.json b/libs/cua-driver/test-harness/scenarios/scenarios.json
index 671cf37921..b753fb3ebd 100644
--- a/libs/cua-driver/test-harness/scenarios/scenarios.json
+++ b/libs/cua-driver/test-harness/scenarios/scenarios.json
@@ -102,6 +102,58 @@
"click_count_label_aid": "lbl-click-count"
}
},
+ {
+ "id": "slider",
+ "kind": "slider_drag",
+ "description": "WPF Slider with ValuePattern. Drives the drag tool (from thumb to a target X) and set_value tool (UIA RangeValue / Value).",
+ "controls": {
+ "slider_aid": "sld-value",
+ "value_label_aid": "lbl-slider-value"
+ }
+ },
+ {
+ "id": "checkable_controls",
+ "kind": "checkbox_radio",
+ "description": "CheckBox + RadioButton group. Drives click via UIA Invoke / TogglePattern. State mirrored to a label.",
+ "controls": {
+ "checkbox_aid": "chk-agreed",
+ "radio_low_aid": "rdo-low",
+ "radio_med_aid": "rdo-med",
+ "radio_high_aid": "rdo-high",
+ "state_label_aid": "lbl-chk-state"
+ }
+ },
+ {
+ "id": "combo_box",
+ "kind": "selection",
+ "description": "ComboBox with named items. Drives selection via click + set_value.",
+ "controls": {
+ "combo_aid": "cbo-color",
+ "value_label_aid": "lbl-combo-value"
+ }
+ },
+ {
+ "id": "list_box",
+ "kind": "list_selection",
+ "description": "ListBox with named items. Tests UIA SelectionPattern.",
+ "controls": {
+ "list_aid": "lst-items",
+ "value_label_aid": "lbl-list-value"
+ }
+ },
+ {
+ "id": "menus",
+ "kind": "menu_and_context",
+ "description": "Top-level Menu + a Button with ContextMenu. Drives MenuItem invocation via click; right-click target for ContextMenu opening.",
+ "controls": {
+ "menu_file_aid": "menu-file",
+ "menu_file_new_aid": "menu-file-new",
+ "menu_edit_copy_aid": "menu-edit-copy",
+ "context_target_aid": "btn-context-target",
+ "ctx_copy_aid": "ctx-copy",
+ "action_label_aid": "lbl-menu-action"
+ }
+ },
{
"id": "scroll_target",
"kind": "scroll_viewer",
@@ -122,6 +174,34 @@
}
]
},
+ "webview": {
+ "process_name": "CuaTestHarness.WebView",
+ "exe_relative_path": "test-apps/harness-webview/CuaTestHarness.WebView.exe",
+ "default_cdp_port": 9222,
+ "main_window": {
+ "title": "CuaTestHarness WebView",
+ "automation_id": "wnd-main"
+ },
+ "description": "WPF host with Microsoft.Edge.WebView2 control. Loads ../shared-web/index.html. Exposes Chromium DevTools Protocol at default_cdp_port for cua-driver's `page` tool.",
+ "shared_dom_aids": [
+ "btn-increment", "btn-reset", "lbl-counter",
+ "txt-input", "lbl-input-mirror",
+ "sld-value", "lbl-slider-value",
+ "border-click-target", "lbl-last-action", "lbl-click-count",
+ "chk-agreed", "rdo-low", "rdo-med", "rdo-high", "lbl-chk-state",
+ "cbo-color", "lbl-combo-value",
+ "lnk-anchor", "lbl-nav-state", "section-target"
+ ]
+ },
+ "electron": {
+ "process_name": "CuaTestHarness.Electron",
+ "exe_relative_path": "test-apps/harness-electron/CuaTestHarness.Electron.exe",
+ "default_cdp_port": 9223,
+ "main_window": {
+ "title": "CuaTestHarness Electron"
+ },
+ "description": "Electron host loading the same ../shared-web/index.html as the WebView harness. Exposes CDP at default_cdp_port via --remote-debugging-port."
+ },
"winui3": {
"process_name": "CuaTestHarness.WinUI3",
"exe_relative_path": "test-apps/harness-winui3/CuaTestHarness.WinUI3.exe",
diff --git a/libs/cua-driver/test-harness/shared-web/index.html b/libs/cua-driver/test-harness/shared-web/index.html
new file mode 100644
index 0000000000..b038c2ec98
--- /dev/null
+++ b/libs/cua-driver/test-harness/shared-web/index.html
@@ -0,0 +1,175 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+ <meta charset="UTF-8" />
+ <title>cua-driver Web Harness</title>
+ <style>
+ :root { color-scheme: light dark; }
+ body {
+ font-family: 'Segoe UI', system-ui, sans-serif;
+ max-width: 860px;
+ margin: 0 auto;
+ padding: 16px;
+ }
+ fieldset { margin-bottom: 12px; padding: 10px 14px; border-radius: 6px; border: 1px solid #ccc; }
+ legend { font-weight: 600; padding: 0 6px; }
+ label { display: inline-block; min-width: 120px; }
+ .row { display: flex; align-items: center; gap: 10px; margin: 6px 0; flex-wrap: wrap; }
+ .mirror, .state { font-family: Consolas, monospace; color: #555; }
+ input[type="range"] { width: 280px; }
+ #click-target {
+ display: inline-block;
+ padding: 8px 18px;
+ background: #eaeaff;
+ border: 1px solid #88a;
+ border-radius: 4px;
+ cursor: pointer;
+ user-select: none;
+ }
+ .marker { font-family: Consolas, monospace; }
+ </style>
+</head>
+<body>
+ <h2 data-cua-id="page-heading">cua-driver Web Harness</h2>
+ <p data-cua-id="page-marker" class="marker">WEB_HARNESS_MARKER_v1</p>
+
+ <fieldset>
+ <legend>counter</legend>
+ <div class="row">
+ <button id="btn-increment" data-cua-id="btn-increment">Increment</button>
+ <button id="btn-reset" data-cua-id="btn-reset">Reset</button>
+ <span class="mirror" id="lbl-counter" data-cua-id="lbl-counter">counter=0</span>
+ </div>
+ </fieldset>
+
+ <fieldset>
+ <legend>text_input</legend>
+ <div class="row">
+ <input type="text" id="txt-input" data-cua-id="txt-input" placeholder="type here" />
+ <span class="mirror" id="lbl-input-mirror" data-cua-id="lbl-input-mirror">mirror=</span>
+ </div>
+ </fieldset>
+
+ <fieldset>
+ <legend>slider</legend>
+ <div class="row">
+ <input type="range" min="0" max="100" value="0"
+ id="sld-value" data-cua-id="sld-value" />
+ <span class="mirror" id="lbl-slider-value" data-cua-id="lbl-slider-value">slider_value=0</span>
+ </div>
+ </fieldset>
+
+ <fieldset>
+ <legend>click_target</legend>
+ <div class="row">
+ <span id="click-target" data-cua-id="border-click-target">Click target (left / right / double)</span>
+ </div>
+ <div class="row">
+ <span class="mirror" id="lbl-last-action" data-cua-id="lbl-last-action">last_action=none</span>
+ <span class="mirror" id="lbl-click-count" data-cua-id="lbl-click-count">clicks=0</span>
+ </div>
+ </fieldset>
+
+ <fieldset>
+ <legend>checkable_controls</legend>
+ <div class="row">
+ <label><input type="checkbox" id="chk-agreed" data-cua-id="chk-agreed" /> I agree</label>
+ </div>
+ <div class="row">
+ <label><input type="radio" name="prio" value="Low" id="rdo-low" data-cua-id="rdo-low" checked /> Low</label>
+ <label><input type="radio" name="prio" value="Medium" id="rdo-med" data-cua-id="rdo-med" /> Medium</label>
+ <label><input type="radio" name="prio" value="High" id="rdo-high" data-cua-id="rdo-high" /> High</label>
+ </div>
+ <div class="row">
+ <span class="state" id="lbl-chk-state" data-cua-id="lbl-chk-state">agreed=false, prio=Low</span>
+ </div>
+ </fieldset>
+
+ <fieldset>
+ <legend>combo_box</legend>
+ <div class="row">
+ <select id="cbo-color" data-cua-id="cbo-color">
+ <option value="red" data-cua-id="cbo-item-red">red</option>
+ <option value="green" data-cua-id="cbo-item-green" selected>green</option>
+ <option value="blue" data-cua-id="cbo-item-blue">blue</option>
+ <option value="orange" data-cua-id="cbo-item-orange">orange</option>
+ </select>
+ <span class="mirror" id="lbl-combo-value" data-cua-id="lbl-combo-value">color=green</span>
+ </div>
+ </fieldset>
+
+ <fieldset>
+ <legend>navigation</legend>
+ <div class="row">
+ <a id="lnk-anchor" data-cua-id="lnk-anchor" href="#section-target">Jump to section</a>
+ <span class="mirror" id="lbl-nav-state" data-cua-id="lbl-nav-state">hash=</span>
+ </div>
+ <div style="height: 100px"></div>
+ <h3 id="section-target" data-cua-id="section-target">SECTION_TARGET_MARKER_v1</h3>
+ </fieldset>
+
+ <script>
+ // ----- state -----
+ let counter = 0;
+ let clickCount = 0;
+
+ function $(id) { return document.getElementById(id); }
+ function updateChkState() {
+ const agreed = $('chk-agreed').checked;
+ const prio = document.querySelector('input[name="prio"]:checked')?.value ?? '?';
+ $('lbl-chk-state').textContent = `agreed=${agreed}, prio=${prio}`;
+ }
+
+ $('btn-increment').addEventListener('click', () => {
+ counter++;
+ $('lbl-counter').textContent = `counter=${counter}`;
+ });
+ $('btn-reset').addEventListener('click', () => {
+ counter = 0;
+ $('lbl-counter').textContent = 'counter=0';
+ });
+
+ $('txt-input').addEventListener('input', e => {
+ $('lbl-input-mirror').textContent = `mirror=${e.target.value}`;
+ });
+
+ $('sld-value').addEventListener('input', e => {
+ $('lbl-slider-value').textContent = `slider_value=${e.target.value}`;
+ });
+
+ const target = $('click-target');
+ target.addEventListener('mousedown', e => {
+ clickCount++;
+ if (e.detail >= 2) {
+ $('lbl-last-action').textContent = 'last_action=double_click';
+ } else if (e.button === 2) {
+ $('lbl-last-action').textContent = 'last_action=right_click';
+ } else {
+ $('lbl-last-action').textContent = 'last_action=left_click';
+ }
+ $('lbl-click-count').textContent = `clicks=${clickCount}`;
+ });
+ target.addEventListener('contextmenu', e => {
+ e.preventDefault(); // suppress browser default so right-click registers cleanly
+ $('lbl-last-action').textContent = 'last_action=right_click';
+ });
+ target.addEventListener('dblclick', () => {
+ $('lbl-last-action').textContent = 'last_action=double_click';
+ });
+
+ $('chk-agreed').addEventListener('change', updateChkState);
+ document.querySelectorAll('input[name="prio"]').forEach(r => r.addEventListener('change', updateChkState));
+
+ $('cbo-color').addEventListener('change', e => {
+ $('lbl-combo-value').textContent = `color=${e.target.value}`;
+ });
+
+ $('lnk-anchor').addEventListener('click', () => {
+ // Defer so the browser has applied the hash by the time we read it.
+ setTimeout(() => {
+ $('lbl-nav-state').textContent = `hash=${location.hash}`;
+ }, 30);
+ });
+ </script>
+</body>
+</html>