Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
fix(cua-driver): drain Windows foreground input before restoring focus (#4500)
wszkxlllllc8edda06be53
diff --git a/libs/cua-driver/docs/test-matrix.md b/libs/cua-driver/docs/test-matrix.mdindex 17777dcb38..e16f850a72 100644--- a/libs/cua-driver/docs/test-matrix.md+++ b/libs/cua-driver/docs/test-matrix.md@@ -168,14 +168,22 @@ Windows native harnesses are repo-local applications built from source:| Harness | Source test | Coverage || --- | --- | --- |-| WPF | `harness_wpf_test.rs` | UIA controls, text, keys, pointer actions, scroll, drag, popups, menus, modal windows |-| WinUI3 | `harness_winui3_test.rs` | XAML controls, text, checkbox/radio, slider, combo, popup |+| WPF | `harness_wpf_test.rs` | UIA controls, text, keys, pointer actions, scroll, drag, popups, menus, modal windows; foreground single-line and multiline text integrity |+| WinUI3 | `harness_winui3_test.rs` | XAML controls, text, checkbox/radio, slider, combo, popup; foreground single-line and multiline text integrity || WebView2 | `harness_web_test.rs` | Window discovery, CDP page access, JavaScript, DOM click path || Desktop target | `desktop_scope_windows_test.rs` | Per-call window/desktop modality, full-display capture, screen-absolute click/scroll, and strict rejection || Desktop invariants | Testkit `DesktopObserver` plus typed launch/capture/cursor owners | Cross-cutting focus, z-order, minimized-launch, screenshot, cursor, and desktop checks |-Native controls use AX/UIA state as their oracle. Pointer actions also use PX-where the tool contract requires coordinates. Current `set_value` rows declare+Native controls use AX/UIA state as their oracle. The foreground typing rows+also compare each fixture's real `TextBox.Text` values from an isolated,+atomically replaced state file, including the target and leakage sentinels.+The single-line row covers CJK, digits, and emoji; the multiline sample covers+LF, CR, CRLF, repeated empty lines, and leading/trailing breaks. The oracle+normalizes line endings only and preserves all other characters and blank+lines. Foreground delivery keeps the target in front until its thread has+read every inserted event (a sentinel-key drain), so text after line breaks is+not handed to the previously active window. Pointer actions also use PX where+the tool contract requires coordinates. Current `set_value` rows declarebackground delivery and attach desktop-side-effect oracles.## Harness E2E: macOSdiff --git a/libs/cua-driver/rust/crates/cua-driver-e2e/tests/harness_winui3_test.rs b/libs/cua-driver/rust/crates/cua-driver-e2e/tests/harness_winui3_test.rsindex 358cef72ac..70302c8269 100644--- a/libs/cua-driver/rust/crates/cua-driver-e2e/tests/harness_winui3_test.rs+++ b/libs/cua-driver/rust/crates/cua-driver-e2e/tests/harness_winui3_test.rs@@ -24,13 +24,17 @@ use std::time::{Duration, Instant};use cua_driver_testkit::ax::element_index_by_id;use cua_driver_testkit::e2e::{- execute_case, native_background_case, native_readonly_case, recording_evidence, DriverRoute,- Evidence, Observation, OracleKind, Targeting,+ execute_case, native_background_case, native_foreground_case, native_readonly_case,+ recording_evidence, DriverRoute, Evidence, Observation, OracleKind, Targeting,};use cua_driver_testkit::observer::TargetWindow;use cua_driver_testkit::sentinel::run_with_background_oracles;use cua_driver_testkit::{harness_app, spawn_in_job, Driver, McpDriver};+#[path = "support/foreground_text_oracle.rs"]+mod foreground_text_oracle;+use foreground_text_oracle::{ascii_escape, normalize_line_endings, stable_text};+/// Resolve the WinUI3 harness exe — the `HARNESS_WINUI3_EXE` override wins (if it/// points at an existing file), else the built path under `test-apps/`.fn harness_winui3_exe() -> PathBuf {@@ -46,17 +50,24 @@ fn harness_winui3_exe() -> PathBuf {/// Launch the WinUI3 harness through the driver's reaper (kill-on-close Job/// Object) and return its pid. Returns `None` (skip) if the harness isn't built.fn launch_winui3(driver: &mut McpDriver) -> Option<u32> {+ launch_winui3_with_state_file(driver, None)+}++fn launch_winui3_with_state_file(+ driver: &mut McpDriver,+ state_path: Option<&std::path::Path>,+) -> Option<u32> {let exe = harness_winui3_exe();if !exe.exists() {eprintln!("WinUI3 harness exe not found at {exe:?} — run tests/fixtures/build/windows.ps1");return None;}- let child = spawn_in_job(- Command::new(&exe)- .stdout(Stdio::null())- .stderr(Stdio::null()),- )- .ok()?;+ let mut command = Command::new(&exe);+ command.stdout(Stdio::null()).stderr(Stdio::null());+ if let Some(path) = state_path {+ command.env("CUA_E2E_FIXTURE_STATE_PATH", path);+ }+ let child = spawn_in_job(&mut command).ok()?;let pid = child.id();driver.reaper().push(child);Some(pid)@@ -277,6 +288,161 @@ fn harness_winui3_type_text() {});}+#[test]+#[ignore]+fn harness_winui3_type_text_foreground_single_line_application_state() {+ const TEXT: &str = "FOREGROUND-SINGLE-19 \u{4e2d}\u{6587} 12345 \u{1f642}";++ let mut case = native_foreground_case(+ "winui3",+ "type_text",+ Targeting::Ax,+ DriverRoute::WindowsSendInput,+ );+ case.cell_id.push_str("-single-line-state");+ let cell_id = case.cell_id.clone();++ execute_case(case, |evidence| {+ let state_dir = tempfile::tempdir().expect("create isolated WinUI3 state directory");+ let state_path = state_dir.path().join("fixture-state.json");+ let mut driver = McpDriver::spawn_named(&cell_id)+ .expect("required source-built Windows driver did not start");+ *evidence = recording_evidence(driver.recording_dir());++ let pid = launch_winui3_with_state_file(&mut driver, Some(&state_path))+ .expect("required WinUI3 harness did not launch");+ let (wid, _) = driver+ .find_window(pid as i64, "CuaTestHarness WinUI3")+ .expect("WinUI3 main window not found");+ wait_for_winui3_ready(&mut driver, pid, wid);+ assert_eq!(stable_text(&state_path, "txt-input"), "");++ let snap = driver.call(+ "get_window_state",+ serde_json::json!({"pid": pid as i64, "window_id": wid, "capture_mode":"ax"}),+ );+ let idx = element_index_by_id(snap.text(), "txt-input")+ .expect("txt-input not in WinUI3 snapshot");+ driver.start_behavior_recording();+ let response = driver.call(+ "type_text",+ serde_json::json!({+ "pid": pid as i64,+ "window_id": wid,+ "element_token": snap.element_token(idx),+ "text": TEXT,+ "delivery_mode": "foreground"+ }),+ );+ assert!(+ !response.is_error(),+ "foreground type_text failed: {}",+ response.text()+ );+ assert_eq!(response.structured()["route"], "global_input");+ assert_eq!(stable_text(&state_path, "txt-input"), TEXT);+ assert_eq!(stable_text(&state_path, "txt-multiline-input"), "");++ drop(driver);+ Observation::delivered(vec![OracleKind::FixtureState], Evidence::default())+ });+}++#[test]+#[ignore]+fn harness_winui3_type_text_multiline_application_state() {+ const TEXT: &str = concat!(+ "\n",+ "CUA-MULTILINE-START\n",+ "ASCII-LF-A-31\nASCII-LF-B-47\n",+ "ASCII-CRLF-A-52\r\nASCII-CRLF-B-68\n",+ "ASCII-CR-A-13\rASCII-CR-B-24\n",+ "\u{8f66}\u{6b21} G7391\r\n",+ "\u{4e2d}\u{6587} 12345\n",+ "EMPTY-A-17\n\n\nEMPTY-B-29\n",+ "EMOJI-A \u{1f642} 123\r\n",+ "EMOJI-B \u{1f680} 456\n",+ "CUA-MULTILINE-END\n"+ );++ let mut case = native_foreground_case(+ "winui3",+ "type_text",+ Targeting::Ax,+ DriverRoute::WindowsSendInput,+ );+ case.cell_id.push_str("-multiline");+ let cell_id = case.cell_id.clone();++ execute_case(case, |evidence| {+ let state_dir = tempfile::tempdir().expect("create isolated WinUI3 state directory");+ let state_path = state_dir.path().join("fixture-state.json");+ let mut driver = McpDriver::spawn_named(&cell_id)+ .expect("required source-built Windows driver did not start");+ *evidence = recording_evidence(driver.recording_dir());++ let pid = launch_winui3_with_state_file(&mut driver, Some(&state_path))+ .expect("required WinUI3 harness did not launch");+ let (wid, _) = driver+ .find_window(pid as i64, "CuaTestHarness WinUI3")+ .expect("WinUI3 main window not found");+ wait_for_winui3_ready(&mut driver, pid, wid);+ assert_eq!(+ stable_text(&state_path, "txt-multiline-input"),+ "",+ "multiline fixture must start blank"+ );++ let snap = driver.call(+ "get_window_state",+ serde_json::json!({+ "pid": pid as i64, "window_id": wid, "capture_mode": "ax"+ }),+ );+ let idx = element_index_by_id(snap.text(), "txt-multiline-input")+ .expect("txt-multiline-input not in WinUI3 snapshot");+ driver.start_behavior_recording();+ let response = driver.call(+ "type_text",+ serde_json::json!({+ "pid": pid as i64,+ "window_id": wid,+ "element_token": snap.element_token(idx),+ "text": TEXT,+ "delivery_mode": "foreground"+ }),+ );+ assert!(+ !response.is_error(),+ "WinUI3 foreground multiline type_text failed: {}",+ response.text()+ );+ assert_eq!(+ response.structured()["route"],+ "global_input",+ "foreground multiline input must use SendInput: {}",+ response.structured()+ );++ let actual = stable_text(&state_path, "txt-multiline-input");+ assert_eq!(+ normalize_line_endings(&actual),+ normalize_line_endings(TEXT),+ "WinUI3 app-owned multiline state differs; actual={} expected={}",+ ascii_escape(&actual),+ ascii_escape(TEXT)+ );+ assert_eq!(+ stable_text(&state_path, "txt-input"),+ "",+ "foreground multiline input leaked to the single-line sibling"+ );++ drop(driver);+ Observation::delivered(vec![OracleKind::FixtureState], Evidence::default())+ });+}+#[test]#[ignore]fn harness_winui3_xaml_popup_open() {diff --git a/libs/cua-driver/rust/crates/cua-driver-e2e/tests/harness_wpf_test.rs b/libs/cua-driver/rust/crates/cua-driver-e2e/tests/harness_wpf_test.rsindex 3f3d3fac93..c6a0740a38 100644--- a/libs/cua-driver/rust/crates/cua-driver-e2e/tests/harness_wpf_test.rs+++ b/libs/cua-driver/rust/crates/cua-driver-e2e/tests/harness_wpf_test.rs@@ -51,6 +51,10 @@ use cua_driver_testkit::{ax, harness_app, spawn_in_job, Driver, McpDriver, ToolRuse windows::Win32::Foundation::HWND;use windows::Win32::UI::WindowsAndMessaging::{IsIconic, ShowWindow, SW_MINIMIZE};+#[path = "support/foreground_text_oracle.rs"]+mod foreground_text_oracle;+use foreground_text_oracle::{ascii_escape, normalize_line_endings, stable_text};+// ── harness launcher ─────────────────────────────────────────────────────────fn harness_exe() -> PathBuf {@@ -847,6 +851,151 @@ fn harness_wpf_type_text() {);}+#[test]+#[ignore]+fn harness_wpf_type_text_foreground_single_line_application_state() {+ const TEXT: &str = "FOREGROUND-SINGLE-19 \u{4e2d}\u{6587} 12345 \u{1f642}";++ let mut case = native_foreground_case(+ "wpf",+ "type_text",+ Targeting::Ax,+ DriverRoute::WindowsSendInput,+ );+ case.cell_id.push_str("-single-line-state");+ let cell_id = case.cell_id.clone();++ execute_case(case, |evidence| {+ let state_dir = tempfile::tempdir().expect("create isolated WPF state directory");+ let state_path = state_dir.path().join("fixture-state.json");+ let mut driver = McpDriver::spawn_named(&cell_id)+ .expect("required source-built Windows driver did not start");+ *evidence = recording_evidence(driver.recording_dir());++ let pid = launch_harness_with_state_file(&mut driver, Some(&state_path))+ .expect("required WPF harness did not launch");+ let (wid, _) = driver+ .find_window(pid as i64, "CuaTestHarness WPF")+ .expect("WPF main window not found");+ assert_eq!(stable_text(&state_path, "txt-input"), "");++ let snap = snapshot(&mut driver, pid, wid);+ let idx = ax::element_index_by_id(snap.tree_text(), "txt-input")+ .expect("txt-input not in WPF snapshot");+ driver.start_behavior_recording();+ let response = driver.call(+ "type_text",+ serde_json::json!({+ "pid": pid as i64,+ "window_id": wid,+ "element_token": snap.element_token(idx),+ "text": TEXT,+ "delivery_mode": "foreground"+ }),+ );+ assert!(+ !response.is_error(),+ "foreground type_text failed: {}",+ response.text()+ );+ assert_eq!(response.structured()["route"], "global_input");+ assert_eq!(stable_text(&state_path, "txt-input"), TEXT);+ assert_eq!(stable_text(&state_path, "txt-deferred-input"), "");++ drop(driver);+ Observation::delivered(vec![OracleKind::FixtureState], Evidence::default())+ });+}++#[test]+#[ignore]+fn harness_wpf_type_text_multiline_application_state() {+ const TEXT: &str = concat!(+ "\n",+ "CUA-MULTILINE-START\n",+ "ASCII-LF-A-31\nASCII-LF-B-47\n",+ "ASCII-CRLF-A-52\r\nASCII-CRLF-B-68\n",+ "ASCII-CR-A-13\rASCII-CR-B-24\n",+ "\u{8f66}\u{6b21} G7391\r\n",+ "\u{4e2d}\u{6587} 12345\n",+ "EMPTY-A-17\n\n\nEMPTY-B-29\n",+ "EMOJI-A \u{1f642} 123\r\n",+ "EMOJI-B \u{1f680} 456\n",+ "CUA-MULTILINE-END\n"+ );++ let mut case = native_foreground_case(+ "wpf",+ "type_text",+ Targeting::Ax,+ DriverRoute::WindowsSendInput,+ );+ case.cell_id.push_str("-multiline");+ let cell_id = case.cell_id.clone();++ execute_case(case, |evidence| {+ let state_dir = tempfile::tempdir().expect("create isolated WPF state directory");+ let state_path = state_dir.path().join("fixture-state.json");+ let mut driver = McpDriver::spawn_named(&cell_id)+ .expect("required source-built Windows driver did not start");+ *evidence = recording_evidence(driver.recording_dir());++ let pid = launch_harness_with_state_file(&mut driver, Some(&state_path))+ .expect("required WPF harness did not launch");+ let (wid, _) = driver+ .find_window(pid as i64, "CuaTestHarness WPF")+ .expect("WPF main window not found");+ assert_eq!(+ stable_text(&state_path, "txt-multiline-input"),+ "",+ "multiline fixture must start blank"+ );++ let snap = snapshot(&mut driver, pid, wid);+ let idx = ax::element_index_by_id(snap.tree_text(), "txt-multiline-input")+ .expect("txt-multiline-input not in WPF snapshot");+ driver.start_behavior_recording();+ let response = driver.call(+ "type_text",+ serde_json::json!({+ "pid": pid as i64,+ "window_id": wid,+ "element_token": snap.element_token(idx),+ "text": TEXT,+ "delivery_mode": "foreground"+ }),+ );+ assert!(+ !response.is_error(),+ "WPF foreground multiline type_text failed: {}",+ response.text()+ );+ assert_eq!(+ response.structured()["route"],+ "global_input",+ "foreground multiline input must use SendInput: {}",+ response.structured()+ );++ let actual = stable_text(&state_path, "txt-multiline-input");+ assert_eq!(+ normalize_line_endings(&actual),+ normalize_line_endings(TEXT),+ "WPF app-owned multiline state differs; actual={} expected={}",+ ascii_escape(&actual),+ ascii_escape(TEXT)+ );+ assert_eq!(+ stable_text(&state_path, "txt-deferred-input"),+ "",+ "foreground multiline input leaked to the startup-focused decoy"+ );++ drop(driver);+ Observation::delivered(vec![OracleKind::FixtureState], Evidence::default())+ });+}+#[test]#[ignore]fn harness_wpf_set_value() {diff --git a/libs/cua-driver/rust/crates/cua-driver-e2e/tests/support/foreground_text_oracle.rs b/libs/cua-driver/rust/crates/cua-driver-e2e/tests/support/foreground_text_oracle.rsnew file mode 100644index 0000000000..8fcd517624--- /dev/null+++ b/libs/cua-driver/rust/crates/cua-driver-e2e/tests/support/foreground_text_oracle.rs@@ -0,0 +1,55 @@+use std::fs;+use std::path::Path;+use std::thread::sleep;+use std::time::{Duration, Instant};++/// Read app-owned fixture state until its text value has remained unchanged+/// for a bounded interval. This observes the control model directly, not the+/// driver's post-action accessibility snapshot.+pub fn stable_text(path: &Path, automation_id: &str) -> String {+ let deadline = Instant::now() + Duration::from_secs(5);+ let mut last_value: Option<String> = None;+ let mut unchanged_since = Instant::now();++ loop {+ if let Ok(bytes) = fs::read(path) {+ if let Ok(state) = serde_json::from_slice::<serde_json::Value>(&bytes) {+ if let Some(value) = state[automation_id]["text"].as_str() {+ let value = value.to_owned();+ if last_value.as_ref() != Some(&value) {+ last_value = Some(value);+ unchanged_since = Instant::now();+ } else if unchanged_since.elapsed() >= Duration::from_millis(500) {+ return value;+ }+ }+ }+ }++ assert!(+ Instant::now() < deadline,+ "fixture state did not stabilize for {automation_id}; last value: {}",+ last_value+ .as_deref()+ .map(ascii_escape)+ .unwrap_or_else(|| "<missing>".into())+ );+ sleep(Duration::from_millis(25));+ }+}++pub fn normalize_line_endings(text: &str) -> String {+ text.replace("\r\n", "\n").replace('\r', "\n")+}++pub fn ascii_escape(text: &str) -> String {+ text.chars()+ .map(|ch| match ch {+ '\n' => "\\n".to_owned(),+ '\r' => "\\r".to_owned(),+ '\t' => "\\t".to_owned(),+ ch if ch.is_ascii_graphic() || ch == ' ' => ch.to_string(),+ ch => format!("\\u{{{:x}}}", ch as u32),+ })+ .collect()+}diff --git a/libs/cua-driver/rust/crates/platform-windows/src/input/keyboard.rs b/libs/cua-driver/rust/crates/platform-windows/src/input/keyboard.rsindex a814511960..116f094529 100644--- a/libs/cua-driver/rust/crates/platform-windows/src/input/keyboard.rs+++ b/libs/cua-driver/rust/crates/platform-windows/src/input/keyboard.rs@@ -618,6 +618,7 @@ fn with_confirmed_foreground<T>();};+ let mut attachment: Option<InputQueueAttachment> = None;let result = (|| {focus()?;let deadline = Instant::now() + Duration::from_millis(250);@@ -646,20 +647,114 @@ fn with_confirmed_foreground<T>(actual.0);}+ // Attach before inserting anything: attaching resets the shared key+ // state, which must not race with modifiers the body is about to send.+ attachment = InputQueueAttachment::attach(actual);body()})();- // Give the target message loop a bounded opportunity to consume the- // inserted sequence before restoring the user's prior foreground.- if result.is_ok() {- sleep(Duration::from_millis(40));+ // Keep the target foreground until its thread has read every inserted+ // event; restoring earlier hands the unread tail to another window (#4477).+ match (&result, attachment.as_ref()) {+ (Ok(_), Some(attachment)) => {+ attachment.wait_for_drain(Duration::from_secs(2));+ }+ (Ok(_), None) => sleep(Duration::from_millis(40)),+ _ => {}}+ drop(attachment);if !previous.0.is_null() && previous != target {let _ = unsafe { SetForegroundWindow(previous) };}result}+/// Unassigned virtual key used as a harmless input-drain sentinel (the same+/// "mask key" convention AutoHotkey uses): applications do not bind it and it+/// produces no character.+const DRAIN_SENTINEL_VK: VIRTUAL_KEY = VIRTUAL_KEY(0xE8);++/// The caller's thread attached to the foreground thread's input queue for+/// the duration of one foreground input transaction; detaches on drop.+struct InputQueueAttachment {+ own_thread: u32,+ target_thread: u32,+}++impl InputQueueAttachment {+ fn attach(foreground: HWND) -> Option<Self> {+ use windows::Win32::System::Threading::{AttachThreadInput, GetCurrentThreadId};+ let target_thread = unsafe { GetWindowThreadProcessId(foreground, None) };+ let own_thread = unsafe { GetCurrentThreadId() };+ if target_thread == 0 || target_thread == own_thread {+ return None;+ }+ unsafe { AttachThreadInput(own_thread, target_thread, true) }+ .as_bool()+ .then_some(Self {+ own_thread,+ target_thread,+ })+ }++ /// Block until the foreground thread has retrieved all input inserted so+ /// far, bounded by `timeout`. A sentinel key press is appended behind the+ /// caller's events; while attached, `GetKeyState` reflects the shared+ /// queue's synchronous key state, whose toggle bit flips only when the+ /// target thread reads the sentinel key-down.+ fn wait_for_drain(&self, timeout: Duration) -> bool {+ use windows::Win32::UI::Input::KeyboardAndMouse::GetKeyState;+ let toggled = || unsafe { GetKeyState(DRAIN_SENTINEL_VK.0 as i32) } & 1;+ let before = toggled();+ let sentinel = [sentinel_key_input(false), sentinel_key_input(true)];+ let sent = unsafe { SendInput(&sentinel, std::mem::size_of::<INPUT>() as i32) };+ if sent as usize != sentinel.len() {+ sleep(Duration::from_millis(40));+ return false;+ }+ let deadline = Instant::now() + timeout;+ loop {+ if toggled() != before {+ return true;+ }+ if Instant::now() >= deadline {+ tracing::warn!(+ "foreground input drain: target thread did not consume the sentinel within {} ms",+ timeout.as_millis()+ );+ return false;+ }+ sleep(Duration::from_millis(5));+ }+ }+}++impl Drop for InputQueueAttachment {+ fn drop(&mut self) {+ use windows::Win32::System::Threading::AttachThreadInput;+ let _ = unsafe { AttachThreadInput(self.own_thread, self.target_thread, false) };+ }+}++fn sentinel_key_input(up: bool) -> INPUT {+ INPUT {+ r#type: INPUT_KEYBOARD,+ Anonymous: INPUT_0 {+ ki: KEYBDINPUT {+ wVk: DRAIN_SENTINEL_VK,+ wScan: 0,+ dwFlags: if up {+ KEYEVENTF_KEYUP+ } else {+ KEYBD_EVENT_FLAGS(0)+ },+ time: 0,+ dwExtraInfo: 0,+ },+ },+ }+}+/// Build a single Unicode keyboard INPUT struct for one UTF-16 code unit,/// either down (`up = false`) or up (`up = true`). Used by/// [`send_text_synthesized`].diff --git a/libs/cua-driver/tests/fixtures/apps/windows/winui3/MainWindow.xaml b/libs/cua-driver/tests/fixtures/apps/windows/winui3/MainWindow.xamlindex 70a15f475a..4ed670d131 100644--- a/libs/cua-driver/tests/fixtures/apps/windows/winui3/MainWindow.xaml+++ b/libs/cua-driver/tests/fixtures/apps/windows/winui3/MainWindow.xaml@@ -65,6 +65,13 @@Text="mirror="/></StackPanel>+ <!-- Dedicated real multiline editor for foreground type_text tests. -->+ <TextBox x:Name="TxtMultilineInput"+ auto:AutomationProperties.AutomationId="txt-multiline-input"+ Width="420" Height="100" HorizontalAlignment="Left"+ AcceptsReturn="True" TextWrapping="Wrap"+ TextChanged="OnMultilineInputChanged"/>+<!-- Slider --><StackPanel Spacing="4"><Slider x:Name="SldValue"diff --git a/libs/cua-driver/tests/fixtures/apps/windows/winui3/MainWindow.xaml.cs b/libs/cua-driver/tests/fixtures/apps/windows/winui3/MainWindow.xaml.csindex 132993f3ba..0e965c5ef5 100644--- a/libs/cua-driver/tests/fixtures/apps/windows/winui3/MainWindow.xaml.cs+++ b/libs/cua-driver/tests/fixtures/apps/windows/winui3/MainWindow.xaml.cs@@ -1,5 +1,9 @@using System;+using System.Collections.Generic;+using System.IO;using System.Runtime.InteropServices;+using System.Text.Json;+using Microsoft.UI.Dispatching;using Microsoft.UI.Xaml;using Microsoft.UI.Xaml.Controls;using Microsoft.UI.Xaml.Input;@@ -11,6 +15,8 @@ public sealed partial class MainWindow : Windowprivate int _counter;private int _clickCount;private DateTime _lastClickTime = DateTime.MinValue;+ private readonly string? _fixtureStatePath = Environment.GetEnvironmentVariable("CUA_E2E_FIXTURE_STATE_PATH");+ private DispatcherQueueTimer? _fixtureStateTimer;// Keep a strong reference to the subclass delegate so the GC doesn't// collect it while Win32 still holds the function pointer.@@ -21,6 +27,14 @@ public sealed partial class MainWindow : WindowInitializeComponent();Title = "CuaTestHarness WinUI3";InstallScrollMessageHook();+ if (!string.IsNullOrWhiteSpace(_fixtureStatePath))+ {+ _fixtureStateTimer = DispatcherQueue.GetForCurrentThread().CreateTimer();+ _fixtureStateTimer.Interval = TimeSpan.FromMilliseconds(50);+ _fixtureStateTimer.IsRepeating = false;+ _fixtureStateTimer.Tick += (_, _) => PublishFixtureState();+ }+ PublishFixtureState();}private void OnIncrementClick(object sender, RoutedEventArgs e)@@ -64,6 +78,40 @@ public sealed partial class MainWindow : Windowprivate void OnInputChanged(object sender, TextChangedEventArgs e){LblInputMirror.Text = $"mirror={TxtInput.Text}";+ ScheduleFixtureStatePublish();+ }++ private void OnMultilineInputChanged(object sender, TextChangedEventArgs e)+ {+ ScheduleFixtureStatePublish();+ }++ private void ScheduleFixtureStatePublish()+ {+ if (_fixtureStateTimer is not { } timer) return;+ timer.Stop();+ timer.Start();+ }++ private void PublishFixtureState()+ {+ if (string.IsNullOrWhiteSpace(_fixtureStatePath)) return;++ var state = new Dictionary<string, object>+ {+ ["txt-input"] = new { text = TxtInput?.Text ?? "" },+ ["txt-multiline-input"] = new { text = TxtMultilineInput?.Text ?? "" },+ };+ try+ {+ var temporaryPath = $"{_fixtureStatePath}.{Environment.ProcessId}.tmp";+ File.WriteAllText(temporaryPath, JsonSerializer.Serialize(state));+ File.Move(temporaryPath, _fixtureStatePath, true);+ }+ catch (Exception ex)+ {+ Console.Error.WriteLine($"WinUI3 fixture state publish failed: {ex.Message}");+ }}private void OnSliderChanged(object sender, Microsoft.UI.Xaml.Controls.Primitives.RangeBaseValueChangedEventArgs e)diff --git a/libs/cua-driver/tests/fixtures/apps/windows/wpf/MainWindow.xaml b/libs/cua-driver/tests/fixtures/apps/windows/wpf/MainWindow.xamlindex 190f9a3d28..74ee62e202 100644--- a/libs/cua-driver/tests/fixtures/apps/windows/wpf/MainWindow.xaml+++ b/libs/cua-driver/tests/fixtures/apps/windows/wpf/MainWindow.xaml@@ -77,6 +77,14 @@</Grid></GroupBox>+ <GroupBox Header="multiline_text_input" Padding="10" Margin="0,0,0,12">+ <TextBox x:Name="TxtMultilineInput"+ AutomationProperties.AutomationId="txt-multiline-input"+ Width="420" Height="100" HorizontalAlignment="Left"+ AcceptsReturn="True" TextWrapping="Wrap"+ TextChanged="OnMultilineInputChanged"/>+ </GroupBox>+<!-- Click target (Button) — also at top for visibility. --><GroupBox Header="click_target" Padding="10" Margin="0,0,0,12"><StackPanel>diff --git a/libs/cua-driver/tests/fixtures/apps/windows/wpf/MainWindow.xaml.cs b/libs/cua-driver/tests/fixtures/apps/windows/wpf/MainWindow.xaml.csindex 4f7eb410a7..0c90d19a1e 100644--- a/libs/cua-driver/tests/fixtures/apps/windows/wpf/MainWindow.xaml.cs+++ b/libs/cua-driver/tests/fixtures/apps/windows/wpf/MainWindow.xaml.cs@@ -23,6 +23,10 @@ public partial class MainWindow : Windowprivate bool _targetPointerSeen;private readonly ScenariosManifest _manifest;private readonly string? _fixtureStatePath;+ private readonly System.Windows.Threading.DispatcherTimer _fixtureStateTimer = new()+ {+ Interval = TimeSpan.FromMilliseconds(50)+ };public MainWindow(){@@ -32,6 +36,11 @@ public partial class MainWindow : WindowTitle = _manifest.Wpf.MainWindow.Title;AutomationProperties.SetAutomationId(this, _manifest.Wpf.MainWindow.AutomationId);+ _fixtureStateTimer.Tick += (_, _) =>+ {+ _fixtureStateTimer.Stop();+ PublishFixtureState();+ };CommandBindings.Add(new CommandBinding(AccelCmd, (s, e) =>{@@ -155,11 +164,25 @@ public partial class MainWindow : Windowprivate void OnInputChanged(object sender, TextChangedEventArgs e){LblInputMirror.Text = $"mirror={TxtInput.Text}";+ ScheduleFixtureStatePublish();+ }++ private void OnMultilineInputChanged(object sender, TextChangedEventArgs e)+ {+ ScheduleFixtureStatePublish();+ }++ private void ScheduleFixtureStatePublish()+ {+ if (string.IsNullOrWhiteSpace(_fixtureStatePath)) return;+ _fixtureStateTimer.Stop();+ _fixtureStateTimer.Start();}private void OnDeferredInputChanged(object sender, TextChangedEventArgs e){LblDeferredInputMirror.Text = $"deferred_mirror={TxtDeferredInput.Text}";+ ScheduleFixtureStatePublish();}private void OnTargetLeftDown(object sender, MouseButtonEventArgs e)@@ -224,6 +247,9 @@ public partial class MainWindow : Window["lbl-counter"] = new { text = LblCounter?.Text ?? "counter=0" },["lbl-last-action"] = new { text = LblLastAction?.Text ?? "last_action=none" },["lbl-click-count"] = new { text = LblClickCount?.Text ?? "clicks=0" },+ ["txt-input"] = new { text = TxtInput?.Text ?? "" },+ ["txt-deferred-input"] = new { text = TxtDeferredInput?.Text ?? "" },+ ["txt-multiline-input"] = new { text = TxtMultilineInput?.Text ?? "" },};try{