Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
feat(cua-driver-rs)(platform-windows): MSAA fallback for SAL/VCL windows (#1709)
Francesco Bonacci02f1f033e926
diff --git a/libs/cua-driver/rust/crates/cua-driver/tests/harness_lo_vcl_test.rs b/libs/cua-driver/rust/crates/cua-driver/tests/harness_lo_vcl_test.rsnew file mode 100644index 0000000000..ccb0da20cb--- /dev/null+++ b/libs/cua-driver/rust/crates/cua-driver/tests/harness_lo_vcl_test.rs@@ -0,0 +1,868 @@+//! Integration tests against a live LibreOffice Writer instance.+//!+//! Regression-guards the VCL/SAL gaps the vision-only LO Writer flow+//! surfaced (manually validated during the PR #1708 follow-up+//! exploration). Each test asserts the *current* gap — it will fail+//! loudly if cua-driver later closes the underlying gap, prompting+//! a flip of the assertion.+//!+//! ## Tests in this file+//!+//! 1. **`harness_lo_vcl_font_color_split_button_exposes_expand`** —+//! Font Color SplitButton reports `actions=[invoke,expand]`.+//!+//! 2. **`harness_lo_vcl_font_color_expand_opens_picker`** — click+//! with `action:"expand"` opens a SALTMPSUBFRAME picker.+//!+//! 3. **`harness_lo_vcl_modal_input_roundtrip_works`** — SAL/VCL+//! modal (Find & Replace) accepts SendInput, snapshot has actionable+//! elements (no more skip stub).+//!+//! 4. **`harness_lo_vcl_all_toolbar_split_buttons_expose_expand`** —+//! *every* toolbar SplitButton in Writer has `expand` (not just+//! Font Color). Guards `msaa::actions_for` BUTTONDROPDOWN family.+//!+//! 5. **`harness_lo_vcl_color_pick_green_end_to_end`** — full+//! workflow: type text → select → open Font Color → click Green →+//! picker closes (proves color landed).+//!+//! 6. **`harness_lo_vcl_recovery_dialog_walks_via_msaa`** — Document+//! Recovery dialog (SALFRAME class) still walks via MSAA, exposes+//! "Discard All" + "Recover Selected" buttons. Guards the+//! routing change for the pre-existing Recovery-dialog flow.+//!+//! 7. **`harness_lo_vcl_calc_msaa_smoke`** — LO Calc (different app+//! surface on the same VCL base) also walks via MSAA with ≥15+//! SplitButtons exposing `expand`. Confirms the path generalizes+//! beyond Writer.+//!+//! ## How to run+//!+//! Local (requires LibreOffice installed):+//! cargo test --test harness_lo_vcl_test -- --ignored --nocapture+//!+//! Tests skip cleanly if `swriter.exe` isn't on disk at one of the+//! standard install locations (override via `LO_SWRITER_EXE`).++#![cfg(target_os = "windows")]++use std::io::{BufRead, BufReader, Write};+use std::path::PathBuf;+use std::process::{Child, ChildStdin, ChildStdout, Command, Stdio};+use std::time::Duration;++// ── workspace / LO paths ─────────────────────────────────────────────────────++fn workspace_root() -> PathBuf {+ let manifest = std::env::var("CARGO_MANIFEST_DIR").expect("CARGO_MANIFEST_DIR");+ PathBuf::from(manifest).parent().unwrap().parent().unwrap().to_owned()+}+fn driver_binary() -> PathBuf { workspace_root().join("target/debug/cua-driver.exe") }++/// LO Writer executable. Honour `LO_SWRITER_EXE` env override (for CI+/// images with non-default install paths), otherwise probe the two+/// standard locations.+fn swriter_exe() -> Option<PathBuf> {+ if let Ok(p) = std::env::var("LO_SWRITER_EXE") {+ let pb = PathBuf::from(p);+ if pb.exists() { return Some(pb); }+ }+ for candidate in [+ r"C:\Program Files\LibreOffice\program\swriter.exe",+ r"C:\Program Files (x86)\LibreOffice\program\swriter.exe",+ ] {+ let pb = PathBuf::from(candidate);+ if pb.exists() { return Some(pb); }+ }+ None+}++/// LO Calc executable, same probe shape as `swriter_exe()`.+fn scalc_exe() -> Option<PathBuf> {+ if let Ok(p) = std::env::var("LO_SCALC_EXE") {+ let pb = PathBuf::from(p);+ if pb.exists() { return Some(pb); }+ }+ for candidate in [+ r"C:\Program Files\LibreOffice\program\scalc.exe",+ r"C:\Program Files (x86)\LibreOffice\program\scalc.exe",+ ] {+ let pb = PathBuf::from(candidate);+ if pb.exists() { return Some(pb); }+ }+ None+}++// ── JSON-RPC plumbing ────────────────────────────────────────────────────────++fn send(stdin: &mut ChildStdin, req: serde_json::Value) {+ writeln!(stdin, "{}", serde_json::to_string(&req).unwrap()).unwrap();+}+fn recv(stdout: &mut BufReader<&mut ChildStdout>) -> serde_json::Value {+ let mut line = String::new();+ stdout.read_line(&mut line).expect("read");+ serde_json::from_str(line.trim()).expect("json")+}+fn init(stdin: &mut ChildStdin, stdout: &mut BufReader<&mut ChildStdout>) {+ send(stdin, serde_json::json!({"jsonrpc":"2.0","id":1,"method":"initialize","params":{}}));+ let _ = recv(stdout);+}+fn call(stdin: &mut ChildStdin, stdout: &mut BufReader<&mut ChildStdout>,+ id: u32, name: &str, args: serde_json::Value) -> serde_json::Value {+ send(stdin, serde_json::json!({+ "jsonrpc":"2.0","id":id,"method":"tools/call",+ "params":{"name":name,"arguments":args}+ }));+ recv(stdout)+}++fn snapshot_text(s: &serde_json::Value) -> &str {+ s["result"]["content"][0]["text"].as_str().unwrap_or("")+}++// ── fixture ──────────────────────────────────────────────────────────────────++struct LoSession {+ _writer: Child,+ driver: Child,+ driver_stdin: ChildStdin,+ driver_stdout: ChildStdout,+ writer_pid: u32,+ writer_wid: u64,+}++impl Drop for LoSession {+ fn drop(&mut self) {+ // Try clean shutdown via cua-driver kill_app first so LO doesn't+ // leak a soffice.bin background daemon across test runs.+ let _ = self.driver.kill(); let _ = self.driver.wait();+ let _ = self._writer.kill(); let _ = self._writer.wait();+ // SAL/VCL soffice.bin daemon hangs around after the parent exits;+ // best-effort sweep via taskkill so the next test starts clean.+ let _ = Command::new("taskkill").args(["/F", "/IM", "soffice.bin"]).output();+ std::thread::sleep(Duration::from_millis(800));+ }+}++fn setup() -> Option<LoSession> {+ let driver = driver_binary();+ if !driver.exists() {+ eprintln!("cua-driver.exe not built — run `cargo build` first"); return None;+ }+ let writer = match swriter_exe() {+ Some(p) => p,+ None => {+ eprintln!("LibreOffice swriter.exe not found — set LO_SWRITER_EXE or install LO");+ return None;+ }+ };++ // Sweep any leftover soffice.bin from a prior run so this test gets+ // a fresh writer pid + clean recovery state.+ let _ = Command::new("taskkill").args(["/F", "/IM", "soffice.bin"]).output();+ std::thread::sleep(Duration::from_millis(800));++ // `-norestore` skips the Document Recovery dialog (which would+ // block the Writer top-level window from ever appearing if the+ // previous LO session crashed). `-nologo` skips the splash screen.+ let _writer = Command::new(&writer)+ .args(["-norestore", "-nologo"])+ .stdout(Stdio::null()).stderr(Stdio::null())+ .spawn().ok()?;+ // LO launches via soffice.bin background daemon, then forks the+ // actual writer process. The pid we just spawned is the launcher;+ // the real writer is a child. Poll list_apps via cua-driver to+ // find it.+ std::thread::sleep(Duration::from_secs(3));++ let mut driver_c = Command::new(&driver)+ .stdin(Stdio::piped()).stdout(Stdio::piped()).stderr(Stdio::null())+ .spawn().ok()?;+ let mut driver_stdin = driver_c.stdin.take().unwrap();+ let mut driver_stdout = driver_c.stdout.take().unwrap();+ {+ let mut stdout = BufReader::new(&mut driver_stdout);+ init(&mut driver_stdin, &mut stdout);+ // Find the LO Writer window by title — its owning pid is the+ // soffice.bin daemon (NOT the launcher pid above). Poll up to+ // 30 s for the recovery dialog (if any) + main window to appear.+ let deadline = std::time::Instant::now() + Duration::from_secs(30);+ loop {+ let resp = call(&mut driver_stdin, &mut stdout, 10, "list_windows", serde_json::json!({}));+ if let Some(wins) = resp["result"]["structuredContent"]["windows"].as_array() {+ for w in wins {+ let title = w["title"].as_str().unwrap_or("");+ let app = w["app_name"].as_str().unwrap_or("");+ if app.contains("soffice") && title.contains("LibreOffice Writer") {+ let pid = w["pid"].as_u64().unwrap_or(0) as u32;+ let wid = w["window_id"].as_u64().unwrap_or(0);+ if pid > 0 && wid > 0 {+ drop(stdout);+ return Some(LoSession {+ _writer, driver: driver_c, driver_stdin, driver_stdout,+ writer_pid: pid, writer_wid: wid,+ });+ }+ }+ }+ }+ if std::time::Instant::now() > deadline {+ eprintln!("Writer window did not appear within 30 s");+ drop(stdout);+ let _ = driver_c.kill();+ return None;+ }+ std::thread::sleep(Duration::from_millis(500));+ }+ }+}++fn with_session<F, R>(fx: &mut LoSession, f: F) -> R+where F: FnOnce(&mut ChildStdin, &mut BufReader<&mut ChildStdout>, u32, u64) -> R {+ let mut stdout = BufReader::new(&mut fx.driver_stdout);+ f(&mut fx.driver_stdin, &mut stdout, fx.writer_pid, fx.writer_wid)+}++// ── Test 1: Font Color SplitButton exposes `expand` via MSAA fallback ───────++/// Confirms the MSAA fallback (`platform-windows/src/msaa.rs`) lets+/// cua-driver see VCL toolbar SplitButtons' dropdown halves.+///+/// Before the MSAA fallback landed: UIA's MSAA→UIA proxy collapsed+/// `ROLE_SYSTEM_BUTTONDROPDOWN` (0x38) to a featureless `SplitButton`+/// with `actions=[invoke]` — agents could re-fire the press half but+/// not open the picker. After the MSAA fallback: SAL-class windows+/// walk via oleacc's `AccessibleObjectFromWindow`, which preserves+/// the BUTTONDROPDOWN role, and cua-driver maps it to+/// `actions=[invoke,expand]`. The `click` tool's `action:"expand"`+/// case clicks the right-edge of the cached rect (the dropdown arrow+/// half) via SendInput.+///+/// If this fails: either the MSAA path stopped applying to SALFRAME+/// (check `uia/mod.rs` SAL class detection), or LO changed its+/// toolbar role (no longer ROLE_SYSTEM_BUTTONDROPDOWN).+#[test]+#[ignore]+fn harness_lo_vcl_font_color_split_button_exposes_expand() {+ let mut fx = match setup() { Some(s) => s, None => return };+ with_session(&mut fx, |stdin, stdout, pid, wid| {+ let snap = call(stdin, stdout, 30, "get_window_state", serde_json::json!({+ "pid": pid as i64, "window_id": wid,+ "capture_mode": "ax", "query": "Font Color"+ }));+ let text = snapshot_text(&snap);+ assert!(text.contains("SplitButton \"Font Color\""),+ "Font Color SplitButton not found in tree: {text:?}");+ let line = text.lines().find(|l| l.contains("\"Font Color\"")).unwrap_or("");+ assert!(line.contains("expand"),+ "Expected `expand` in actions on Font Color SplitButton — got {line:?}. \+ The MSAA fallback may not be running for SALFRAME (check uia/mod.rs \+ SAL class detection — should route ALL SAL* classes through msaa.rs).");+ assert!(line.contains("invoke"),+ "Expected `invoke` to remain in actions alongside `expand` — got {line:?}. \+ MSAA walker should expose both press and dropdown halves.");+ });+}++// ── Test 2: action:"expand" actually opens the color picker ─────────────────++/// End-to-end test that the `click(element_index, action:"expand")`+/// dispatch on a MSAA BUTTONDROPDOWN actually opens the dropdown.+///+/// Asserts:+/// 1. `get_window_state` finds Font Color by name and yields an+/// element_index.+/// 2. `click(element_index=X, action:"expand")` returns success.+/// 3. A new top-level window appears under the LO pid with title+/// "Font Color" (the SALTMPSUBFRAME color picker).+///+/// Failures point at:+/// - (1) MSAA walker not finding Font Color (check msaa.rs walker+/// budget / depth, or LO renamed the button).+/// - (2) cua-driver click tool's MSAA dispatch broke (check+/// `tools/impl_.rs` BUTTONDROPDOWN branch).+/// - (3) Right-edge offset wrong for current LO version's toolbar+/// scale (check `rect.right - 4` heuristic in+/// `tools/impl_.rs`).+#[test]+#[ignore]+fn harness_lo_vcl_font_color_expand_opens_picker() {+ let mut fx = match setup() { Some(s) => s, None => return };+ with_session(&mut fx, |stdin, stdout, pid, wid| {+ // Bring Writer to foreground so SendInput click lands on it.+ let _ = call(stdin, stdout, 30, "bring_to_front", serde_json::json!({+ "pid": pid as i64, "window_id": wid+ }));+ std::thread::sleep(Duration::from_millis(400));++ let snap = call(stdin, stdout, 31, "get_window_state", serde_json::json!({+ "pid": pid as i64, "window_id": wid,+ "capture_mode": "ax", "query": "Font Color"+ }));+ let text = snapshot_text(&snap);+ let line = text.lines()+ .find(|l| l.contains("\"Font Color\"") && l.contains("expand"))+ .unwrap_or_else(|| panic!("Font Color SplitButton with `expand` action not found: {text:?}"));+ let s = line.find('[').expect("element_index bracket open");+ let e = line[s..].find(']').expect("element_index bracket close") + s;+ let idx: u64 = line[s+1..e].trim().parse()+ .unwrap_or_else(|_| panic!("could not parse element_index from line {line:?}"));++ // Snapshot windows under our pid BEFORE the click.+ let before = call(stdin, stdout, 32, "list_windows", serde_json::json!({+ "pid": pid as i64+ }));+ let before_ids: std::collections::HashSet<u64> = before+ ["result"]["structuredContent"]["windows"].as_array()+ .map(|a| a.iter().filter_map(|w| w["window_id"].as_u64()).collect())+ .unwrap_or_default();++ let resp = call(stdin, stdout, 33, "click", serde_json::json!({+ "pid": pid as i64, "window_id": wid,+ "element_index": idx, "action": "expand"+ }));+ let resp_text = resp["result"]["content"][0]["text"].as_str().unwrap_or("");+ assert!(resp_text.starts_with("✅"),+ "click(action:expand) failed: {resp_text:?}");+ assert!(resp_text.contains("dropdown half"),+ "Expected response to mention dropdown half — got {resp_text:?}. \+ The MSAA dispatch path may not have triggered.");++ // Let the picker spawn.+ std::thread::sleep(Duration::from_millis(900));++ let after = call(stdin, stdout, 34, "list_windows", serde_json::json!({+ "pid": pid as i64+ }));+ let new_wins: Vec<&serde_json::Value> = after+ ["result"]["structuredContent"]["windows"].as_array()+ .map(|a| a.iter()+ .filter(|w| {+ let id = w["window_id"].as_u64();+ id.map(|i| !before_ids.contains(&i)).unwrap_or(false)+ })+ .collect())+ .unwrap_or_default();+ let picker = new_wins.iter().find(|w| {+ w["title"].as_str().map(|t| t.contains("Font Color")).unwrap_or(false)+ });+ assert!(picker.is_some(),+ "No new window titled 'Font Color' appeared after click(action:expand). \+ new_windows={new_wins:?}. The right-edge dispatch may have hit the \+ wrong pixel (LO's toolbar scaled differently?) or the picker spawned \+ as a child window instead of a top-level.");+ });+}++// ── Working path: SALSUBFRAME modal dialogs accept SendInput input ─────────++/// Confirms the MSAA fallback gives a walkable tree on SALSUBFRAME+/// modal dialogs too (was the "SAL/VCL target, UIA walk skipped" stub+/// before the MSAA path landed), AND that the dialog accepts+/// SendInput-injected keyboard input.+///+/// Specifically:+/// - `hotkey(ctrl+h, foreground)` against the main Writer SALFRAME+/// opens the Find & Replace dialog (a SALSUBFRAME).+/// - `get_window_state(capture_mode:"ax")` on the dialog returns a+/// full element tree via the MSAA walker — includes the "Close"+/// button and the Find/Replace edit fields as addressable+/// element_indices (a recent capability — the pre-MSAA stub had+/// zero actionable elements).+/// - `press_key(escape, foreground)` against the SALSUBFRAME closes+/// it cleanly.+///+/// If this test ever fails, one of three things regressed:+/// (a) Ctrl+H accelerator on the main Writer window stopped firing+/// under SendInput (foreground key dispatch broken on SALFRAME).+/// (b) MSAA walker stopped finding actionable elements in+/// SALSUBFRAME (check `msaa.rs` budget / depth).+/// (c) Foreground Escape stopped reaching the SALSUBFRAME (SAL+/// filter changed in a newer LO).+#[test]+#[ignore]+fn harness_lo_vcl_modal_input_roundtrip_works() {+ let mut fx = match setup() { Some(s) => s, None => return };+ with_session(&mut fx, |stdin, stdout, pid, wid| {+ std::thread::sleep(Duration::from_millis(800));++ let open_resp = call(stdin, stdout, 30, "hotkey", serde_json::json!({+ "pid": pid as i64, "window_id": wid,+ "keys": ["ctrl", "h"], "dispatch": "foreground"+ }));+ let open_text = open_resp["result"]["content"][0]["text"].as_str().unwrap_or("");+ assert!(open_text.starts_with("✅"),+ "hotkey(ctrl+h, foreground) failed: {open_text:?}");++ let dialog_wid = {+ let deadline = std::time::Instant::now() + Duration::from_secs(8);+ loop {+ let wins = call(stdin, stdout, 31, "list_windows", serde_json::json!({+ "pid": pid as i64+ }));+ let found = wins["result"]["structuredContent"]["windows"].as_array()+ .and_then(|a| a.iter().find_map(|w| {+ let title = w["title"].as_str().unwrap_or("");+ let id = w["window_id"].as_u64();+ if id != Some(wid)+ && (title.contains("Find") || title.contains("Replace"))+ { id } else { None }+ }));+ if let Some(id) = found { break id; }+ if std::time::Instant::now() > deadline {+ panic!("Find & Replace dialog did not appear after Ctrl+H — \+ Writer accelerator may have regressed under \+ foreground SendInput on SALFRAME.");+ }+ std::thread::sleep(Duration::from_millis(300));+ }+ };++ // Snapshot the dialog — MSAA walker should produce a real tree+ // with the dialog's buttons and edit fields, NOT the old skip+ // stub.+ let dialog_ax = call(stdin, stdout, 32, "get_window_state", serde_json::json!({+ "pid": pid as i64, "window_id": dialog_wid, "capture_mode": "ax"+ }));+ let dialog_text = snapshot_text(&dialog_ax);+ assert!(!dialog_text.contains("SAL/VCL target, UIA walk skipped"),+ "Got the old skip-stub message — MSAA fallback did not engage on \+ this SALSUBFRAME. Snapshot was: {dialog_text:?}");+ assert!(dialog_text.contains("Button \"Close\""),+ "Expected MSAA walker to expose a 'Close' Button in the Find & Replace \+ dialog tree (it's a documented child via accChild). Snapshot was: {dialog_text:?}");+ // Sanity: should have several actionable element_indices, not zero.+ let actionable_count = dialog_text.lines()+ .filter(|l| l.contains("actions=[invoke"))+ .count();+ assert!(actionable_count >= 4,+ "Expected ≥4 actionable elements in Find & Replace via MSAA walker, \+ got {actionable_count}. Tree: {dialog_text:?}");++ // Foreground Escape should close the dialog.+ let esc_resp = call(stdin, stdout, 33, "press_key", serde_json::json!({+ "pid": pid as i64, "window_id": dialog_wid,+ "key": "escape", "dispatch": "foreground"+ }));+ let esc_text = esc_resp["result"]["content"][0]["text"].as_str().unwrap_or("");+ assert!(esc_text.starts_with("✅"),+ "press_key(escape, foreground) returned an error: {esc_text:?}");+ std::thread::sleep(Duration::from_millis(700));++ let wins_after = call(stdin, stdout, 34, "list_windows", serde_json::json!({+ "pid": pid as i64+ }));+ let still_open = wins_after["result"]["structuredContent"]["windows"].as_array()+ .map(|a| a.iter().any(|w| w["window_id"].as_u64() == Some(dialog_wid)))+ .unwrap_or(false);+ assert!(!still_open,+ "Find & Replace dialog stayed open after foreground Escape — \+ SAL modal input dispatch regressed. Check uia/mod.rs SAL-class \+ handling and the press_key SendInput path.");++ // Cleanup: kill the LO process — Drop impl sweeps soffice.bin.+ let _ = call(stdin, stdout, 35, "kill_app", serde_json::json!({+ "pid": pid as i64+ }));+ });+}++// ── Test 4: All toolbar SplitButtons expose `expand` ────────────────────────++/// Confirms the MSAA fallback exposes `actions=[invoke,expand]` on EVERY+/// toolbar SplitButton in LO Writer — not just Font Color.+///+/// Pre-MSAA, the UIA→MSAA proxy collapsed every BUTTONDROPDOWN to a+/// featureless `SplitButton actions=[invoke]`. Post-MSAA, ALL of them+/// (Save, Open, Undo, Redo, Paste, Table, Field, Symbol, Show Tracked+/// Changes, Record, Basic Shapes, Underline, Font Color, Character+/// Highlighting Color, Background Color, Unordered List, Ordered List,+/// Outline Format, Line Spacing, Character Spacing — plus the "System"+/// menu button) report `expand`. Guards against accidental regressions+/// in `msaa::actions_for` (e.g. dropping one of the BUTTONDROPDOWN+/// variants from the role match).+///+/// Asserts ≥15 SplitButtons report `expand` (current count is 20-21+/// depending on toolbar config; floor at 15 to absorb minor LO updates).+#[test]+#[ignore]+fn harness_lo_vcl_all_toolbar_split_buttons_expose_expand() {+ let mut fx = match setup() { Some(s) => s, None => return };+ with_session(&mut fx, |stdin, stdout, pid, wid| {+ let snap = call(stdin, stdout, 30, "get_window_state", serde_json::json!({+ "pid": pid as i64, "window_id": wid, "capture_mode": "ax"+ }));+ let text = snapshot_text(&snap);+ let splitbutton_lines: Vec<&str> = text+ .lines()+ .filter(|l| l.contains("SplitButton") && l.contains("actions=["))+ .collect();+ let with_expand: Vec<&&str> = splitbutton_lines.iter()+ .filter(|l| l.contains("expand"))+ .collect();+ assert!(splitbutton_lines.len() >= 15,+ "Expected ≥15 SplitButtons in LO Writer toolbar tree, got {}. \+ Possible MSAA walker regression. Full snapshot first 1500 chars: {}",+ splitbutton_lines.len(),+ &text.chars().take(1500).collect::<String>());+ assert_eq!(with_expand.len(), splitbutton_lines.len(),+ "{}/{} SplitButtons report `expand`. The MSAA role→actions \+ mapping may have dropped one of BUTTONDROPDOWN / BUTTONMENU / \+ BUTTONDROPDOWNGRID / SPLITBUTTON. Missing-expand lines:\n{}",+ with_expand.len(), splitbutton_lines.len(),+ splitbutton_lines.iter()+ .filter(|l| !l.contains("expand"))+ .map(|l| l.trim())+ .collect::<Vec<_>>()+ .join("\n"));+ });+}++// ── Test 5: End-to-end color-pick ───────────────────────────────────────────++/// End-to-end regression for the headline workflow: type text, select it,+/// open the Font Color picker via `action:"expand"`, click a named color,+/// verify the picker closes (the canonical signal that LO applied the+/// color).+///+/// Failure points:+/// - MSAA walker can't find Font Color → `msaa.rs` role match broken.+/// - click(action:"expand") doesn't open picker → right-edge dispatch+/// offset wrong for current LO version's toolbar scale.+/// - Picker MSAA walk doesn't find "Green" → walker depth/budget+/// truncating the color grid.+/// - Picker doesn't close after picking → click on color cell isn't+/// landing (offset / role mismatch on ListItem).+#[test]+#[ignore]+fn harness_lo_vcl_color_pick_green_end_to_end() {+ let mut fx = match setup() { Some(s) => s, None => return };+ with_session(&mut fx, |stdin, stdout, pid, wid| {+ // Foreground first so type_text + Ctrl+A land on Writer.+ let _ = call(stdin, stdout, 30, "bring_to_front", serde_json::json!({+ "pid": pid as i64, "window_id": wid+ }));+ std::thread::sleep(Duration::from_millis(400));++ let _ = call(stdin, stdout, 31, "type_text", serde_json::json!({+ "pid": pid as i64, "window_id": wid,+ "text": "color pick regression"+ }));+ std::thread::sleep(Duration::from_millis(400));++ let _ = call(stdin, stdout, 32, "hotkey", serde_json::json!({+ "pid": pid as i64, "window_id": wid,+ "keys": ["ctrl", "a"], "dispatch": "foreground"+ }));+ std::thread::sleep(Duration::from_millis(400));++ // Locate Font Color element_index in Writer's MSAA tree.+ let snap = call(stdin, stdout, 33, "get_window_state", serde_json::json!({+ "pid": pid as i64, "window_id": wid,+ "capture_mode": "ax", "query": "Font Color"+ }));+ let snap_text = snapshot_text(&snap);+ let fc_line = snap_text.lines()+ .find(|l| l.contains("\"Font Color\"") && l.contains("expand"))+ .unwrap_or_else(|| panic!("Font Color with `expand` not found in tree: {snap_text:?}"));+ let s = fc_line.find('[').expect("[ in fc_line");+ let e = fc_line[s..].find(']').expect("] in fc_line") + s;+ let fc_idx: u64 = fc_line[s+1..e].trim().parse().expect("fc element_index");++ // Snapshot which windows existed BEFORE opening the picker so+ // we can isolate the picker on appearance.+ let before = call(stdin, stdout, 34, "list_windows", serde_json::json!({+ "pid": pid as i64+ }));+ let before_ids: std::collections::HashSet<u64> = before+ ["result"]["structuredContent"]["windows"].as_array()+ .map(|a| a.iter().filter_map(|w| w["window_id"].as_u64()).collect())+ .unwrap_or_default();++ // Open the dropdown.+ let open_resp = call(stdin, stdout, 35, "click", serde_json::json!({+ "pid": pid as i64, "window_id": wid,+ "element_index": fc_idx, "action": "expand"+ }));+ let open_text = open_resp["result"]["content"][0]["text"].as_str().unwrap_or("");+ assert!(open_text.contains("dropdown half"),+ "click(action:expand) did not dispatch via MSAA dropdown path: {open_text:?}");+ std::thread::sleep(Duration::from_millis(900));++ // Find the picker window.+ let after = call(stdin, stdout, 36, "list_windows", serde_json::json!({+ "pid": pid as i64+ }));+ let picker_wid = after["result"]["structuredContent"]["windows"]+ .as_array()+ .and_then(|a| a.iter().find_map(|w| {+ let id = w["window_id"].as_u64();+ let title = w["title"].as_str().unwrap_or("");+ if id.map(|i| !before_ids.contains(&i)).unwrap_or(false)+ && title.contains("Font Color")+ { id } else { None }+ }))+ .unwrap_or_else(|| panic!("No new 'Font Color' picker window appeared"));++ // Walk the picker tree to find a green color cell.+ let psnap = call(stdin, stdout, 37, "get_window_state", serde_json::json!({+ "pid": pid as i64, "window_id": picker_wid, "capture_mode": "ax"+ }));+ let psnap_text = snapshot_text(&psnap);+ let green_line = psnap_text.lines()+ .find(|l| l.contains("\"Green\"") && l.contains("[") && l.contains("actions=[invoke"))+ .unwrap_or_else(|| panic!(+ "No 'Green' ListItem in picker tree. The MSAA walker may have \+ truncated the color grid or LO's locale renamed the color. \+ First 1500 chars: {}",+ &psnap_text.chars().take(1500).collect::<String>()));+ let s = green_line.find('[').expect("[ in green");+ let e = green_line[s..].find(']').expect("] in green") + s;+ let green_idx: u64 = green_line[s+1..e].trim().parse().expect("green element_index");++ // Pick it. Default action (invoke) clicks center via MSAA dispatch.+ let pick_resp = call(stdin, stdout, 38, "click", serde_json::json!({+ "pid": pid as i64, "window_id": picker_wid,+ "element_index": green_idx+ }));+ let pick_text = pick_resp["result"]["content"][0]["text"].as_str().unwrap_or("");+ assert!(pick_text.starts_with("✅"),+ "click on Green color cell failed: {pick_text:?}");+ std::thread::sleep(Duration::from_millis(700));++ // Verify the picker closed (LO closes the popup after a color is+ // selected — the canonical "color applied" signal). If it+ // didn't, the click didn't actually land on the cell.+ let after2 = call(stdin, stdout, 39, "list_windows", serde_json::json!({+ "pid": pid as i64+ }));+ let still_open = after2["result"]["structuredContent"]["windows"]+ .as_array()+ .map(|a| a.iter().any(|w| w["window_id"].as_u64() == Some(picker_wid)))+ .unwrap_or(false);+ assert!(!still_open,+ "Picker stayed open after click on Green — the cell click \+ didn't land. LO didn't apply the color.");+ });+}++// ── Test 6: Recovery dialog walks via MSAA ──────────────────────────────────++/// Confirms that the LibreOffice Document Recovery dialog (which used to+/// be walked via UIA) is still walkable after the MSAA-routes-all-SAL+/// change. Pre-MSAA: UIA path returned a walkable tree with "Discard+/// All" / "Recover Selected" buttons addressable. Post-MSAA: same flow+/// must continue to work through the MSAA walker.+///+/// Triggering Recovery requires a previous LO session that exited+/// ungracefully — this test creates that condition by typing into a+/// fresh writer and then `kill_app`-ing it, then re-launches. If the+/// Recovery dialog doesn't appear (LO can't always be coaxed into+/// recovery state, e.g. when the recovery feature is disabled in+/// user config), the test logs and returns OK rather than panicking —+/// it's a regression guard for the post-MSAA Recovery walk, not for+/// LO's recovery-trigger behavior.+#[test]+#[ignore]+fn harness_lo_vcl_recovery_dialog_walks_via_msaa() {+ // First: launch Writer WITHOUT -norestore (we WANT recovery+ // behavior), type something, then force-kill so LO marks the+ // session as crashed.+ let writer = match swriter_exe() { Some(p) => p, None => return };+ let _ = Command::new("taskkill").args(["/F", "/IM", "soffice.bin"]).output();+ std::thread::sleep(Duration::from_millis(800));++ // Launch (no -norestore, no -nologo flag → default behavior).+ let mut crash_proc = match Command::new(&writer)+ .stdout(Stdio::null()).stderr(Stdio::null())+ .spawn() {+ Ok(p) => p,+ Err(_) => return,+ };+ std::thread::sleep(Duration::from_secs(5));+ let _ = crash_proc.kill();+ let _ = crash_proc.wait();+ // Force-kill the actual soffice.bin worker too (the launcher is what+ // we just killed; the real LO process is a fork).+ let _ = Command::new("taskkill").args(["/F", "/IM", "soffice.bin"]).output();+ std::thread::sleep(Duration::from_secs(2));++ // Re-launch Writer — Recovery dialog should appear.+ let driver = driver_binary();+ if !driver.exists() { return; }+ let mut writer_proc = match Command::new(&writer)+ .stdout(Stdio::null()).stderr(Stdio::null())+ .spawn() {+ Ok(p) => p,+ Err(_) => return,+ };+ std::thread::sleep(Duration::from_secs(5));++ let mut driver_c = match Command::new(&driver)+ .stdin(Stdio::piped()).stdout(Stdio::piped()).stderr(Stdio::null())+ .spawn() {+ Ok(p) => p,+ Err(_) => {+ let _ = writer_proc.kill();+ return;+ }+ };+ let mut stdin = driver_c.stdin.take().unwrap();+ let mut stdout_raw = driver_c.stdout.take().unwrap();+ let mut stdout = BufReader::new(&mut stdout_raw);++ init(&mut stdin, &mut stdout);++ // Poll for the Recovery dialog.+ let recovery_wid_pid: Option<(u64, u32)> = {+ let deadline = std::time::Instant::now() + Duration::from_secs(20);+ let mut found = None;+ while std::time::Instant::now() < deadline {+ let resp = call(&mut stdin, &mut stdout, 1, "list_windows", serde_json::json!({}));+ if let Some(wins) = resp["result"]["structuredContent"]["windows"].as_array() {+ if let Some(w) = wins.iter().find(|w| {+ w["title"].as_str().map(|t| t.contains("Recovery")).unwrap_or(false)+ }) {+ let id = w["window_id"].as_u64();+ let pid = w["pid"].as_u64().map(|p| p as u32);+ if let (Some(i), Some(p)) = (id, pid) {+ found = Some((i, p));+ break;+ }+ }+ }+ std::thread::sleep(Duration::from_millis(500));+ }+ found+ };++ match recovery_wid_pid {+ None => {+ eprintln!("Recovery dialog did not appear — LO didn't enter recovery \+ state (possibly disabled in user config, or our crash \+ trigger didn't take effect). Test cannot assert MSAA \+ walk on a dialog that doesn't exist; logging and \+ returning OK.");+ }+ Some((rwid, rpid)) => {+ let snap = call(&mut stdin, &mut stdout, 2, "get_window_state",+ serde_json::json!({+ "pid": rpid as i64, "window_id": rwid, "capture_mode": "ax"+ }));+ let text = snapshot_text(&snap);+ assert!(!text.contains("SAL/VCL target, UIA walk skipped"),+ "Recovery dialog returned the old SAL-skip stub — MSAA fallback \+ isn't engaging on SALFRAME Recovery dialogs.");+ assert!(text.contains("Button \"Discard All\""),+ "Recovery dialog tree should expose `Discard All` Button via MSAA. \+ Got: {text:?}");+ assert!(text.contains("Button \"Recover Selected\""),+ "Recovery dialog tree should expose `Recover Selected` Button via MSAA. \+ Got: {text:?}");+ }+ }++ // Cleanup.+ let _ = driver_c.kill(); let _ = driver_c.wait();+ let _ = writer_proc.kill(); let _ = writer_proc.wait();+ let _ = Command::new("taskkill").args(["/F", "/IM", "soffice.bin"]).output();+ std::thread::sleep(Duration::from_millis(800));+}++// ── Test 7: LO Calc smoke ──────────────────────────────────────────────────++/// Confirms the MSAA fallback generalizes beyond Writer. Launches LO+/// Calc, walks the main SALFRAME, asserts ≥15 SplitButtons with `expand`+/// — Calc has Calc-specific dropdowns (Select Function, Row, Column,+/// Freeze Panes) on top of the standard ones (Save, Open, Undo, Redo,+/// Paste, Font Color, etc.).+///+/// Skips cleanly if scalc.exe isn't installed (set LO_SCALC_EXE for+/// non-default paths).+#[test]+#[ignore]+fn harness_lo_vcl_calc_msaa_smoke() {+ let scalc = match scalc_exe() { Some(p) => p, None => return };+ let driver = driver_binary();+ if !driver.exists() { return; }++ let _ = Command::new("taskkill").args(["/F", "/IM", "soffice.bin"]).output();+ std::thread::sleep(Duration::from_millis(800));++ let mut calc_proc = match Command::new(&scalc)+ .args(["-norestore", "-nologo"])+ .stdout(Stdio::null()).stderr(Stdio::null())+ .spawn() {+ Ok(p) => p,+ Err(_) => return,+ };+ std::thread::sleep(Duration::from_secs(5));++ let mut driver_c = match Command::new(&driver)+ .stdin(Stdio::piped()).stdout(Stdio::piped()).stderr(Stdio::null())+ .spawn() {+ Ok(p) => p,+ Err(_) => {+ let _ = calc_proc.kill();+ return;+ }+ };+ let mut stdin_drv = driver_c.stdin.take().unwrap();+ let mut stdout_raw = driver_c.stdout.take().unwrap();+ let mut stdout = BufReader::new(&mut stdout_raw);+ init(&mut stdin_drv, &mut stdout);++ // Poll for the Calc window.+ let calc_wid_pid: Option<(u64, u32)> = {+ let deadline = std::time::Instant::now() + Duration::from_secs(20);+ let mut found = None;+ while std::time::Instant::now() < deadline {+ let r = call(&mut stdin_drv, &mut stdout, 1, "list_windows", serde_json::json!({}));+ if let Some(wins) = r["result"]["structuredContent"]["windows"].as_array() {+ if let Some(w) = wins.iter().find(|w| {+ w["title"].as_str().map(|t| t.contains("LibreOffice Calc")).unwrap_or(false)+ }) {+ let id = w["window_id"].as_u64();+ let pid = w["pid"].as_u64().map(|p| p as u32);+ if let (Some(i), Some(p)) = (id, pid) {+ found = Some((i, p));+ break;+ }+ }+ }+ std::thread::sleep(Duration::from_millis(500));+ }+ found+ };++ let (cwid, cpid) = calc_wid_pid.unwrap_or_else(|| {+ let _ = driver_c.kill();+ let _ = calc_proc.kill();+ let _ = Command::new("taskkill").args(["/F", "/IM", "soffice.bin"]).output();+ panic!("LO Calc window did not appear within 20 s");+ });++ let snap = call(&mut stdin_drv, &mut stdout, 2, "get_window_state",+ serde_json::json!({+ "pid": cpid as i64, "window_id": cwid, "capture_mode": "ax"+ }));+ let text = snapshot_text(&snap);+ assert!(!text.contains("SAL/VCL target, UIA walk skipped"),+ "Calc returned the old SAL-skip stub — MSAA didn't engage.");+ let sb_with_expand = text.lines()+ .filter(|l| l.contains("SplitButton") && l.contains("expand"))+ .count();+ assert!(sb_with_expand >= 15,+ "Expected ≥15 SplitButtons with `expand` in Calc toolbar (proves MSAA \+ generalizes beyond Writer). Got {sb_with_expand}. The MSAA role \+ mapping may differ in Calc, or the toolbar config is unexpectedly slim.");++ // Cleanup.+ let _ = call(&mut stdin_drv, &mut stdout, 3, "kill_app", serde_json::json!({+ "pid": cpid as i64+ }));+ let _ = driver_c.kill(); let _ = driver_c.wait();+ let _ = calc_proc.kill(); let _ = calc_proc.wait();+ let _ = Command::new("taskkill").args(["/F", "/IM", "soffice.bin"]).output();+ std::thread::sleep(Duration::from_millis(800));+}diff --git a/libs/cua-driver/rust/crates/platform-windows/src/lib.rs b/libs/cua-driver/rust/crates/platform-windows/src/lib.rsindex e56c19188e..f918ea58cc 100644--- a/libs/cua-driver/rust/crates/platform-windows/src/lib.rs+++ b/libs/cua-driver/rust/crates/platform-windows/src/lib.rs@@ -21,6 +21,9 @@ pub mod win32;#[cfg(target_os = "windows")]pub mod uia;+#[cfg(target_os = "windows")]+pub mod msaa;+#[cfg(target_os = "windows")]pub mod input;diff --git a/libs/cua-driver/rust/crates/platform-windows/src/msaa.rs b/libs/cua-driver/rust/crates/platform-windows/src/msaa.rsnew file mode 100644index 0000000000..fb121310d7--- /dev/null+++ b/libs/cua-driver/rust/crates/platform-windows/src/msaa.rs@@ -0,0 +1,323 @@+//! MSAA (Microsoft Active Accessibility) tree walker.+//!+//! Fallback for SAL/VCL window classes (LibreOffice, OpenOffice) where the+//! UIA walker hangs on `BuildUpdatedCache(Subtree)`. MSAA via oleacc.dll's+//! `AccessibleObjectFromWindow` + recursive `accChild` walks these windows+//! cleanly because it doesn't go through the bulk-cache RPC that SAL's UIA+//! provider deadlocks on under the daemon's MTA pool.+//!+//! Bonus payoff: MSAA preserves the `ROLE_SYSTEM_BUTTONDROPDOWN` role (0x38)+//! that Windows' built-in MSAA→UIA proxy collapses to a featureless+//! `SplitButton` (no `ExpandCollapse` pattern, no separable dropdown child).+//! For BUTTONDROPDOWN, this walker emits `actions=["invoke","expand"]` —+//! the `click` tool routes `action:"expand"` to a right-edge SendInput+//! click that opens the dropdown half (e.g. LO Writer "Font Color" → color+//! picker) instead of just re-firing the press half.+//!+//! Discovery context: `flash-repro/ia2_pixel_click.ps1` proved the path —+//! `accLocation("Font Color")` → click `(rect.right - 4, center_y)` →+//! `SALTMPSUBFRAME 'Font Color'` popup appears.++use std::ptr::null_mut;++use windows::core::{Interface, VARIANT};+use windows::Win32::Foundation::HWND;+use windows::Win32::UI::Accessibility::{AccessibleObjectFromWindow, IAccessible};++use crate::uia::{UiaNode, UiaTreeResult};++const OBJID_CLIENT: u32 = 0xFFFFFFFC;++// MSAA role codes (subset; full list in oleacc.h).+const ROLE_SYSTEM_TITLEBAR: i32 = 0x01;+const ROLE_SYSTEM_MENUBAR: i32 = 0x02;+const ROLE_SYSTEM_SCROLLBAR: i32 = 0x03;+const ROLE_SYSTEM_WINDOW: i32 = 0x09;+const ROLE_SYSTEM_CLIENT: i32 = 0x0A;+const ROLE_SYSTEM_MENUPOPUP: i32 = 0x0B;+const ROLE_SYSTEM_MENUITEM: i32 = 0x0C;+const ROLE_SYSTEM_TOOLTIP: i32 = 0x0D;+const ROLE_SYSTEM_DIALOG: i32 = 0x12;+const ROLE_SYSTEM_GROUPING: i32 = 0x14;+const ROLE_SYSTEM_TOOLBAR: i32 = 0x16;+const ROLE_SYSTEM_STATUSBAR: i32 = 0x17;+const ROLE_SYSTEM_LINK: i32 = 0x1E;+const ROLE_SYSTEM_LIST: i32 = 0x21;+const ROLE_SYSTEM_LISTITEM: i32 = 0x22;+const ROLE_SYSTEM_PAGETAB: i32 = 0x25;+const ROLE_SYSTEM_GRAPHIC: i32 = 0x28;+const ROLE_SYSTEM_STATICTEXT: i32 = 0x29;+const ROLE_SYSTEM_TEXT: i32 = 0x2A;+const ROLE_SYSTEM_PUSHBUTTON: i32 = 0x2B;+const ROLE_SYSTEM_CHECKBUTTON: i32 = 0x2C;+const ROLE_SYSTEM_RADIOBUTTON: i32 = 0x2D;+const ROLE_SYSTEM_COMBOBOX: i32 = 0x2E;+const ROLE_SYSTEM_PROGRESSBAR: i32 = 0x30;+const ROLE_SYSTEM_SLIDER: i32 = 0x33;+const ROLE_SYSTEM_BUTTONDROPDOWN: i32 = 0x38;+const ROLE_SYSTEM_BUTTONMENU: i32 = 0x39;+const ROLE_SYSTEM_BUTTONDROPDOWNGRID: i32 = 0x3A;+const ROLE_SYSTEM_PAGETABLIST: i32 = 0x3C;+const ROLE_SYSTEM_SPLITBUTTON: i32 = 0x3E;++const MAX_DEPTH: usize = 25;+const MAX_TOTAL_ELEMENTS: usize = 5000;++/// Walk the MSAA tree for the window with the given HWND. Used as fallback+/// for SAL/VCL targets where the UIA walker would hang.+pub fn walk_msaa_tree(hwnd: u64) -> UiaTreeResult {+ unsafe { walk_unsafe(hwnd) }+}++unsafe fn walk_unsafe(hwnd: u64) -> UiaTreeResult {+ let hwnd_win = HWND(hwnd as *mut _);+ let mut raw_root: *mut std::ffi::c_void = null_mut();+ // AccessibleObjectFromWindow returns IAccessible (as `*mut c_void` typed+ // via the IID we pass). We pass IID_IAccessible.+ let iid_iaccessible = IAccessible::IID;+ let hr = AccessibleObjectFromWindow(+ hwnd_win,+ OBJID_CLIENT,+ &iid_iaccessible,+ &mut raw_root as *mut _ as *mut _,+ );+ if hr.is_err() || raw_root.is_null() {+ return UiaTreeResult {+ tree_markdown: format!(+ "- Window <SAL/VCL — MSAA fallback failed (AccessibleObjectFromWindow hr={hr:?})>\n"+ ),+ nodes: Vec::new(),+ };+ }+ let root: IAccessible = IAccessible::from_raw(raw_root);++ let mut nodes: Vec<UiaNode> = Vec::new();+ let mut lines: Vec<(usize, String)> = Vec::new();+ let mut counter = 0usize;+ let mut total = 0usize;++ walk(&root, 0, &mut nodes, &mut lines, &mut counter, &mut total);++ let tree_markdown = render_lines(&lines);+ UiaTreeResult { tree_markdown, nodes }+}++unsafe fn walk(+ acc: &IAccessible,+ depth: usize,+ nodes: &mut Vec<UiaNode>,+ lines: &mut Vec<(usize, String)>,+ counter: &mut usize,+ total: &mut usize,+) {+ if depth >= MAX_DEPTH || *total >= MAX_TOTAL_ELEMENTS {+ return;+ }+ *total += 1;++ let self_var: VARIANT = VARIANT::from(0i32); // CHILDID_SELF++ // Properties — each call wrapped to swallow per-element COM errors.+ let role_int: Option<i32> = acc+ .get_accRole(&self_var)+ .ok()+ .and_then(|v| variant_to_i32(&v));+ let name: Option<String> = acc+ .get_accName(&self_var)+ .ok()+ .map(|b| b.to_string())+ .filter(|s| !s.trim().is_empty());+ let default_action: Option<String> = acc+ .get_accDefaultAction(&self_var)+ .ok()+ .map(|b| b.to_string())+ .filter(|s| !s.trim().is_empty());++ // accLocation: out left, top, width, height (screen coords).+ let rect: Option<(i32, i32, i32, i32)> = {+ let mut l = 0i32;+ let mut t = 0i32;+ let mut w = 0i32;+ let mut h = 0i32;+ if acc+ .accLocation(&mut l, &mut t, &mut w, &mut h, &self_var)+ .is_ok()+ && w > 0+ && h > 0+ {+ Some((l, t, l + w, t + h))+ } else {+ None+ }+ };++ let control_type = role_to_control_type(role_int.unwrap_or(0));+ let actions = actions_for(role_int.unwrap_or(0), default_action.as_deref());+ let is_actionable = !actions.is_empty();+ let has_content = name.is_some();++ if is_actionable || has_content {+ // Retain the IAccessible pointer for the cache. Mirror what the UIA+ // walker does: clone, get raw, forget the local — the cache's Drop+ // releases via IUnknown.+ let retained: IAccessible = acc.clone();+ let ptr = retained.as_raw() as usize;+ std::mem::forget(retained);++ let (center_x, center_y) = rect+ .map(|(l, t, r, b)| ((l + r) / 2, (t + b) / 2))+ .unwrap_or((0, 0));++ let node = if is_actionable {+ let idx = *counter;+ *counter += 1;+ UiaNode {+ element_index: Some(idx),+ control_type: control_type.clone(),+ name: name.clone(),+ value: None,+ automation_id: None,+ help_text: None,+ actions: actions.clone(),+ element_ptr: ptr,+ center_x,+ center_y,+ rect,+ msaa_role: role_int,+ }+ } else {+ UiaNode {+ element_index: None,+ control_type: control_type.clone(),+ name: name.clone(),+ value: None,+ automation_id: None,+ help_text: None,+ actions: Vec::new(),+ element_ptr: ptr,+ center_x: 0,+ center_y: 0,+ rect,+ msaa_role: role_int,+ }+ };+ lines.push((depth, crate::uia::format_node_line(&node)));+ nodes.push(node);+ }++ // Recurse via accChildCount + get_accChild.+ let child_count: i32 = acc.accChildCount().unwrap_or(0);+ for i in 1..=child_count {+ let child_var = VARIANT::from(i);+ // accChild returns IDispatch — query for IAccessible.+ if let Ok(child_disp) = acc.get_accChild(&child_var) {+ if let Ok(child_acc) = child_disp.cast::<IAccessible>() {+ walk(&child_acc, depth + 1, nodes, lines, counter, total);+ }+ }+ }+}++/// Extract i32 from a VARIANT — accRole returns VT_I4 in practice (sometimes+/// VT_BSTR for custom roles, which we map to 0 = unknown).+unsafe fn variant_to_i32(v: &VARIANT) -> Option<i32> {+ let raw = v.as_raw();+ let vt = raw.Anonymous.Anonymous.vt;+ if vt == 3 {+ // VT_I4+ Some(raw.Anonymous.Anonymous.Anonymous.lVal)+ } else {+ None+ }+}++/// Map MSAA role id → control_type string. For roles not in this list we+/// emit `Role_<hex>` so the agent at least sees something diagnostic.+fn role_to_control_type(role: i32) -> String {+ match role {+ ROLE_SYSTEM_TITLEBAR => "TitleBar",+ ROLE_SYSTEM_MENUBAR => "MenuBar",+ ROLE_SYSTEM_SCROLLBAR => "ScrollBar",+ ROLE_SYSTEM_WINDOW => "Window",+ ROLE_SYSTEM_CLIENT => "Pane",+ ROLE_SYSTEM_MENUPOPUP => "Menu",+ ROLE_SYSTEM_MENUITEM => "MenuItem",+ ROLE_SYSTEM_TOOLTIP => "ToolTip",+ ROLE_SYSTEM_DIALOG => "Window",+ ROLE_SYSTEM_GROUPING => "Group",+ ROLE_SYSTEM_TOOLBAR => "ToolBar",+ ROLE_SYSTEM_STATUSBAR => "StatusBar",+ ROLE_SYSTEM_LINK => "Hyperlink",+ ROLE_SYSTEM_LIST => "List",+ ROLE_SYSTEM_LISTITEM => "ListItem",+ ROLE_SYSTEM_PAGETAB => "TabItem",+ ROLE_SYSTEM_PAGETABLIST => "Tab",+ ROLE_SYSTEM_GRAPHIC => "Image",+ ROLE_SYSTEM_STATICTEXT => "Text",+ ROLE_SYSTEM_TEXT => "Edit",+ ROLE_SYSTEM_PUSHBUTTON => "Button",+ ROLE_SYSTEM_CHECKBUTTON => "CheckBox",+ ROLE_SYSTEM_RADIOBUTTON => "RadioButton",+ ROLE_SYSTEM_COMBOBOX => "ComboBox",+ ROLE_SYSTEM_PROGRESSBAR => "ProgressBar",+ ROLE_SYSTEM_SLIDER => "Slider",+ ROLE_SYSTEM_BUTTONDROPDOWN+ | ROLE_SYSTEM_BUTTONMENU+ | ROLE_SYSTEM_BUTTONDROPDOWNGRID+ | ROLE_SYSTEM_SPLITBUTTON => "SplitButton",+ 0 => "Unknown",+ other => return format!("Role_0x{:X}", other),+ }+ .into()+}++/// Compute `actions=[...]` for a MSAA element. Roles with a meaningful+/// default action get `invoke`. Dropdown-flavored roles ALSO get `expand`+/// so callers can address the dropdown half separately — the click tool+/// routes `action:"expand"` to a right-edge SendInput click rather than+/// just calling `accDoDefaultAction` (which fires the press half).+fn actions_for(role: i32, default_action: Option<&str>) -> Vec<String> {+ let has_action = default_action+ .map(|s| !s.trim().is_empty())+ .unwrap_or(false);+ let mut actions = Vec::new();++ let is_dropdown = matches!(+ role,+ ROLE_SYSTEM_BUTTONDROPDOWN+ | ROLE_SYSTEM_BUTTONMENU+ | ROLE_SYSTEM_BUTTONDROPDOWNGRID+ | ROLE_SYSTEM_SPLITBUTTON+ );+ let is_clickable = matches!(+ role,+ ROLE_SYSTEM_PUSHBUTTON+ | ROLE_SYSTEM_CHECKBUTTON+ | ROLE_SYSTEM_RADIOBUTTON+ | ROLE_SYSTEM_LINK+ | ROLE_SYSTEM_MENUITEM+ | ROLE_SYSTEM_LISTITEM+ | ROLE_SYSTEM_PAGETAB+ | ROLE_SYSTEM_COMBOBOX+ );++ if has_action || is_dropdown || is_clickable {+ actions.push("invoke".into());+ }+ if is_dropdown {+ actions.push("expand".into());+ }+ actions+}++fn render_lines(lines: &[(usize, String)]) -> String {+ let mut out = String::new();+ for (depth, line) in lines {+ for _ in 0..*depth {+ out.push_str(" ");+ }+ out.push_str(line);+ out.push('\n');+ }+ out+}diff --git a/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs b/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rsindex 428ea0211b..af122cec36 100644--- a/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs+++ b/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs@@ -669,7 +669,16 @@ impl Tool for GetWindowStateTool {let count = tr.nodes.iter().filter(|n| n.element_index.is_some()).count();let header = format!("window_id={hwnd} pid={pid} elements={count}\n\n");content.push(mcp_server::protocol::Content::text(header + &tr.tree_markdown));- state.element_cache.update(pid, hwnd, &tr.nodes);+ // Route the cache to the matching dispatch path: any+ // node whose msaa_role is Some came from the MSAA+ // walker, so the entire snapshot must Drop via+ // IAccessible and click must dispatch through MSAA.+ let is_msaa = tr.nodes.iter().any(|n| n.msaa_role.is_some());+ if is_msaa {+ state.element_cache.update_msaa(pid, hwnd, &tr.nodes);+ } else {+ state.element_cache.update(pid, hwnd, &tr.nodes);+ }structured["element_count"] = json!(count);structured["tree_markdown"] = json!(tr.tree_markdown);}@@ -1775,6 +1784,7 @@ impl Tool for ClickTool {async fn invoke(&self, args: Value) -> ToolResult {use mcp_server::tool_args::ArgsExt;use crate::input::dispatch::{DispatchMode, EventKind, background_unavailable_error};+ use crate::uia::cache::SnapshotKind;let pid = match args.require_u32("pid") { Ok(v) => v, Err(e) => return e };let hwnd_opt = args.opt_u64("window_id");let elem_idx = args.opt_u64("element_index").map(|v| v as usize);@@ -1783,6 +1793,13 @@ impl Tool for ClickTool {let button = args.str_or("button", "left");let count = args.u64_or("count", 1) as usize;let dispatch = DispatchMode::from_args(&args);+ // Optional `action` arg picks among the actions exposed in the+ // accessibility tree. Today this only changes behavior for MSAA+ // BUTTONDROPDOWN: `"expand"` clicks the right-edge (dropdown arrow+ // half) instead of the center (press half). Defaults to first+ // action in the element's `actions=[...]` list, which preserves+ // existing semantics for UIA elements.+ let action_req = args.get("action").and_then(|v| v.as_str()).map(|s| s.to_string());// Resolve HWND: explicit, or auto from pid.let hwnd = match hwnd_opt {@@ -1797,6 +1814,84 @@ impl Tool for ClickTool {};if let Some(idx) = elem_idx {+ // ── MSAA dispatch (SAL/VCL targets) ────────────────────────────+ // For MSAA elements, route by role:+ // - `action:"expand"` on BUTTONDROPDOWN → SendInput at the+ // cached rect's right-edge (the dropdown arrow half).+ // Opens the picker (e.g. LO Writer Font Color → SALTMPSUBFRAME).+ // - default action / `"invoke"` → SendInput at center+ // (matches `accDoDefaultAction` semantics for VCL controls+ // and works through dispatch:"foreground"). We DO NOT call+ // `accDoDefaultAction` here because LO's MSAA impl applies+ // the change asynchronously and returns S_OK either way,+ // so the visible behavior is the same as a center click.+ if let Some((SnapshotKind::Msaa, role)) =+ self.state.element_cache.get_element_kind_and_role(pid, hwnd, idx)+ {+ const ROLE_BUTTONDROPDOWN: i32 = 0x38;+ const ROLE_BUTTONMENU: i32 = 0x39;+ const ROLE_BUTTONDROPDOWNGRID: i32 = 0x3A;+ const ROLE_SPLITBUTTON: i32 = 0x3E;+ let want_expand = action_req.as_deref() == Some("expand");+ let is_dropdown_role = matches!(+ role,+ Some(ROLE_BUTTONDROPDOWN | ROLE_BUTTONMENU | ROLE_BUTTONDROPDOWNGRID | ROLE_SPLITBUTTON)+ );+ let (tx, ty) = if want_expand && is_dropdown_role {+ match self.state.element_cache.get_element_rect(pid, hwnd, idx) {+ Some((_l, t, r, b)) => {+ // Right-edge of the SplitButton — the dropdown+ // arrow half. -4 puts the click safely inside+ // the arrow region for the typical ~12-16 px+ // arrow width VCL uses.+ (r - 4, (t + b) / 2)+ }+ None => {+ return ToolResult::error(format!(+ "MSAA element [{idx}] has no cached rect — \+ cannot dispatch action:\"expand\". Re-run \+ get_window_state to refresh the cache."+ ));+ }+ }+ } else if want_expand && !is_dropdown_role {+ return ToolResult::error(format!(+ "action:\"expand\" requested for MSAA element [{idx}] \+ but its role ({role:?}) is not a dropdown button. \+ Use action:\"invoke\" or omit the action arg."+ ));+ } else {+ match self.state.element_cache.get_element_center(pid, hwnd, idx) {+ Some(v) => v,+ None => return ToolResult::error(format!(+ "MSAA element [{idx}] not in cache for hwnd={hwnd}. \+ Call get_window_state first."+ )),+ }+ };+ pin_overlay_above(hwnd);+ overlay_glide_to(tx as f64, ty as f64).await;+ crate::overlay::send_command(cursor_overlay::OverlayCommand::ClickPulse {+ x: tx as f64, y: ty as f64,+ });+ let btn_fg = button.clone();+ let prev_fg_addr = unsafe {+ windows::Win32::UI::WindowsAndMessaging::GetForegroundWindow().0 as usize+ };+ let send_result = tokio::task::spawn_blocking(move || {+ crate::input::send_click_synthesized(hwnd, tx, ty, count, &btn_fg)+ }).await;+ tokio::spawn(restore_foreground_polling_best_effort(prev_fg_addr, pid));+ let half = if want_expand { "dropdown" } else { "press" };+ return match send_result {+ Ok(Ok(())) => ToolResult::text(format!(+ "✅ Performed SendInput click on MSAA [{idx}] {half} half at ({tx},{ty})."+ )),+ Ok(Err(e)) => ToolResult::error(e.to_string()),+ Err(e) => ToolResult::error(format!("Task error: {e}")),+ };+ }+// UIA path: get cached center (no COM call needed — captured at walk time).let (cx, cy) = match self.state.element_cache.get_element_center(pid, hwnd, idx) {Some(v) => v,diff --git a/libs/cua-driver/rust/crates/platform-windows/src/uia/cache.rs b/libs/cua-driver/rust/crates/platform-windows/src/uia/cache.rsindex fbf2dfc509..02e5705db7 100644--- a/libs/cua-driver/rust/crates/platform-windows/src/uia/cache.rs+++ b/libs/cua-driver/rust/crates/platform-windows/src/uia/cache.rs@@ -1,11 +1,12 @@-//! UIA element cache for Windows.+//! Accessibility element cache for Windows.//!//! Mirrors the macOS ElementCache design: per-(pid, window_id) snapshots of-//! retained IUIAutomationElement COM pointers.+//! retained COM pointers (either `IUIAutomationElement` for the UIA primary+//! path, or `IAccessible` for the MSAA fallback used on SAL/VCL targets).//!-//! Memory contract: UiaNode::element_ptr is a raw IUIAutomationElement vtable-//! pointer with an extra AddRef from clone()+forget() in the walker. Drop here-//! calls Release() to balance.+//! Memory contract: `UiaNode::element_ptr` is a raw COM vtable pointer with+//! an extra AddRef from `clone()+forget()` in the walker. `CachedSnapshot::Drop`+//! calls `Release()` via the kind-appropriate vtable to balance.//!//! The locked-HashMap plumbing lives in `mcp_server::element_cache` — see//! `docs/dedup-audit.md` item #3. This module owns the Windows-specific@@ -14,8 +15,8 @@use super::UiaNode;use mcp_server::element_cache::ElementCacheCore;-use windows::Win32::UI::Accessibility::IUIAutomationElement;use windows::core::Interface;+use windows::Win32::UI::Accessibility::{IAccessible, IUIAutomationElement};#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]pub struct CacheKey {@@ -23,21 +24,52 @@ pub struct CacheKey {pub hwnd: u64,}+/// Which COM interface the cached pointers reference. Determines the+/// `from_raw` cast `Drop` performs and the dispatch path the click tool+/// takes when an element is invoked.+#[derive(Debug, Clone, Copy, PartialEq, Eq)]+pub enum SnapshotKind {+ /// `element_ptr` values are `IUIAutomationElement` pointers; click+ /// tool tries Invoke/Toggle/SelectionItem/ExpandCollapse patterns.+ Uia,+ /// `element_ptr` values are `IAccessible` pointers (MSAA fallback for+ /// SAL/VCL classes); click tool dispatches via `accDoDefaultAction`+ /// for invoke, right-edge SendInput for expand.+ Msaa,+}+pub struct CachedSnapshot {- /// element_index → raw IUIAutomationElement pointer (retained).+ pub kind: SnapshotKind,+ /// element_index → raw COM pointer (retained).pub elements: Vec<usize>,/// element_index → screen-coordinate center (captured at walk time).pub centers: Vec<(i32, i32)>,+ /// element_index → screen-coordinate rect (l, t, r, b). `None` when+ /// the element didn't report a meaningful bounding box.+ pub rects: Vec<Option<(i32, i32, i32, i32)>>,+ /// element_index → MSAA role code (`Some` for MSAA-walked entries,+ /// `None` for UIA-walked entries). The click tool reads this to gate+ /// right-edge dispatch on `ROLE_SYSTEM_BUTTONDROPDOWN` etc.+ pub msaa_roles: Vec<Option<i32>>,}impl Drop for CachedSnapshot {fn drop(&mut self) {for ptr in &self.elements {- if *ptr != 0 {- // Reconstruct an interface reference and drop it (calls Release).- unsafe {- let iface: IUIAutomationElement = IUIAutomationElement::from_raw(*ptr as *mut _);- drop(iface);+ if *ptr == 0 {+ continue;+ }+ unsafe {+ match self.kind {+ SnapshotKind::Uia => {+ let iface: IUIAutomationElement =+ IUIAutomationElement::from_raw(*ptr as *mut _);+ drop(iface);+ }+ SnapshotKind::Msaa => {+ let iface: IAccessible = IAccessible::from_raw(*ptr as *mut _);+ drop(iface);+ }}}}@@ -50,14 +82,34 @@ pub struct ElementCache {impl ElementCache {pub fn new() -> Self {- Self { core: ElementCacheCore::new() }+ Self {+ core: ElementCacheCore::new(),+ }}+ /// Update the snapshot for (pid, hwnd) with the actionable elements+ /// from a UIA walk. Mirrors `update_msaa` for the MSAA path.pub fn update(&self, pid: u32, hwnd: u64, nodes: &[UiaNode]) {+ self.update_with_kind(pid, hwnd, nodes, SnapshotKind::Uia);+ }++ /// Same as `update` but tags the snapshot as MSAA so Drop releases the+ /// pointers as `IAccessible` and the click tool routes through the+ /// MSAA dispatch path.+ pub fn update_msaa(&self, pid: u32, hwnd: u64, nodes: &[UiaNode]) {+ self.update_with_kind(pid, hwnd, nodes, SnapshotKind::Msaa);+ }++ fn update_with_kind(&self, pid: u32, hwnd: u64, nodes: &[UiaNode], kind: SnapshotKind) {let actionable: Vec<&UiaNode> = nodes.iter().filter(|n| n.element_index.is_some()).collect();let elements: Vec<usize> = actionable.iter().map(|n| n.element_ptr).collect();let centers: Vec<(i32, i32)> = actionable.iter().map(|n| (n.center_x, n.center_y)).collect();- self.core.insert(CacheKey { pid, hwnd }, CachedSnapshot { elements, centers });+ let rects: Vec<Option<(i32, i32, i32, i32)>> = actionable.iter().map(|n| n.rect).collect();+ let msaa_roles: Vec<Option<i32>> = actionable.iter().map(|n| n.msaa_role).collect();+ self.core.insert(+ CacheKey { pid, hwnd },+ CachedSnapshot { kind, elements, centers, rects, msaa_roles },+ );}pub fn get_element_ptr(&self, pid: u32, hwnd: u64, element_index: usize) -> Option<usize> {@@ -72,6 +124,32 @@ impl ElementCache {.flatten()}+ /// Cached screen rect for the element. Used by the click tool to+ /// compute the right-edge dispatch point for `action:"expand"` on+ /// MSAA BUTTONDROPDOWN.+ pub fn get_element_rect(&self, pid: u32, hwnd: u64, element_index: usize) -> Option<(i32, i32, i32, i32)> {+ self.core+ .with_snapshot(&CacheKey { pid, hwnd }, |s| s.rects.get(element_index).copied().flatten())+ .flatten()+ }++ /// Kind + MSAA role for the element. `(Msaa, Some(0x38))` identifies a+ /// MSAA BUTTONDROPDOWN; click tool routes `action:"expand"` to+ /// right-edge SendInput for these.+ pub fn get_element_kind_and_role(+ &self,+ pid: u32,+ hwnd: u64,+ element_index: usize,+ ) -> Option<(SnapshotKind, Option<i32>)> {+ self.core+ .with_snapshot(&CacheKey { pid, hwnd }, |s| {+ let role = s.msaa_roles.get(element_index).copied().flatten();+ Some((s.kind, role))+ })+ .flatten()+ }+pub fn element_count(&self, pid: u32, hwnd: u64) -> usize {self.core.with_snapshot(&CacheKey { pid, hwnd }, |s| s.elements.len())@@ -80,5 +158,7 @@ impl ElementCache {}impl Default for ElementCache {- fn default() -> Self { Self::new() }+ fn default() -> Self {+ Self::new()+ }}diff --git a/libs/cua-driver/rust/crates/platform-windows/src/uia/mod.rs b/libs/cua-driver/rust/crates/platform-windows/src/uia/mod.rsindex 76e9dacf21..b58157539e 100644--- a/libs/cua-driver/rust/crates/platform-windows/src/uia/mod.rs+++ b/libs/cua-driver/rust/crates/platform-windows/src/uia/mod.rs@@ -32,7 +32,11 @@ pub use windows_enum::enumerate_top_level_windows;const MAX_DEPTH: usize = 25;const MAX_TOTAL_ELEMENTS: usize = 5000;-/// A single node in the UIA tree.+/// A single node in the accessibility tree.+///+/// Same shape for the UIA primary path AND the MSAA fallback (used for+/// SAL/VCL window classes — see `msaa.rs`). MSAA-only fields use the+/// `_ptr is IAccessible` / `msaa_role = Some(...)` discriminator.#[derive(Clone)]pub struct UiaNode {pub element_index: Option<usize>,@@ -42,11 +46,22 @@ pub struct UiaNode {pub automation_id: Option<String>,pub help_text: Option<String>,pub actions: Vec<String>,- /// Raw IUIAutomationElement pointer as usize (retained for cache).+ /// Raw COM pointer (IUIAutomationElement for UIA path, IAccessible for+ /// MSAA path) as usize. Retained — `ElementCache` Drop releases it via+ /// the `kind`-appropriate vtable.pub element_ptr: usize,/// Screen-coordinate center, captured at walk time to avoid later COM calls.pub center_x: i32,pub center_y: i32,+ /// Full screen-coord rect (left, top, right, bottom). Available for+ /// elements that report a meaningful bounding box. Used by the click+ /// tool when `action:"expand"` needs the right-edge offset.+ pub rect: Option<(i32, i32, i32, i32)>,+ /// MSAA role code (e.g. 0x38 = `ROLE_SYSTEM_BUTTONDROPDOWN`). `Some`+ /// iff this node came from the MSAA walker — the click tool checks+ /// this to route `action:"expand"` to a right-edge SendInput click+ /// instead of an unsupported UIA pattern lookup.+ pub msaa_role: Option<i32>,}pub struct UiaTreeResult {@@ -117,55 +132,33 @@ unsafe fn walk_tree_unsafe(hwnd: u64, query: Option<&str>) -> UiaTreeResult {let hwnd_win = windows::Win32::Foundation::HWND(hwnd as *mut _);- // SAL (LibreOffice / OpenOffice) fast-path: VCL's UIA provider hangs- // on `BuildUpdatedCache(TreeScope.Subtree)` for transient dialogs- // (SALSUBFRAME class — Confirmation, Warning, modal Yes/No). Both- // the atomic `ElementFromHandleBuildCache` and the split- // `ElementFromHandle + BuildUpdatedCache(Subtree)` paths block- // indefinitely under the daemon's MTA thread pool (the same code- // runs fine via CLI in-process, suggesting a COM apartment /- // concurrent-access interaction with SAL's provider). The 4 s- // outer timeout fires and the caller gets the structured- // diagnostic, but that's a wasted 4 s every call.+ // SAL (LibreOffice / OpenOffice) fallback: ALL SAL-class windows go+ // through the MSAA walker.+ //+ // Two reasons:+ // 1. **Hang avoidance** for SALSUBFRAME / SALMENU / SALTMPSUBFRAME —+ // VCL's UIA provider hangs on `BuildUpdatedCache(TreeScope.Subtree)`+ // under the daemon's MTA pool, wasting the 4 s outer timeout.+ // 2. **Role fidelity** for SALFRAME (main document window, Recovery+ // dialog) — UIA technically walks fine, but the built-in+ // MSAA→UIA proxy collapses `ROLE_SYSTEM_BUTTONDROPDOWN` (0x38) to+ // a featureless `SplitButton` with no separable dropdown+ // affordance. MSAA via oleacc.dll preserves the role, letting+ // the click tool route `action:"expand"` to a right-edge+ // SendInput click that opens the dropdown half (e.g. LO Writer+ // "Font Color" → SALTMPSUBFRAME color picker).//- // Short-circuit by class detection: if the target's class starts- // with "SAL", skip the bulk-cache walk entirely and return the- // same synthetic stub the post-walk SAL-skip would emit, with the- // three actionable fallback options. Saves the 4 s wait and the- // ambiguity of "did the walk run? did it find nothing? did it- // hang?" The caller sees the stub immediately and routes to the- // appropriate workaround.- // Class lookup once — used both for the fast-path skip decision- // and the diagnostic stub. SALSUBFRAME is the empirically-confirmed- // hang class (modal Confirmation / Warning dialogs). SALFRAME- // (the document window, the Recovery dialog) walks fine, so don't- // include it in the skip list — pixel + element_index against- // Recovery's Discard All button worked end-to-end through the- // primary path. SALMENU / SALTMPSUBFRAME are transient menu hosts- // that the user typically doesn't get_window_state on; default to- // skipping out of caution. If a real use-case emerges, narrow.- let sal_class: Option<String> = {+ // The UIA pattern dispatches we lose for SALFRAME (Toggle / Select /+ // ExpandCollapse) only matter for WinUI3 controls, which VCL doesn't+ // host. Net win.+ let sal_class: bool = {use windows::Win32::UI::WindowsAndMessaging::GetClassNameW;let mut buf = [0u16; 64];let n = GetClassNameW(hwnd_win, &mut buf);- if n > 0 {- let s = String::from_utf16_lossy(&buf[..n as usize]);- if s == "SALSUBFRAME" || s == "SALMENU" || s == "SALTMPSUBFRAME" {- Some(s)- } else { None }- } else { None }+ n > 0 && String::from_utf16_lossy(&buf[..n as usize]).starts_with("SAL")};- if let Some(class) = sal_class {- let stub = format!(- "- Window <{class} — SAL/VCL target, UIA walk skipped>\n\- (SAL's UIA provider hangs on TreeScope.Subtree BuildUpdatedCache \- when the daemon is running in MTA; the cua-driver fast-path \- returns immediately rather than wait 4 s. Use one of: \- (a) `screenshot(pid, window_id)` + pixel `click(x, y)`; \- (b) `press_key` with `dispatch:\"foreground\"` (Esc / Enter / Y / N); \- (c) `get_window_state` with `capture_mode:\"vision\"`.)\n"- );- return UiaTreeResult { tree_markdown: stub, nodes: Vec::new() };+ if sal_class {+ return crate::msaa::walk_msaa_tree(hwnd);}// Two-call sequence (ElementFromHandle + BuildUpdatedCache) instead of@@ -408,7 +401,7 @@ unsafe fn walk_cached(let node = if is_actionable {let idx = *counter;*counter += 1;- let (center_x, center_y) = read_cached_bounding_rect(element);+ let (center_x, center_y, rect) = read_cached_bounding_rect_full(element);UiaNode {element_index: Some(idx),control_type: control_type.clone(),@@ -420,6 +413,8 @@ unsafe fn walk_cached(element_ptr: ptr,center_x,center_y,+ rect,+ msaa_role: None,}} else {UiaNode {@@ -433,6 +428,8 @@ unsafe fn walk_cached(element_ptr: ptr,center_x: 0,center_y: 0,+ rect: None,+ msaa_role: None,}};@@ -496,11 +493,19 @@ fn read_cached_bool(element: &IUIAutomationElement, property_id: windows::Win32:}}-fn read_cached_bounding_rect(element: &IUIAutomationElement) -> (i32, i32) {+/// Read bounding rect as (center_x, center_y, Some((l,t,r,b))). Returns+/// rect=None when the element has no meaningful BoundingRectangle (offscreen+/// containers, structure-only elements).+fn read_cached_bounding_rect_full(element: &IUIAutomationElement) -> (i32, i32, Option<(i32, i32, i32, i32)>) {unsafe {- element.CachedBoundingRectangle()- .map(|r| ((r.left + r.right) / 2, (r.top + r.bottom) / 2))- .unwrap_or((0, 0))+ match element.CachedBoundingRectangle() {+ Ok(r) if r.right > r.left && r.bottom > r.top => (+ (r.left + r.right) / 2,+ (r.top + r.bottom) / 2,+ Some((r.left, r.top, r.right, r.bottom)),+ ),+ _ => (0, 0, None),+ }}}@@ -587,7 +592,7 @@ fn control_type_name(id: i32) -> String {}.into()}-fn format_node_line(node: &UiaNode) -> String {+pub(crate) fn format_node_line(node: &UiaNode) -> String {let mut s = String::new();if let Some(idx) = node.element_index {s.push_str(&format!("- [{}] {}", idx, node.control_type));