Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

feat(cua-driver-rs)(platform-windows): MSAA fallback for SAL/VCL windows (#1709)

Francesco Bonacci02f1f033e926
diff --git a/libs/cua-driver/rust/crates/cua-driver/tests/harness_lo_vcl_test.rs b/libs/cua-driver/rust/crates/cua-driver/tests/harness_lo_vcl_test.rs
new file mode 100644
index 0000000000..ccb0da20cb
--- /dev/null
+++ b/libs/cua-driver/rust/crates/cua-driver/tests/harness_lo_vcl_test.rs
@@ -0,0 +1,868 @@
+//! Integration tests against a live LibreOffice Writer instance.
+//!
+//! Regression-guards the VCL/SAL gaps the vision-only LO Writer flow
+//! surfaced (manually validated during the PR #1708 follow-up
+//! exploration). Each test asserts the *current* gap — it will fail
+//! loudly if cua-driver later closes the underlying gap, prompting
+//! a flip of the assertion.
+//!
+//! ## Tests in this file
+//!
+//! 1. **`harness_lo_vcl_font_color_split_button_exposes_expand`** —
+//! Font Color SplitButton reports `actions=[invoke,expand]`.
+//!
+//! 2. **`harness_lo_vcl_font_color_expand_opens_picker`** — click
+//! with `action:"expand"` opens a SALTMPSUBFRAME picker.
+//!
+//! 3. **`harness_lo_vcl_modal_input_roundtrip_works`** — SAL/VCL
+//! modal (Find & Replace) accepts SendInput, snapshot has actionable
+//! elements (no more skip stub).
+//!
+//! 4. **`harness_lo_vcl_all_toolbar_split_buttons_expose_expand`** —
+//! *every* toolbar SplitButton in Writer has `expand` (not just
+//! Font Color). Guards `msaa::actions_for` BUTTONDROPDOWN family.
+//!
+//! 5. **`harness_lo_vcl_color_pick_green_end_to_end`** — full
+//! workflow: type text → select → open Font Color → click Green →
+//! picker closes (proves color landed).
+//!
+//! 6. **`harness_lo_vcl_recovery_dialog_walks_via_msaa`** — Document
+//! Recovery dialog (SALFRAME class) still walks via MSAA, exposes
+//! "Discard All" + "Recover Selected" buttons. Guards the
+//! routing change for the pre-existing Recovery-dialog flow.
+//!
+//! 7. **`harness_lo_vcl_calc_msaa_smoke`** — LO Calc (different app
+//! surface on the same VCL base) also walks via MSAA with ≥15
+//! SplitButtons exposing `expand`. Confirms the path generalizes
+//! beyond Writer.
+//!
+//! ## How to run
+//!
+//! Local (requires LibreOffice installed):
+//! cargo test --test harness_lo_vcl_test -- --ignored --nocapture
+//!
+//! Tests skip cleanly if `swriter.exe` isn't on disk at one of the
+//! standard install locations (override via `LO_SWRITER_EXE`).
+
+#![cfg(target_os = "windows")]
+
+use std::io::{BufRead, BufReader, Write};
+use std::path::PathBuf;
+use std::process::{Child, ChildStdin, ChildStdout, Command, Stdio};
+use std::time::Duration;
+
+// ── workspace / LO paths ─────────────────────────────────────────────────────
+
+fn workspace_root() -> PathBuf {
+ let manifest = std::env::var("CARGO_MANIFEST_DIR").expect("CARGO_MANIFEST_DIR");
+ PathBuf::from(manifest).parent().unwrap().parent().unwrap().to_owned()
+}
+fn driver_binary() -> PathBuf { workspace_root().join("target/debug/cua-driver.exe") }
+
+/// LO Writer executable. Honour `LO_SWRITER_EXE` env override (for CI
+/// images with non-default install paths), otherwise probe the two
+/// standard locations.
+fn swriter_exe() -> Option<PathBuf> {
+ if let Ok(p) = std::env::var("LO_SWRITER_EXE") {
+ let pb = PathBuf::from(p);
+ if pb.exists() { return Some(pb); }
+ }
+ for candidate in [
+ r"C:\Program Files\LibreOffice\program\swriter.exe",
+ r"C:\Program Files (x86)\LibreOffice\program\swriter.exe",
+ ] {
+ let pb = PathBuf::from(candidate);
+ if pb.exists() { return Some(pb); }
+ }
+ None
+}
+
+/// LO Calc executable, same probe shape as `swriter_exe()`.
+fn scalc_exe() -> Option<PathBuf> {
+ if let Ok(p) = std::env::var("LO_SCALC_EXE") {
+ let pb = PathBuf::from(p);
+ if pb.exists() { return Some(pb); }
+ }
+ for candidate in [
+ r"C:\Program Files\LibreOffice\program\scalc.exe",
+ r"C:\Program Files (x86)\LibreOffice\program\scalc.exe",
+ ] {
+ let pb = PathBuf::from(candidate);
+ if pb.exists() { return Some(pb); }
+ }
+ None
+}
+
+// ── JSON-RPC plumbing ────────────────────────────────────────────────────────
+
+fn send(stdin: &mut ChildStdin, req: serde_json::Value) {
+ writeln!(stdin, "{}", serde_json::to_string(&req).unwrap()).unwrap();
+}
+fn recv(stdout: &mut BufReader<&mut ChildStdout>) -> serde_json::Value {
+ let mut line = String::new();
+ stdout.read_line(&mut line).expect("read");
+ serde_json::from_str(line.trim()).expect("json")
+}
+fn init(stdin: &mut ChildStdin, stdout: &mut BufReader<&mut ChildStdout>) {
+ send(stdin, serde_json::json!({"jsonrpc":"2.0","id":1,"method":"initialize","params":{}}));
+ let _ = recv(stdout);
+}
+fn call(stdin: &mut ChildStdin, stdout: &mut BufReader<&mut ChildStdout>,
+ id: u32, name: &str, args: serde_json::Value) -> serde_json::Value {
+ send(stdin, serde_json::json!({
+ "jsonrpc":"2.0","id":id,"method":"tools/call",
+ "params":{"name":name,"arguments":args}
+ }));
+ recv(stdout)
+}
+
+fn snapshot_text(s: &serde_json::Value) -> &str {
+ s["result"]["content"][0]["text"].as_str().unwrap_or("")
+}
+
+// ── fixture ──────────────────────────────────────────────────────────────────
+
+struct LoSession {
+ _writer: Child,
+ driver: Child,
+ driver_stdin: ChildStdin,
+ driver_stdout: ChildStdout,
+ writer_pid: u32,
+ writer_wid: u64,
+}
+
+impl Drop for LoSession {
+ fn drop(&mut self) {
+ // Try clean shutdown via cua-driver kill_app first so LO doesn't
+ // leak a soffice.bin background daemon across test runs.
+ let _ = self.driver.kill(); let _ = self.driver.wait();
+ let _ = self._writer.kill(); let _ = self._writer.wait();
+ // SAL/VCL soffice.bin daemon hangs around after the parent exits;
+ // best-effort sweep via taskkill so the next test starts clean.
+ let _ = Command::new("taskkill").args(["/F", "/IM", "soffice.bin"]).output();
+ std::thread::sleep(Duration::from_millis(800));
+ }
+}
+
+fn setup() -> Option<LoSession> {
+ let driver = driver_binary();
+ if !driver.exists() {
+ eprintln!("cua-driver.exe not built — run `cargo build` first"); return None;
+ }
+ let writer = match swriter_exe() {
+ Some(p) => p,
+ None => {
+ eprintln!("LibreOffice swriter.exe not found — set LO_SWRITER_EXE or install LO");
+ return None;
+ }
+ };
+
+ // Sweep any leftover soffice.bin from a prior run so this test gets
+ // a fresh writer pid + clean recovery state.
+ let _ = Command::new("taskkill").args(["/F", "/IM", "soffice.bin"]).output();
+ std::thread::sleep(Duration::from_millis(800));
+
+ // `-norestore` skips the Document Recovery dialog (which would
+ // block the Writer top-level window from ever appearing if the
+ // previous LO session crashed). `-nologo` skips the splash screen.
+ let _writer = Command::new(&writer)
+ .args(["-norestore", "-nologo"])
+ .stdout(Stdio::null()).stderr(Stdio::null())
+ .spawn().ok()?;
+ // LO launches via soffice.bin background daemon, then forks the
+ // actual writer process. The pid we just spawned is the launcher;
+ // the real writer is a child. Poll list_apps via cua-driver to
+ // find it.
+ std::thread::sleep(Duration::from_secs(3));
+
+ let mut driver_c = Command::new(&driver)
+ .stdin(Stdio::piped()).stdout(Stdio::piped()).stderr(Stdio::null())
+ .spawn().ok()?;
+ let mut driver_stdin = driver_c.stdin.take().unwrap();
+ let mut driver_stdout = driver_c.stdout.take().unwrap();
+ {
+ let mut stdout = BufReader::new(&mut driver_stdout);
+ init(&mut driver_stdin, &mut stdout);
+ // Find the LO Writer window by title — its owning pid is the
+ // soffice.bin daemon (NOT the launcher pid above). Poll up to
+ // 30 s for the recovery dialog (if any) + main window to appear.
+ let deadline = std::time::Instant::now() + Duration::from_secs(30);
+ loop {
+ let resp = call(&mut driver_stdin, &mut stdout, 10, "list_windows", serde_json::json!({}));
+ if let Some(wins) = resp["result"]["structuredContent"]["windows"].as_array() {
+ for w in wins {
+ let title = w["title"].as_str().unwrap_or("");
+ let app = w["app_name"].as_str().unwrap_or("");
+ if app.contains("soffice") && title.contains("LibreOffice Writer") {
+ let pid = w["pid"].as_u64().unwrap_or(0) as u32;
+ let wid = w["window_id"].as_u64().unwrap_or(0);
+ if pid > 0 && wid > 0 {
+ drop(stdout);
+ return Some(LoSession {
+ _writer, driver: driver_c, driver_stdin, driver_stdout,
+ writer_pid: pid, writer_wid: wid,
+ });
+ }
+ }
+ }
+ }
+ if std::time::Instant::now() > deadline {
+ eprintln!("Writer window did not appear within 30 s");
+ drop(stdout);
+ let _ = driver_c.kill();
+ return None;
+ }
+ std::thread::sleep(Duration::from_millis(500));
+ }
+ }
+}
+
+fn with_session<F, R>(fx: &mut LoSession, f: F) -> R
+where F: FnOnce(&mut ChildStdin, &mut BufReader<&mut ChildStdout>, u32, u64) -> R {
+ let mut stdout = BufReader::new(&mut fx.driver_stdout);
+ f(&mut fx.driver_stdin, &mut stdout, fx.writer_pid, fx.writer_wid)
+}
+
+// ── Test 1: Font Color SplitButton exposes `expand` via MSAA fallback ───────
+
+/// Confirms the MSAA fallback (`platform-windows/src/msaa.rs`) lets
+/// cua-driver see VCL toolbar SplitButtons' dropdown halves.
+///
+/// Before the MSAA fallback landed: UIA's MSAA→UIA proxy collapsed
+/// `ROLE_SYSTEM_BUTTONDROPDOWN` (0x38) to a featureless `SplitButton`
+/// with `actions=[invoke]` — agents could re-fire the press half but
+/// not open the picker. After the MSAA fallback: SAL-class windows
+/// walk via oleacc's `AccessibleObjectFromWindow`, which preserves
+/// the BUTTONDROPDOWN role, and cua-driver maps it to
+/// `actions=[invoke,expand]`. The `click` tool's `action:"expand"`
+/// case clicks the right-edge of the cached rect (the dropdown arrow
+/// half) via SendInput.
+///
+/// If this fails: either the MSAA path stopped applying to SALFRAME
+/// (check `uia/mod.rs` SAL class detection), or LO changed its
+/// toolbar role (no longer ROLE_SYSTEM_BUTTONDROPDOWN).
+#[test]
+#[ignore]
+fn harness_lo_vcl_font_color_split_button_exposes_expand() {
+ let mut fx = match setup() { Some(s) => s, None => return };
+ with_session(&mut fx, |stdin, stdout, pid, wid| {
+ let snap = call(stdin, stdout, 30, "get_window_state", serde_json::json!({
+ "pid": pid as i64, "window_id": wid,
+ "capture_mode": "ax", "query": "Font Color"
+ }));
+ let text = snapshot_text(&snap);
+ assert!(text.contains("SplitButton \"Font Color\""),
+ "Font Color SplitButton not found in tree: {text:?}");
+ let line = text.lines().find(|l| l.contains("\"Font Color\"")).unwrap_or("");
+ assert!(line.contains("expand"),
+ "Expected `expand` in actions on Font Color SplitButton — got {line:?}. \
+ The MSAA fallback may not be running for SALFRAME (check uia/mod.rs \
+ SAL class detection — should route ALL SAL* classes through msaa.rs).");
+ assert!(line.contains("invoke"),
+ "Expected `invoke` to remain in actions alongside `expand` — got {line:?}. \
+ MSAA walker should expose both press and dropdown halves.");
+ });
+}
+
+// ── Test 2: action:"expand" actually opens the color picker ─────────────────
+
+/// End-to-end test that the `click(element_index, action:"expand")`
+/// dispatch on a MSAA BUTTONDROPDOWN actually opens the dropdown.
+///
+/// Asserts:
+/// 1. `get_window_state` finds Font Color by name and yields an
+/// element_index.
+/// 2. `click(element_index=X, action:"expand")` returns success.
+/// 3. A new top-level window appears under the LO pid with title
+/// "Font Color" (the SALTMPSUBFRAME color picker).
+///
+/// Failures point at:
+/// - (1) MSAA walker not finding Font Color (check msaa.rs walker
+/// budget / depth, or LO renamed the button).
+/// - (2) cua-driver click tool's MSAA dispatch broke (check
+/// `tools/impl_.rs` BUTTONDROPDOWN branch).
+/// - (3) Right-edge offset wrong for current LO version's toolbar
+/// scale (check `rect.right - 4` heuristic in
+/// `tools/impl_.rs`).
+#[test]
+#[ignore]
+fn harness_lo_vcl_font_color_expand_opens_picker() {
+ let mut fx = match setup() { Some(s) => s, None => return };
+ with_session(&mut fx, |stdin, stdout, pid, wid| {
+ // Bring Writer to foreground so SendInput click lands on it.
+ let _ = call(stdin, stdout, 30, "bring_to_front", serde_json::json!({
+ "pid": pid as i64, "window_id": wid
+ }));
+ std::thread::sleep(Duration::from_millis(400));
+
+ let snap = call(stdin, stdout, 31, "get_window_state", serde_json::json!({
+ "pid": pid as i64, "window_id": wid,
+ "capture_mode": "ax", "query": "Font Color"
+ }));
+ let text = snapshot_text(&snap);
+ let line = text.lines()
+ .find(|l| l.contains("\"Font Color\"") && l.contains("expand"))
+ .unwrap_or_else(|| panic!("Font Color SplitButton with `expand` action not found: {text:?}"));
+ let s = line.find('[').expect("element_index bracket open");
+ let e = line[s..].find(']').expect("element_index bracket close") + s;
+ let idx: u64 = line[s+1..e].trim().parse()
+ .unwrap_or_else(|_| panic!("could not parse element_index from line {line:?}"));
+
+ // Snapshot windows under our pid BEFORE the click.
+ let before = call(stdin, stdout, 32, "list_windows", serde_json::json!({
+ "pid": pid as i64
+ }));
+ let before_ids: std::collections::HashSet<u64> = before
+ ["result"]["structuredContent"]["windows"].as_array()
+ .map(|a| a.iter().filter_map(|w| w["window_id"].as_u64()).collect())
+ .unwrap_or_default();
+
+ let resp = call(stdin, stdout, 33, "click", serde_json::json!({
+ "pid": pid as i64, "window_id": wid,
+ "element_index": idx, "action": "expand"
+ }));
+ let resp_text = resp["result"]["content"][0]["text"].as_str().unwrap_or("");
+ assert!(resp_text.starts_with("✅"),
+ "click(action:expand) failed: {resp_text:?}");
+ assert!(resp_text.contains("dropdown half"),
+ "Expected response to mention dropdown half — got {resp_text:?}. \
+ The MSAA dispatch path may not have triggered.");
+
+ // Let the picker spawn.
+ std::thread::sleep(Duration::from_millis(900));
+
+ let after = call(stdin, stdout, 34, "list_windows", serde_json::json!({
+ "pid": pid as i64
+ }));
+ let new_wins: Vec<&serde_json::Value> = after
+ ["result"]["structuredContent"]["windows"].as_array()
+ .map(|a| a.iter()
+ .filter(|w| {
+ let id = w["window_id"].as_u64();
+ id.map(|i| !before_ids.contains(&i)).unwrap_or(false)
+ })
+ .collect())
+ .unwrap_or_default();
+ let picker = new_wins.iter().find(|w| {
+ w["title"].as_str().map(|t| t.contains("Font Color")).unwrap_or(false)
+ });
+ assert!(picker.is_some(),
+ "No new window titled 'Font Color' appeared after click(action:expand). \
+ new_windows={new_wins:?}. The right-edge dispatch may have hit the \
+ wrong pixel (LO's toolbar scaled differently?) or the picker spawned \
+ as a child window instead of a top-level.");
+ });
+}
+
+// ── Working path: SALSUBFRAME modal dialogs accept SendInput input ─────────
+
+/// Confirms the MSAA fallback gives a walkable tree on SALSUBFRAME
+/// modal dialogs too (was the "SAL/VCL target, UIA walk skipped" stub
+/// before the MSAA path landed), AND that the dialog accepts
+/// SendInput-injected keyboard input.
+///
+/// Specifically:
+/// - `hotkey(ctrl+h, foreground)` against the main Writer SALFRAME
+/// opens the Find & Replace dialog (a SALSUBFRAME).
+/// - `get_window_state(capture_mode:"ax")` on the dialog returns a
+/// full element tree via the MSAA walker — includes the "Close"
+/// button and the Find/Replace edit fields as addressable
+/// element_indices (a recent capability — the pre-MSAA stub had
+/// zero actionable elements).
+/// - `press_key(escape, foreground)` against the SALSUBFRAME closes
+/// it cleanly.
+///
+/// If this test ever fails, one of three things regressed:
+/// (a) Ctrl+H accelerator on the main Writer window stopped firing
+/// under SendInput (foreground key dispatch broken on SALFRAME).
+/// (b) MSAA walker stopped finding actionable elements in
+/// SALSUBFRAME (check `msaa.rs` budget / depth).
+/// (c) Foreground Escape stopped reaching the SALSUBFRAME (SAL
+/// filter changed in a newer LO).
+#[test]
+#[ignore]
+fn harness_lo_vcl_modal_input_roundtrip_works() {
+ let mut fx = match setup() { Some(s) => s, None => return };
+ with_session(&mut fx, |stdin, stdout, pid, wid| {
+ std::thread::sleep(Duration::from_millis(800));
+
+ let open_resp = call(stdin, stdout, 30, "hotkey", serde_json::json!({
+ "pid": pid as i64, "window_id": wid,
+ "keys": ["ctrl", "h"], "dispatch": "foreground"
+ }));
+ let open_text = open_resp["result"]["content"][0]["text"].as_str().unwrap_or("");
+ assert!(open_text.starts_with("✅"),
+ "hotkey(ctrl+h, foreground) failed: {open_text:?}");
+
+ let dialog_wid = {
+ let deadline = std::time::Instant::now() + Duration::from_secs(8);
+ loop {
+ let wins = call(stdin, stdout, 31, "list_windows", serde_json::json!({
+ "pid": pid as i64
+ }));
+ let found = wins["result"]["structuredContent"]["windows"].as_array()
+ .and_then(|a| a.iter().find_map(|w| {
+ let title = w["title"].as_str().unwrap_or("");
+ let id = w["window_id"].as_u64();
+ if id != Some(wid)
+ && (title.contains("Find") || title.contains("Replace"))
+ { id } else { None }
+ }));
+ if let Some(id) = found { break id; }
+ if std::time::Instant::now() > deadline {
+ panic!("Find & Replace dialog did not appear after Ctrl+H — \
+ Writer accelerator may have regressed under \
+ foreground SendInput on SALFRAME.");
+ }
+ std::thread::sleep(Duration::from_millis(300));
+ }
+ };
+
+ // Snapshot the dialog — MSAA walker should produce a real tree
+ // with the dialog's buttons and edit fields, NOT the old skip
+ // stub.
+ let dialog_ax = call(stdin, stdout, 32, "get_window_state", serde_json::json!({
+ "pid": pid as i64, "window_id": dialog_wid, "capture_mode": "ax"
+ }));
+ let dialog_text = snapshot_text(&dialog_ax);
+ assert!(!dialog_text.contains("SAL/VCL target, UIA walk skipped"),
+ "Got the old skip-stub message — MSAA fallback did not engage on \
+ this SALSUBFRAME. Snapshot was: {dialog_text:?}");
+ assert!(dialog_text.contains("Button \"Close\""),
+ "Expected MSAA walker to expose a 'Close' Button in the Find & Replace \
+ dialog tree (it's a documented child via accChild). Snapshot was: {dialog_text:?}");
+ // Sanity: should have several actionable element_indices, not zero.
+ let actionable_count = dialog_text.lines()
+ .filter(|l| l.contains("actions=[invoke"))
+ .count();
+ assert!(actionable_count >= 4,
+ "Expected ≥4 actionable elements in Find & Replace via MSAA walker, \
+ got {actionable_count}. Tree: {dialog_text:?}");
+
+ // Foreground Escape should close the dialog.
+ let esc_resp = call(stdin, stdout, 33, "press_key", serde_json::json!({
+ "pid": pid as i64, "window_id": dialog_wid,
+ "key": "escape", "dispatch": "foreground"
+ }));
+ let esc_text = esc_resp["result"]["content"][0]["text"].as_str().unwrap_or("");
+ assert!(esc_text.starts_with("✅"),
+ "press_key(escape, foreground) returned an error: {esc_text:?}");
+ std::thread::sleep(Duration::from_millis(700));
+
+ let wins_after = call(stdin, stdout, 34, "list_windows", serde_json::json!({
+ "pid": pid as i64
+ }));
+ let still_open = wins_after["result"]["structuredContent"]["windows"].as_array()
+ .map(|a| a.iter().any(|w| w["window_id"].as_u64() == Some(dialog_wid)))
+ .unwrap_or(false);
+ assert!(!still_open,
+ "Find & Replace dialog stayed open after foreground Escape — \
+ SAL modal input dispatch regressed. Check uia/mod.rs SAL-class \
+ handling and the press_key SendInput path.");
+
+ // Cleanup: kill the LO process — Drop impl sweeps soffice.bin.
+ let _ = call(stdin, stdout, 35, "kill_app", serde_json::json!({
+ "pid": pid as i64
+ }));
+ });
+}
+
+// ── Test 4: All toolbar SplitButtons expose `expand` ────────────────────────
+
+/// Confirms the MSAA fallback exposes `actions=[invoke,expand]` on EVERY
+/// toolbar SplitButton in LO Writer — not just Font Color.
+///
+/// Pre-MSAA, the UIA→MSAA proxy collapsed every BUTTONDROPDOWN to a
+/// featureless `SplitButton actions=[invoke]`. Post-MSAA, ALL of them
+/// (Save, Open, Undo, Redo, Paste, Table, Field, Symbol, Show Tracked
+/// Changes, Record, Basic Shapes, Underline, Font Color, Character
+/// Highlighting Color, Background Color, Unordered List, Ordered List,
+/// Outline Format, Line Spacing, Character Spacing — plus the "System"
+/// menu button) report `expand`. Guards against accidental regressions
+/// in `msaa::actions_for` (e.g. dropping one of the BUTTONDROPDOWN
+/// variants from the role match).
+///
+/// Asserts ≥15 SplitButtons report `expand` (current count is 20-21
+/// depending on toolbar config; floor at 15 to absorb minor LO updates).
+#[test]
+#[ignore]
+fn harness_lo_vcl_all_toolbar_split_buttons_expose_expand() {
+ let mut fx = match setup() { Some(s) => s, None => return };
+ with_session(&mut fx, |stdin, stdout, pid, wid| {
+ let snap = call(stdin, stdout, 30, "get_window_state", serde_json::json!({
+ "pid": pid as i64, "window_id": wid, "capture_mode": "ax"
+ }));
+ let text = snapshot_text(&snap);
+ let splitbutton_lines: Vec<&str> = text
+ .lines()
+ .filter(|l| l.contains("SplitButton") && l.contains("actions=["))
+ .collect();
+ let with_expand: Vec<&&str> = splitbutton_lines.iter()
+ .filter(|l| l.contains("expand"))
+ .collect();
+ assert!(splitbutton_lines.len() >= 15,
+ "Expected ≥15 SplitButtons in LO Writer toolbar tree, got {}. \
+ Possible MSAA walker regression. Full snapshot first 1500 chars: {}",
+ splitbutton_lines.len(),
+ &text.chars().take(1500).collect::<String>());
+ assert_eq!(with_expand.len(), splitbutton_lines.len(),
+ "{}/{} SplitButtons report `expand`. The MSAA role→actions \
+ mapping may have dropped one of BUTTONDROPDOWN / BUTTONMENU / \
+ BUTTONDROPDOWNGRID / SPLITBUTTON. Missing-expand lines:\n{}",
+ with_expand.len(), splitbutton_lines.len(),
+ splitbutton_lines.iter()
+ .filter(|l| !l.contains("expand"))
+ .map(|l| l.trim())
+ .collect::<Vec<_>>()
+ .join("\n"));
+ });
+}
+
+// ── Test 5: End-to-end color-pick ───────────────────────────────────────────
+
+/// End-to-end regression for the headline workflow: type text, select it,
+/// open the Font Color picker via `action:"expand"`, click a named color,
+/// verify the picker closes (the canonical signal that LO applied the
+/// color).
+///
+/// Failure points:
+/// - MSAA walker can't find Font Color → `msaa.rs` role match broken.
+/// - click(action:"expand") doesn't open picker → right-edge dispatch
+/// offset wrong for current LO version's toolbar scale.
+/// - Picker MSAA walk doesn't find "Green" → walker depth/budget
+/// truncating the color grid.
+/// - Picker doesn't close after picking → click on color cell isn't
+/// landing (offset / role mismatch on ListItem).
+#[test]
+#[ignore]
+fn harness_lo_vcl_color_pick_green_end_to_end() {
+ let mut fx = match setup() { Some(s) => s, None => return };
+ with_session(&mut fx, |stdin, stdout, pid, wid| {
+ // Foreground first so type_text + Ctrl+A land on Writer.
+ let _ = call(stdin, stdout, 30, "bring_to_front", serde_json::json!({
+ "pid": pid as i64, "window_id": wid
+ }));
+ std::thread::sleep(Duration::from_millis(400));
+
+ let _ = call(stdin, stdout, 31, "type_text", serde_json::json!({
+ "pid": pid as i64, "window_id": wid,
+ "text": "color pick regression"
+ }));
+ std::thread::sleep(Duration::from_millis(400));
+
+ let _ = call(stdin, stdout, 32, "hotkey", serde_json::json!({
+ "pid": pid as i64, "window_id": wid,
+ "keys": ["ctrl", "a"], "dispatch": "foreground"
+ }));
+ std::thread::sleep(Duration::from_millis(400));
+
+ // Locate Font Color element_index in Writer's MSAA tree.
+ let snap = call(stdin, stdout, 33, "get_window_state", serde_json::json!({
+ "pid": pid as i64, "window_id": wid,
+ "capture_mode": "ax", "query": "Font Color"
+ }));
+ let snap_text = snapshot_text(&snap);
+ let fc_line = snap_text.lines()
+ .find(|l| l.contains("\"Font Color\"") && l.contains("expand"))
+ .unwrap_or_else(|| panic!("Font Color with `expand` not found in tree: {snap_text:?}"));
+ let s = fc_line.find('[').expect("[ in fc_line");
+ let e = fc_line[s..].find(']').expect("] in fc_line") + s;
+ let fc_idx: u64 = fc_line[s+1..e].trim().parse().expect("fc element_index");
+
+ // Snapshot which windows existed BEFORE opening the picker so
+ // we can isolate the picker on appearance.
+ let before = call(stdin, stdout, 34, "list_windows", serde_json::json!({
+ "pid": pid as i64
+ }));
+ let before_ids: std::collections::HashSet<u64> = before
+ ["result"]["structuredContent"]["windows"].as_array()
+ .map(|a| a.iter().filter_map(|w| w["window_id"].as_u64()).collect())
+ .unwrap_or_default();
+
+ // Open the dropdown.
+ let open_resp = call(stdin, stdout, 35, "click", serde_json::json!({
+ "pid": pid as i64, "window_id": wid,
+ "element_index": fc_idx, "action": "expand"
+ }));
+ let open_text = open_resp["result"]["content"][0]["text"].as_str().unwrap_or("");
+ assert!(open_text.contains("dropdown half"),
+ "click(action:expand) did not dispatch via MSAA dropdown path: {open_text:?}");
+ std::thread::sleep(Duration::from_millis(900));
+
+ // Find the picker window.
+ let after = call(stdin, stdout, 36, "list_windows", serde_json::json!({
+ "pid": pid as i64
+ }));
+ let picker_wid = after["result"]["structuredContent"]["windows"]
+ .as_array()
+ .and_then(|a| a.iter().find_map(|w| {
+ let id = w["window_id"].as_u64();
+ let title = w["title"].as_str().unwrap_or("");
+ if id.map(|i| !before_ids.contains(&i)).unwrap_or(false)
+ && title.contains("Font Color")
+ { id } else { None }
+ }))
+ .unwrap_or_else(|| panic!("No new 'Font Color' picker window appeared"));
+
+ // Walk the picker tree to find a green color cell.
+ let psnap = call(stdin, stdout, 37, "get_window_state", serde_json::json!({
+ "pid": pid as i64, "window_id": picker_wid, "capture_mode": "ax"
+ }));
+ let psnap_text = snapshot_text(&psnap);
+ let green_line = psnap_text.lines()
+ .find(|l| l.contains("\"Green\"") && l.contains("[") && l.contains("actions=[invoke"))
+ .unwrap_or_else(|| panic!(
+ "No 'Green' ListItem in picker tree. The MSAA walker may have \
+ truncated the color grid or LO's locale renamed the color. \
+ First 1500 chars: {}",
+ &psnap_text.chars().take(1500).collect::<String>()));
+ let s = green_line.find('[').expect("[ in green");
+ let e = green_line[s..].find(']').expect("] in green") + s;
+ let green_idx: u64 = green_line[s+1..e].trim().parse().expect("green element_index");
+
+ // Pick it. Default action (invoke) clicks center via MSAA dispatch.
+ let pick_resp = call(stdin, stdout, 38, "click", serde_json::json!({
+ "pid": pid as i64, "window_id": picker_wid,
+ "element_index": green_idx
+ }));
+ let pick_text = pick_resp["result"]["content"][0]["text"].as_str().unwrap_or("");
+ assert!(pick_text.starts_with("✅"),
+ "click on Green color cell failed: {pick_text:?}");
+ std::thread::sleep(Duration::from_millis(700));
+
+ // Verify the picker closed (LO closes the popup after a color is
+ // selected — the canonical "color applied" signal). If it
+ // didn't, the click didn't actually land on the cell.
+ let after2 = call(stdin, stdout, 39, "list_windows", serde_json::json!({
+ "pid": pid as i64
+ }));
+ let still_open = after2["result"]["structuredContent"]["windows"]
+ .as_array()
+ .map(|a| a.iter().any(|w| w["window_id"].as_u64() == Some(picker_wid)))
+ .unwrap_or(false);
+ assert!(!still_open,
+ "Picker stayed open after click on Green — the cell click \
+ didn't land. LO didn't apply the color.");
+ });
+}
+
+// ── Test 6: Recovery dialog walks via MSAA ──────────────────────────────────
+
+/// Confirms that the LibreOffice Document Recovery dialog (which used to
+/// be walked via UIA) is still walkable after the MSAA-routes-all-SAL
+/// change. Pre-MSAA: UIA path returned a walkable tree with "Discard
+/// All" / "Recover Selected" buttons addressable. Post-MSAA: same flow
+/// must continue to work through the MSAA walker.
+///
+/// Triggering Recovery requires a previous LO session that exited
+/// ungracefully — this test creates that condition by typing into a
+/// fresh writer and then `kill_app`-ing it, then re-launches. If the
+/// Recovery dialog doesn't appear (LO can't always be coaxed into
+/// recovery state, e.g. when the recovery feature is disabled in
+/// user config), the test logs and returns OK rather than panicking —
+/// it's a regression guard for the post-MSAA Recovery walk, not for
+/// LO's recovery-trigger behavior.
+#[test]
+#[ignore]
+fn harness_lo_vcl_recovery_dialog_walks_via_msaa() {
+ // First: launch Writer WITHOUT -norestore (we WANT recovery
+ // behavior), type something, then force-kill so LO marks the
+ // session as crashed.
+ let writer = match swriter_exe() { Some(p) => p, None => return };
+ let _ = Command::new("taskkill").args(["/F", "/IM", "soffice.bin"]).output();
+ std::thread::sleep(Duration::from_millis(800));
+
+ // Launch (no -norestore, no -nologo flag → default behavior).
+ let mut crash_proc = match Command::new(&writer)
+ .stdout(Stdio::null()).stderr(Stdio::null())
+ .spawn() {
+ Ok(p) => p,
+ Err(_) => return,
+ };
+ std::thread::sleep(Duration::from_secs(5));
+ let _ = crash_proc.kill();
+ let _ = crash_proc.wait();
+ // Force-kill the actual soffice.bin worker too (the launcher is what
+ // we just killed; the real LO process is a fork).
+ let _ = Command::new("taskkill").args(["/F", "/IM", "soffice.bin"]).output();
+ std::thread::sleep(Duration::from_secs(2));
+
+ // Re-launch Writer — Recovery dialog should appear.
+ let driver = driver_binary();
+ if !driver.exists() { return; }
+ let mut writer_proc = match Command::new(&writer)
+ .stdout(Stdio::null()).stderr(Stdio::null())
+ .spawn() {
+ Ok(p) => p,
+ Err(_) => return,
+ };
+ std::thread::sleep(Duration::from_secs(5));
+
+ let mut driver_c = match Command::new(&driver)
+ .stdin(Stdio::piped()).stdout(Stdio::piped()).stderr(Stdio::null())
+ .spawn() {
+ Ok(p) => p,
+ Err(_) => {
+ let _ = writer_proc.kill();
+ return;
+ }
+ };
+ let mut stdin = driver_c.stdin.take().unwrap();
+ let mut stdout_raw = driver_c.stdout.take().unwrap();
+ let mut stdout = BufReader::new(&mut stdout_raw);
+
+ init(&mut stdin, &mut stdout);
+
+ // Poll for the Recovery dialog.
+ let recovery_wid_pid: Option<(u64, u32)> = {
+ let deadline = std::time::Instant::now() + Duration::from_secs(20);
+ let mut found = None;
+ while std::time::Instant::now() < deadline {
+ let resp = call(&mut stdin, &mut stdout, 1, "list_windows", serde_json::json!({}));
+ if let Some(wins) = resp["result"]["structuredContent"]["windows"].as_array() {
+ if let Some(w) = wins.iter().find(|w| {
+ w["title"].as_str().map(|t| t.contains("Recovery")).unwrap_or(false)
+ }) {
+ let id = w["window_id"].as_u64();
+ let pid = w["pid"].as_u64().map(|p| p as u32);
+ if let (Some(i), Some(p)) = (id, pid) {
+ found = Some((i, p));
+ break;
+ }
+ }
+ }
+ std::thread::sleep(Duration::from_millis(500));
+ }
+ found
+ };
+
+ match recovery_wid_pid {
+ None => {
+ eprintln!("Recovery dialog did not appear — LO didn't enter recovery \
+ state (possibly disabled in user config, or our crash \
+ trigger didn't take effect). Test cannot assert MSAA \
+ walk on a dialog that doesn't exist; logging and \
+ returning OK.");
+ }
+ Some((rwid, rpid)) => {
+ let snap = call(&mut stdin, &mut stdout, 2, "get_window_state",
+ serde_json::json!({
+ "pid": rpid as i64, "window_id": rwid, "capture_mode": "ax"
+ }));
+ let text = snapshot_text(&snap);
+ assert!(!text.contains("SAL/VCL target, UIA walk skipped"),
+ "Recovery dialog returned the old SAL-skip stub — MSAA fallback \
+ isn't engaging on SALFRAME Recovery dialogs.");
+ assert!(text.contains("Button \"Discard All\""),
+ "Recovery dialog tree should expose `Discard All` Button via MSAA. \
+ Got: {text:?}");
+ assert!(text.contains("Button \"Recover Selected\""),
+ "Recovery dialog tree should expose `Recover Selected` Button via MSAA. \
+ Got: {text:?}");
+ }
+ }
+
+ // Cleanup.
+ let _ = driver_c.kill(); let _ = driver_c.wait();
+ let _ = writer_proc.kill(); let _ = writer_proc.wait();
+ let _ = Command::new("taskkill").args(["/F", "/IM", "soffice.bin"]).output();
+ std::thread::sleep(Duration::from_millis(800));
+}
+
+// ── Test 7: LO Calc smoke ──────────────────────────────────────────────────
+
+/// Confirms the MSAA fallback generalizes beyond Writer. Launches LO
+/// Calc, walks the main SALFRAME, asserts ≥15 SplitButtons with `expand`
+/// — Calc has Calc-specific dropdowns (Select Function, Row, Column,
+/// Freeze Panes) on top of the standard ones (Save, Open, Undo, Redo,
+/// Paste, Font Color, etc.).
+///
+/// Skips cleanly if scalc.exe isn't installed (set LO_SCALC_EXE for
+/// non-default paths).
+#[test]
+#[ignore]
+fn harness_lo_vcl_calc_msaa_smoke() {
+ let scalc = match scalc_exe() { Some(p) => p, None => return };
+ let driver = driver_binary();
+ if !driver.exists() { return; }
+
+ let _ = Command::new("taskkill").args(["/F", "/IM", "soffice.bin"]).output();
+ std::thread::sleep(Duration::from_millis(800));
+
+ let mut calc_proc = match Command::new(&scalc)
+ .args(["-norestore", "-nologo"])
+ .stdout(Stdio::null()).stderr(Stdio::null())
+ .spawn() {
+ Ok(p) => p,
+ Err(_) => return,
+ };
+ std::thread::sleep(Duration::from_secs(5));
+
+ let mut driver_c = match Command::new(&driver)
+ .stdin(Stdio::piped()).stdout(Stdio::piped()).stderr(Stdio::null())
+ .spawn() {
+ Ok(p) => p,
+ Err(_) => {
+ let _ = calc_proc.kill();
+ return;
+ }
+ };
+ let mut stdin_drv = driver_c.stdin.take().unwrap();
+ let mut stdout_raw = driver_c.stdout.take().unwrap();
+ let mut stdout = BufReader::new(&mut stdout_raw);
+ init(&mut stdin_drv, &mut stdout);
+
+ // Poll for the Calc window.
+ let calc_wid_pid: Option<(u64, u32)> = {
+ let deadline = std::time::Instant::now() + Duration::from_secs(20);
+ let mut found = None;
+ while std::time::Instant::now() < deadline {
+ let r = call(&mut stdin_drv, &mut stdout, 1, "list_windows", serde_json::json!({}));
+ if let Some(wins) = r["result"]["structuredContent"]["windows"].as_array() {
+ if let Some(w) = wins.iter().find(|w| {
+ w["title"].as_str().map(|t| t.contains("LibreOffice Calc")).unwrap_or(false)
+ }) {
+ let id = w["window_id"].as_u64();
+ let pid = w["pid"].as_u64().map(|p| p as u32);
+ if let (Some(i), Some(p)) = (id, pid) {
+ found = Some((i, p));
+ break;
+ }
+ }
+ }
+ std::thread::sleep(Duration::from_millis(500));
+ }
+ found
+ };
+
+ let (cwid, cpid) = calc_wid_pid.unwrap_or_else(|| {
+ let _ = driver_c.kill();
+ let _ = calc_proc.kill();
+ let _ = Command::new("taskkill").args(["/F", "/IM", "soffice.bin"]).output();
+ panic!("LO Calc window did not appear within 20 s");
+ });
+
+ let snap = call(&mut stdin_drv, &mut stdout, 2, "get_window_state",
+ serde_json::json!({
+ "pid": cpid as i64, "window_id": cwid, "capture_mode": "ax"
+ }));
+ let text = snapshot_text(&snap);
+ assert!(!text.contains("SAL/VCL target, UIA walk skipped"),
+ "Calc returned the old SAL-skip stub — MSAA didn't engage.");
+ let sb_with_expand = text.lines()
+ .filter(|l| l.contains("SplitButton") && l.contains("expand"))
+ .count();
+ assert!(sb_with_expand >= 15,
+ "Expected ≥15 SplitButtons with `expand` in Calc toolbar (proves MSAA \
+ generalizes beyond Writer). Got {sb_with_expand}. The MSAA role \
+ mapping may differ in Calc, or the toolbar config is unexpectedly slim.");
+
+ // Cleanup.
+ let _ = call(&mut stdin_drv, &mut stdout, 3, "kill_app", serde_json::json!({
+ "pid": cpid as i64
+ }));
+ let _ = driver_c.kill(); let _ = driver_c.wait();
+ let _ = calc_proc.kill(); let _ = calc_proc.wait();
+ let _ = Command::new("taskkill").args(["/F", "/IM", "soffice.bin"]).output();
+ std::thread::sleep(Duration::from_millis(800));
+}
diff --git a/libs/cua-driver/rust/crates/platform-windows/src/lib.rs b/libs/cua-driver/rust/crates/platform-windows/src/lib.rs
index e56c19188e..f918ea58cc 100644
--- a/libs/cua-driver/rust/crates/platform-windows/src/lib.rs
+++ b/libs/cua-driver/rust/crates/platform-windows/src/lib.rs
@@ -21,6 +21,9 @@ pub mod win32;
#[cfg(target_os = "windows")]
pub mod uia;
+#[cfg(target_os = "windows")]
+pub mod msaa;
+
#[cfg(target_os = "windows")]
pub mod input;
diff --git a/libs/cua-driver/rust/crates/platform-windows/src/msaa.rs b/libs/cua-driver/rust/crates/platform-windows/src/msaa.rs
new file mode 100644
index 0000000000..fb121310d7
--- /dev/null
+++ b/libs/cua-driver/rust/crates/platform-windows/src/msaa.rs
@@ -0,0 +1,323 @@
+//! MSAA (Microsoft Active Accessibility) tree walker.
+//!
+//! Fallback for SAL/VCL window classes (LibreOffice, OpenOffice) where the
+//! UIA walker hangs on `BuildUpdatedCache(Subtree)`. MSAA via oleacc.dll's
+//! `AccessibleObjectFromWindow` + recursive `accChild` walks these windows
+//! cleanly because it doesn't go through the bulk-cache RPC that SAL's UIA
+//! provider deadlocks on under the daemon's MTA pool.
+//!
+//! Bonus payoff: MSAA preserves the `ROLE_SYSTEM_BUTTONDROPDOWN` role (0x38)
+//! that Windows' built-in MSAA→UIA proxy collapses to a featureless
+//! `SplitButton` (no `ExpandCollapse` pattern, no separable dropdown child).
+//! For BUTTONDROPDOWN, this walker emits `actions=["invoke","expand"]` —
+//! the `click` tool routes `action:"expand"` to a right-edge SendInput
+//! click that opens the dropdown half (e.g. LO Writer "Font Color" → color
+//! picker) instead of just re-firing the press half.
+//!
+//! Discovery context: `flash-repro/ia2_pixel_click.ps1` proved the path —
+//! `accLocation("Font Color")` → click `(rect.right - 4, center_y)` →
+//! `SALTMPSUBFRAME 'Font Color'` popup appears.
+
+use std::ptr::null_mut;
+
+use windows::core::{Interface, VARIANT};
+use windows::Win32::Foundation::HWND;
+use windows::Win32::UI::Accessibility::{AccessibleObjectFromWindow, IAccessible};
+
+use crate::uia::{UiaNode, UiaTreeResult};
+
+const OBJID_CLIENT: u32 = 0xFFFFFFFC;
+
+// MSAA role codes (subset; full list in oleacc.h).
+const ROLE_SYSTEM_TITLEBAR: i32 = 0x01;
+const ROLE_SYSTEM_MENUBAR: i32 = 0x02;
+const ROLE_SYSTEM_SCROLLBAR: i32 = 0x03;
+const ROLE_SYSTEM_WINDOW: i32 = 0x09;
+const ROLE_SYSTEM_CLIENT: i32 = 0x0A;
+const ROLE_SYSTEM_MENUPOPUP: i32 = 0x0B;
+const ROLE_SYSTEM_MENUITEM: i32 = 0x0C;
+const ROLE_SYSTEM_TOOLTIP: i32 = 0x0D;
+const ROLE_SYSTEM_DIALOG: i32 = 0x12;
+const ROLE_SYSTEM_GROUPING: i32 = 0x14;
+const ROLE_SYSTEM_TOOLBAR: i32 = 0x16;
+const ROLE_SYSTEM_STATUSBAR: i32 = 0x17;
+const ROLE_SYSTEM_LINK: i32 = 0x1E;
+const ROLE_SYSTEM_LIST: i32 = 0x21;
+const ROLE_SYSTEM_LISTITEM: i32 = 0x22;
+const ROLE_SYSTEM_PAGETAB: i32 = 0x25;
+const ROLE_SYSTEM_GRAPHIC: i32 = 0x28;
+const ROLE_SYSTEM_STATICTEXT: i32 = 0x29;
+const ROLE_SYSTEM_TEXT: i32 = 0x2A;
+const ROLE_SYSTEM_PUSHBUTTON: i32 = 0x2B;
+const ROLE_SYSTEM_CHECKBUTTON: i32 = 0x2C;
+const ROLE_SYSTEM_RADIOBUTTON: i32 = 0x2D;
+const ROLE_SYSTEM_COMBOBOX: i32 = 0x2E;
+const ROLE_SYSTEM_PROGRESSBAR: i32 = 0x30;
+const ROLE_SYSTEM_SLIDER: i32 = 0x33;
+const ROLE_SYSTEM_BUTTONDROPDOWN: i32 = 0x38;
+const ROLE_SYSTEM_BUTTONMENU: i32 = 0x39;
+const ROLE_SYSTEM_BUTTONDROPDOWNGRID: i32 = 0x3A;
+const ROLE_SYSTEM_PAGETABLIST: i32 = 0x3C;
+const ROLE_SYSTEM_SPLITBUTTON: i32 = 0x3E;
+
+const MAX_DEPTH: usize = 25;
+const MAX_TOTAL_ELEMENTS: usize = 5000;
+
+/// Walk the MSAA tree for the window with the given HWND. Used as fallback
+/// for SAL/VCL targets where the UIA walker would hang.
+pub fn walk_msaa_tree(hwnd: u64) -> UiaTreeResult {
+ unsafe { walk_unsafe(hwnd) }
+}
+
+unsafe fn walk_unsafe(hwnd: u64) -> UiaTreeResult {
+ let hwnd_win = HWND(hwnd as *mut _);
+ let mut raw_root: *mut std::ffi::c_void = null_mut();
+ // AccessibleObjectFromWindow returns IAccessible (as `*mut c_void` typed
+ // via the IID we pass). We pass IID_IAccessible.
+ let iid_iaccessible = IAccessible::IID;
+ let hr = AccessibleObjectFromWindow(
+ hwnd_win,
+ OBJID_CLIENT,
+ &iid_iaccessible,
+ &mut raw_root as *mut _ as *mut _,
+ );
+ if hr.is_err() || raw_root.is_null() {
+ return UiaTreeResult {
+ tree_markdown: format!(
+ "- Window <SAL/VCL — MSAA fallback failed (AccessibleObjectFromWindow hr={hr:?})>\n"
+ ),
+ nodes: Vec::new(),
+ };
+ }
+ let root: IAccessible = IAccessible::from_raw(raw_root);
+
+ let mut nodes: Vec<UiaNode> = Vec::new();
+ let mut lines: Vec<(usize, String)> = Vec::new();
+ let mut counter = 0usize;
+ let mut total = 0usize;
+
+ walk(&root, 0, &mut nodes, &mut lines, &mut counter, &mut total);
+
+ let tree_markdown = render_lines(&lines);
+ UiaTreeResult { tree_markdown, nodes }
+}
+
+unsafe fn walk(
+ acc: &IAccessible,
+ depth: usize,
+ nodes: &mut Vec<UiaNode>,
+ lines: &mut Vec<(usize, String)>,
+ counter: &mut usize,
+ total: &mut usize,
+) {
+ if depth >= MAX_DEPTH || *total >= MAX_TOTAL_ELEMENTS {
+ return;
+ }
+ *total += 1;
+
+ let self_var: VARIANT = VARIANT::from(0i32); // CHILDID_SELF
+
+ // Properties — each call wrapped to swallow per-element COM errors.
+ let role_int: Option<i32> = acc
+ .get_accRole(&self_var)
+ .ok()
+ .and_then(|v| variant_to_i32(&v));
+ let name: Option<String> = acc
+ .get_accName(&self_var)
+ .ok()
+ .map(|b| b.to_string())
+ .filter(|s| !s.trim().is_empty());
+ let default_action: Option<String> = acc
+ .get_accDefaultAction(&self_var)
+ .ok()
+ .map(|b| b.to_string())
+ .filter(|s| !s.trim().is_empty());
+
+ // accLocation: out left, top, width, height (screen coords).
+ let rect: Option<(i32, i32, i32, i32)> = {
+ let mut l = 0i32;
+ let mut t = 0i32;
+ let mut w = 0i32;
+ let mut h = 0i32;
+ if acc
+ .accLocation(&mut l, &mut t, &mut w, &mut h, &self_var)
+ .is_ok()
+ && w > 0
+ && h > 0
+ {
+ Some((l, t, l + w, t + h))
+ } else {
+ None
+ }
+ };
+
+ let control_type = role_to_control_type(role_int.unwrap_or(0));
+ let actions = actions_for(role_int.unwrap_or(0), default_action.as_deref());
+ let is_actionable = !actions.is_empty();
+ let has_content = name.is_some();
+
+ if is_actionable || has_content {
+ // Retain the IAccessible pointer for the cache. Mirror what the UIA
+ // walker does: clone, get raw, forget the local — the cache's Drop
+ // releases via IUnknown.
+ let retained: IAccessible = acc.clone();
+ let ptr = retained.as_raw() as usize;
+ std::mem::forget(retained);
+
+ let (center_x, center_y) = rect
+ .map(|(l, t, r, b)| ((l + r) / 2, (t + b) / 2))
+ .unwrap_or((0, 0));
+
+ let node = if is_actionable {
+ let idx = *counter;
+ *counter += 1;
+ UiaNode {
+ element_index: Some(idx),
+ control_type: control_type.clone(),
+ name: name.clone(),
+ value: None,
+ automation_id: None,
+ help_text: None,
+ actions: actions.clone(),
+ element_ptr: ptr,
+ center_x,
+ center_y,
+ rect,
+ msaa_role: role_int,
+ }
+ } else {
+ UiaNode {
+ element_index: None,
+ control_type: control_type.clone(),
+ name: name.clone(),
+ value: None,
+ automation_id: None,
+ help_text: None,
+ actions: Vec::new(),
+ element_ptr: ptr,
+ center_x: 0,
+ center_y: 0,
+ rect,
+ msaa_role: role_int,
+ }
+ };
+ lines.push((depth, crate::uia::format_node_line(&node)));
+ nodes.push(node);
+ }
+
+ // Recurse via accChildCount + get_accChild.
+ let child_count: i32 = acc.accChildCount().unwrap_or(0);
+ for i in 1..=child_count {
+ let child_var = VARIANT::from(i);
+ // accChild returns IDispatch — query for IAccessible.
+ if let Ok(child_disp) = acc.get_accChild(&child_var) {
+ if let Ok(child_acc) = child_disp.cast::<IAccessible>() {
+ walk(&child_acc, depth + 1, nodes, lines, counter, total);
+ }
+ }
+ }
+}
+
+/// Extract i32 from a VARIANT — accRole returns VT_I4 in practice (sometimes
+/// VT_BSTR for custom roles, which we map to 0 = unknown).
+unsafe fn variant_to_i32(v: &VARIANT) -> Option<i32> {
+ let raw = v.as_raw();
+ let vt = raw.Anonymous.Anonymous.vt;
+ if vt == 3 {
+ // VT_I4
+ Some(raw.Anonymous.Anonymous.Anonymous.lVal)
+ } else {
+ None
+ }
+}
+
+/// Map MSAA role id → control_type string. For roles not in this list we
+/// emit `Role_<hex>` so the agent at least sees something diagnostic.
+fn role_to_control_type(role: i32) -> String {
+ match role {
+ ROLE_SYSTEM_TITLEBAR => "TitleBar",
+ ROLE_SYSTEM_MENUBAR => "MenuBar",
+ ROLE_SYSTEM_SCROLLBAR => "ScrollBar",
+ ROLE_SYSTEM_WINDOW => "Window",
+ ROLE_SYSTEM_CLIENT => "Pane",
+ ROLE_SYSTEM_MENUPOPUP => "Menu",
+ ROLE_SYSTEM_MENUITEM => "MenuItem",
+ ROLE_SYSTEM_TOOLTIP => "ToolTip",
+ ROLE_SYSTEM_DIALOG => "Window",
+ ROLE_SYSTEM_GROUPING => "Group",
+ ROLE_SYSTEM_TOOLBAR => "ToolBar",
+ ROLE_SYSTEM_STATUSBAR => "StatusBar",
+ ROLE_SYSTEM_LINK => "Hyperlink",
+ ROLE_SYSTEM_LIST => "List",
+ ROLE_SYSTEM_LISTITEM => "ListItem",
+ ROLE_SYSTEM_PAGETAB => "TabItem",
+ ROLE_SYSTEM_PAGETABLIST => "Tab",
+ ROLE_SYSTEM_GRAPHIC => "Image",
+ ROLE_SYSTEM_STATICTEXT => "Text",
+ ROLE_SYSTEM_TEXT => "Edit",
+ ROLE_SYSTEM_PUSHBUTTON => "Button",
+ ROLE_SYSTEM_CHECKBUTTON => "CheckBox",
+ ROLE_SYSTEM_RADIOBUTTON => "RadioButton",
+ ROLE_SYSTEM_COMBOBOX => "ComboBox",
+ ROLE_SYSTEM_PROGRESSBAR => "ProgressBar",
+ ROLE_SYSTEM_SLIDER => "Slider",
+ ROLE_SYSTEM_BUTTONDROPDOWN
+ | ROLE_SYSTEM_BUTTONMENU
+ | ROLE_SYSTEM_BUTTONDROPDOWNGRID
+ | ROLE_SYSTEM_SPLITBUTTON => "SplitButton",
+ 0 => "Unknown",
+ other => return format!("Role_0x{:X}", other),
+ }
+ .into()
+}
+
+/// Compute `actions=[...]` for a MSAA element. Roles with a meaningful
+/// default action get `invoke`. Dropdown-flavored roles ALSO get `expand`
+/// so callers can address the dropdown half separately — the click tool
+/// routes `action:"expand"` to a right-edge SendInput click rather than
+/// just calling `accDoDefaultAction` (which fires the press half).
+fn actions_for(role: i32, default_action: Option<&str>) -> Vec<String> {
+ let has_action = default_action
+ .map(|s| !s.trim().is_empty())
+ .unwrap_or(false);
+ let mut actions = Vec::new();
+
+ let is_dropdown = matches!(
+ role,
+ ROLE_SYSTEM_BUTTONDROPDOWN
+ | ROLE_SYSTEM_BUTTONMENU
+ | ROLE_SYSTEM_BUTTONDROPDOWNGRID
+ | ROLE_SYSTEM_SPLITBUTTON
+ );
+ let is_clickable = matches!(
+ role,
+ ROLE_SYSTEM_PUSHBUTTON
+ | ROLE_SYSTEM_CHECKBUTTON
+ | ROLE_SYSTEM_RADIOBUTTON
+ | ROLE_SYSTEM_LINK
+ | ROLE_SYSTEM_MENUITEM
+ | ROLE_SYSTEM_LISTITEM
+ | ROLE_SYSTEM_PAGETAB
+ | ROLE_SYSTEM_COMBOBOX
+ );
+
+ if has_action || is_dropdown || is_clickable {
+ actions.push("invoke".into());
+ }
+ if is_dropdown {
+ actions.push("expand".into());
+ }
+ actions
+}
+
+fn render_lines(lines: &[(usize, String)]) -> String {
+ let mut out = String::new();
+ for (depth, line) in lines {
+ for _ in 0..*depth {
+ out.push_str(" ");
+ }
+ out.push_str(line);
+ out.push('\n');
+ }
+ out
+}
diff --git a/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs b/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs
index 428ea0211b..af122cec36 100644
--- a/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs
+++ b/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs
@@ -669,7 +669,16 @@ impl Tool for GetWindowStateTool {
let count = tr.nodes.iter().filter(|n| n.element_index.is_some()).count();
let header = format!("window_id={hwnd} pid={pid} elements={count}\n\n");
content.push(mcp_server::protocol::Content::text(header + &tr.tree_markdown));
- state.element_cache.update(pid, hwnd, &tr.nodes);
+ // Route the cache to the matching dispatch path: any
+ // node whose msaa_role is Some came from the MSAA
+ // walker, so the entire snapshot must Drop via
+ // IAccessible and click must dispatch through MSAA.
+ let is_msaa = tr.nodes.iter().any(|n| n.msaa_role.is_some());
+ if is_msaa {
+ state.element_cache.update_msaa(pid, hwnd, &tr.nodes);
+ } else {
+ state.element_cache.update(pid, hwnd, &tr.nodes);
+ }
structured["element_count"] = json!(count);
structured["tree_markdown"] = json!(tr.tree_markdown);
}
@@ -1775,6 +1784,7 @@ impl Tool for ClickTool {
async fn invoke(&self, args: Value) -> ToolResult {
use mcp_server::tool_args::ArgsExt;
use crate::input::dispatch::{DispatchMode, EventKind, background_unavailable_error};
+ use crate::uia::cache::SnapshotKind;
let pid = match args.require_u32("pid") { Ok(v) => v, Err(e) => return e };
let hwnd_opt = args.opt_u64("window_id");
let elem_idx = args.opt_u64("element_index").map(|v| v as usize);
@@ -1783,6 +1793,13 @@ impl Tool for ClickTool {
let button = args.str_or("button", "left");
let count = args.u64_or("count", 1) as usize;
let dispatch = DispatchMode::from_args(&args);
+ // Optional `action` arg picks among the actions exposed in the
+ // accessibility tree. Today this only changes behavior for MSAA
+ // BUTTONDROPDOWN: `"expand"` clicks the right-edge (dropdown arrow
+ // half) instead of the center (press half). Defaults to first
+ // action in the element's `actions=[...]` list, which preserves
+ // existing semantics for UIA elements.
+ let action_req = args.get("action").and_then(|v| v.as_str()).map(|s| s.to_string());
// Resolve HWND: explicit, or auto from pid.
let hwnd = match hwnd_opt {
@@ -1797,6 +1814,84 @@ impl Tool for ClickTool {
};
if let Some(idx) = elem_idx {
+ // ── MSAA dispatch (SAL/VCL targets) ────────────────────────────
+ // For MSAA elements, route by role:
+ // - `action:"expand"` on BUTTONDROPDOWN → SendInput at the
+ // cached rect's right-edge (the dropdown arrow half).
+ // Opens the picker (e.g. LO Writer Font Color → SALTMPSUBFRAME).
+ // - default action / `"invoke"` → SendInput at center
+ // (matches `accDoDefaultAction` semantics for VCL controls
+ // and works through dispatch:"foreground"). We DO NOT call
+ // `accDoDefaultAction` here because LO's MSAA impl applies
+ // the change asynchronously and returns S_OK either way,
+ // so the visible behavior is the same as a center click.
+ if let Some((SnapshotKind::Msaa, role)) =
+ self.state.element_cache.get_element_kind_and_role(pid, hwnd, idx)
+ {
+ const ROLE_BUTTONDROPDOWN: i32 = 0x38;
+ const ROLE_BUTTONMENU: i32 = 0x39;
+ const ROLE_BUTTONDROPDOWNGRID: i32 = 0x3A;
+ const ROLE_SPLITBUTTON: i32 = 0x3E;
+ let want_expand = action_req.as_deref() == Some("expand");
+ let is_dropdown_role = matches!(
+ role,
+ Some(ROLE_BUTTONDROPDOWN | ROLE_BUTTONMENU | ROLE_BUTTONDROPDOWNGRID | ROLE_SPLITBUTTON)
+ );
+ let (tx, ty) = if want_expand && is_dropdown_role {
+ match self.state.element_cache.get_element_rect(pid, hwnd, idx) {
+ Some((_l, t, r, b)) => {
+ // Right-edge of the SplitButton — the dropdown
+ // arrow half. -4 puts the click safely inside
+ // the arrow region for the typical ~12-16 px
+ // arrow width VCL uses.
+ (r - 4, (t + b) / 2)
+ }
+ None => {
+ return ToolResult::error(format!(
+ "MSAA element [{idx}] has no cached rect — \
+ cannot dispatch action:\"expand\". Re-run \
+ get_window_state to refresh the cache."
+ ));
+ }
+ }
+ } else if want_expand && !is_dropdown_role {
+ return ToolResult::error(format!(
+ "action:\"expand\" requested for MSAA element [{idx}] \
+ but its role ({role:?}) is not a dropdown button. \
+ Use action:\"invoke\" or omit the action arg."
+ ));
+ } else {
+ match self.state.element_cache.get_element_center(pid, hwnd, idx) {
+ Some(v) => v,
+ None => return ToolResult::error(format!(
+ "MSAA element [{idx}] not in cache for hwnd={hwnd}. \
+ Call get_window_state first."
+ )),
+ }
+ };
+ pin_overlay_above(hwnd);
+ overlay_glide_to(tx as f64, ty as f64).await;
+ crate::overlay::send_command(cursor_overlay::OverlayCommand::ClickPulse {
+ x: tx as f64, y: ty as f64,
+ });
+ let btn_fg = button.clone();
+ let prev_fg_addr = unsafe {
+ windows::Win32::UI::WindowsAndMessaging::GetForegroundWindow().0 as usize
+ };
+ let send_result = tokio::task::spawn_blocking(move || {
+ crate::input::send_click_synthesized(hwnd, tx, ty, count, &btn_fg)
+ }).await;
+ tokio::spawn(restore_foreground_polling_best_effort(prev_fg_addr, pid));
+ let half = if want_expand { "dropdown" } else { "press" };
+ return match send_result {
+ Ok(Ok(())) => ToolResult::text(format!(
+ "✅ Performed SendInput click on MSAA [{idx}] {half} half at ({tx},{ty})."
+ )),
+ Ok(Err(e)) => ToolResult::error(e.to_string()),
+ Err(e) => ToolResult::error(format!("Task error: {e}")),
+ };
+ }
+
// UIA path: get cached center (no COM call needed — captured at walk time).
let (cx, cy) = match self.state.element_cache.get_element_center(pid, hwnd, idx) {
Some(v) => v,
diff --git a/libs/cua-driver/rust/crates/platform-windows/src/uia/cache.rs b/libs/cua-driver/rust/crates/platform-windows/src/uia/cache.rs
index fbf2dfc509..02e5705db7 100644
--- a/libs/cua-driver/rust/crates/platform-windows/src/uia/cache.rs
+++ b/libs/cua-driver/rust/crates/platform-windows/src/uia/cache.rs
@@ -1,11 +1,12 @@
-//! UIA element cache for Windows.
+//! Accessibility element cache for Windows.
//!
//! Mirrors the macOS ElementCache design: per-(pid, window_id) snapshots of
-//! retained IUIAutomationElement COM pointers.
+//! retained COM pointers (either `IUIAutomationElement` for the UIA primary
+//! path, or `IAccessible` for the MSAA fallback used on SAL/VCL targets).
//!
-//! Memory contract: UiaNode::element_ptr is a raw IUIAutomationElement vtable
-//! pointer with an extra AddRef from clone()+forget() in the walker. Drop here
-//! calls Release() to balance.
+//! Memory contract: `UiaNode::element_ptr` is a raw COM vtable pointer with
+//! an extra AddRef from `clone()+forget()` in the walker. `CachedSnapshot::Drop`
+//! calls `Release()` via the kind-appropriate vtable to balance.
//!
//! The locked-HashMap plumbing lives in `mcp_server::element_cache` — see
//! `docs/dedup-audit.md` item #3. This module owns the Windows-specific
@@ -14,8 +15,8 @@
use super::UiaNode;
use mcp_server::element_cache::ElementCacheCore;
-use windows::Win32::UI::Accessibility::IUIAutomationElement;
use windows::core::Interface;
+use windows::Win32::UI::Accessibility::{IAccessible, IUIAutomationElement};
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub struct CacheKey {
@@ -23,21 +24,52 @@ pub struct CacheKey {
pub hwnd: u64,
}
+/// Which COM interface the cached pointers reference. Determines the
+/// `from_raw` cast `Drop` performs and the dispatch path the click tool
+/// takes when an element is invoked.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum SnapshotKind {
+ /// `element_ptr` values are `IUIAutomationElement` pointers; click
+ /// tool tries Invoke/Toggle/SelectionItem/ExpandCollapse patterns.
+ Uia,
+ /// `element_ptr` values are `IAccessible` pointers (MSAA fallback for
+ /// SAL/VCL classes); click tool dispatches via `accDoDefaultAction`
+ /// for invoke, right-edge SendInput for expand.
+ Msaa,
+}
+
pub struct CachedSnapshot {
- /// element_index → raw IUIAutomationElement pointer (retained).
+ pub kind: SnapshotKind,
+ /// element_index → raw COM pointer (retained).
pub elements: Vec<usize>,
/// element_index → screen-coordinate center (captured at walk time).
pub centers: Vec<(i32, i32)>,
+ /// element_index → screen-coordinate rect (l, t, r, b). `None` when
+ /// the element didn't report a meaningful bounding box.
+ pub rects: Vec<Option<(i32, i32, i32, i32)>>,
+ /// element_index → MSAA role code (`Some` for MSAA-walked entries,
+ /// `None` for UIA-walked entries). The click tool reads this to gate
+ /// right-edge dispatch on `ROLE_SYSTEM_BUTTONDROPDOWN` etc.
+ pub msaa_roles: Vec<Option<i32>>,
}
impl Drop for CachedSnapshot {
fn drop(&mut self) {
for ptr in &self.elements {
- if *ptr != 0 {
- // Reconstruct an interface reference and drop it (calls Release).
- unsafe {
- let iface: IUIAutomationElement = IUIAutomationElement::from_raw(*ptr as *mut _);
- drop(iface);
+ if *ptr == 0 {
+ continue;
+ }
+ unsafe {
+ match self.kind {
+ SnapshotKind::Uia => {
+ let iface: IUIAutomationElement =
+ IUIAutomationElement::from_raw(*ptr as *mut _);
+ drop(iface);
+ }
+ SnapshotKind::Msaa => {
+ let iface: IAccessible = IAccessible::from_raw(*ptr as *mut _);
+ drop(iface);
+ }
}
}
}
@@ -50,14 +82,34 @@ pub struct ElementCache {
impl ElementCache {
pub fn new() -> Self {
- Self { core: ElementCacheCore::new() }
+ Self {
+ core: ElementCacheCore::new(),
+ }
}
+ /// Update the snapshot for (pid, hwnd) with the actionable elements
+ /// from a UIA walk. Mirrors `update_msaa` for the MSAA path.
pub fn update(&self, pid: u32, hwnd: u64, nodes: &[UiaNode]) {
+ self.update_with_kind(pid, hwnd, nodes, SnapshotKind::Uia);
+ }
+
+ /// Same as `update` but tags the snapshot as MSAA so Drop releases the
+ /// pointers as `IAccessible` and the click tool routes through the
+ /// MSAA dispatch path.
+ pub fn update_msaa(&self, pid: u32, hwnd: u64, nodes: &[UiaNode]) {
+ self.update_with_kind(pid, hwnd, nodes, SnapshotKind::Msaa);
+ }
+
+ fn update_with_kind(&self, pid: u32, hwnd: u64, nodes: &[UiaNode], kind: SnapshotKind) {
let actionable: Vec<&UiaNode> = nodes.iter().filter(|n| n.element_index.is_some()).collect();
let elements: Vec<usize> = actionable.iter().map(|n| n.element_ptr).collect();
let centers: Vec<(i32, i32)> = actionable.iter().map(|n| (n.center_x, n.center_y)).collect();
- self.core.insert(CacheKey { pid, hwnd }, CachedSnapshot { elements, centers });
+ let rects: Vec<Option<(i32, i32, i32, i32)>> = actionable.iter().map(|n| n.rect).collect();
+ let msaa_roles: Vec<Option<i32>> = actionable.iter().map(|n| n.msaa_role).collect();
+ self.core.insert(
+ CacheKey { pid, hwnd },
+ CachedSnapshot { kind, elements, centers, rects, msaa_roles },
+ );
}
pub fn get_element_ptr(&self, pid: u32, hwnd: u64, element_index: usize) -> Option<usize> {
@@ -72,6 +124,32 @@ impl ElementCache {
.flatten()
}
+ /// Cached screen rect for the element. Used by the click tool to
+ /// compute the right-edge dispatch point for `action:"expand"` on
+ /// MSAA BUTTONDROPDOWN.
+ pub fn get_element_rect(&self, pid: u32, hwnd: u64, element_index: usize) -> Option<(i32, i32, i32, i32)> {
+ self.core
+ .with_snapshot(&CacheKey { pid, hwnd }, |s| s.rects.get(element_index).copied().flatten())
+ .flatten()
+ }
+
+ /// Kind + MSAA role for the element. `(Msaa, Some(0x38))` identifies a
+ /// MSAA BUTTONDROPDOWN; click tool routes `action:"expand"` to
+ /// right-edge SendInput for these.
+ pub fn get_element_kind_and_role(
+ &self,
+ pid: u32,
+ hwnd: u64,
+ element_index: usize,
+ ) -> Option<(SnapshotKind, Option<i32>)> {
+ self.core
+ .with_snapshot(&CacheKey { pid, hwnd }, |s| {
+ let role = s.msaa_roles.get(element_index).copied().flatten();
+ Some((s.kind, role))
+ })
+ .flatten()
+ }
+
pub fn element_count(&self, pid: u32, hwnd: u64) -> usize {
self.core
.with_snapshot(&CacheKey { pid, hwnd }, |s| s.elements.len())
@@ -80,5 +158,7 @@ impl ElementCache {
}
impl Default for ElementCache {
- fn default() -> Self { Self::new() }
+ fn default() -> Self {
+ Self::new()
+ }
}
diff --git a/libs/cua-driver/rust/crates/platform-windows/src/uia/mod.rs b/libs/cua-driver/rust/crates/platform-windows/src/uia/mod.rs
index 76e9dacf21..b58157539e 100644
--- a/libs/cua-driver/rust/crates/platform-windows/src/uia/mod.rs
+++ b/libs/cua-driver/rust/crates/platform-windows/src/uia/mod.rs
@@ -32,7 +32,11 @@ pub use windows_enum::enumerate_top_level_windows;
const MAX_DEPTH: usize = 25;
const MAX_TOTAL_ELEMENTS: usize = 5000;
-/// A single node in the UIA tree.
+/// A single node in the accessibility tree.
+///
+/// Same shape for the UIA primary path AND the MSAA fallback (used for
+/// SAL/VCL window classes — see `msaa.rs`). MSAA-only fields use the
+/// `_ptr is IAccessible` / `msaa_role = Some(...)` discriminator.
#[derive(Clone)]
pub struct UiaNode {
pub element_index: Option<usize>,
@@ -42,11 +46,22 @@ pub struct UiaNode {
pub automation_id: Option<String>,
pub help_text: Option<String>,
pub actions: Vec<String>,
- /// Raw IUIAutomationElement pointer as usize (retained for cache).
+ /// Raw COM pointer (IUIAutomationElement for UIA path, IAccessible for
+ /// MSAA path) as usize. Retained — `ElementCache` Drop releases it via
+ /// the `kind`-appropriate vtable.
pub element_ptr: usize,
/// Screen-coordinate center, captured at walk time to avoid later COM calls.
pub center_x: i32,
pub center_y: i32,
+ /// Full screen-coord rect (left, top, right, bottom). Available for
+ /// elements that report a meaningful bounding box. Used by the click
+ /// tool when `action:"expand"` needs the right-edge offset.
+ pub rect: Option<(i32, i32, i32, i32)>,
+ /// MSAA role code (e.g. 0x38 = `ROLE_SYSTEM_BUTTONDROPDOWN`). `Some`
+ /// iff this node came from the MSAA walker — the click tool checks
+ /// this to route `action:"expand"` to a right-edge SendInput click
+ /// instead of an unsupported UIA pattern lookup.
+ pub msaa_role: Option<i32>,
}
pub struct UiaTreeResult {
@@ -117,55 +132,33 @@ unsafe fn walk_tree_unsafe(hwnd: u64, query: Option<&str>) -> UiaTreeResult {
let hwnd_win = windows::Win32::Foundation::HWND(hwnd as *mut _);
- // SAL (LibreOffice / OpenOffice) fast-path: VCL's UIA provider hangs
- // on `BuildUpdatedCache(TreeScope.Subtree)` for transient dialogs
- // (SALSUBFRAME class — Confirmation, Warning, modal Yes/No). Both
- // the atomic `ElementFromHandleBuildCache` and the split
- // `ElementFromHandle + BuildUpdatedCache(Subtree)` paths block
- // indefinitely under the daemon's MTA thread pool (the same code
- // runs fine via CLI in-process, suggesting a COM apartment /
- // concurrent-access interaction with SAL's provider). The 4 s
- // outer timeout fires and the caller gets the structured
- // diagnostic, but that's a wasted 4 s every call.
+ // SAL (LibreOffice / OpenOffice) fallback: ALL SAL-class windows go
+ // through the MSAA walker.
+ //
+ // Two reasons:
+ // 1. **Hang avoidance** for SALSUBFRAME / SALMENU / SALTMPSUBFRAME —
+ // VCL's UIA provider hangs on `BuildUpdatedCache(TreeScope.Subtree)`
+ // under the daemon's MTA pool, wasting the 4 s outer timeout.
+ // 2. **Role fidelity** for SALFRAME (main document window, Recovery
+ // dialog) — UIA technically walks fine, but the built-in
+ // MSAA→UIA proxy collapses `ROLE_SYSTEM_BUTTONDROPDOWN` (0x38) to
+ // a featureless `SplitButton` with no separable dropdown
+ // affordance. MSAA via oleacc.dll preserves the role, letting
+ // the click tool route `action:"expand"` to a right-edge
+ // SendInput click that opens the dropdown half (e.g. LO Writer
+ // "Font Color" → SALTMPSUBFRAME color picker).
//
- // Short-circuit by class detection: if the target's class starts
- // with "SAL", skip the bulk-cache walk entirely and return the
- // same synthetic stub the post-walk SAL-skip would emit, with the
- // three actionable fallback options. Saves the 4 s wait and the
- // ambiguity of "did the walk run? did it find nothing? did it
- // hang?" The caller sees the stub immediately and routes to the
- // appropriate workaround.
- // Class lookup once — used both for the fast-path skip decision
- // and the diagnostic stub. SALSUBFRAME is the empirically-confirmed
- // hang class (modal Confirmation / Warning dialogs). SALFRAME
- // (the document window, the Recovery dialog) walks fine, so don't
- // include it in the skip list — pixel + element_index against
- // Recovery's Discard All button worked end-to-end through the
- // primary path. SALMENU / SALTMPSUBFRAME are transient menu hosts
- // that the user typically doesn't get_window_state on; default to
- // skipping out of caution. If a real use-case emerges, narrow.
- let sal_class: Option<String> = {
+ // The UIA pattern dispatches we lose for SALFRAME (Toggle / Select /
+ // ExpandCollapse) only matter for WinUI3 controls, which VCL doesn't
+ // host. Net win.
+ let sal_class: bool = {
use windows::Win32::UI::WindowsAndMessaging::GetClassNameW;
let mut buf = [0u16; 64];
let n = GetClassNameW(hwnd_win, &mut buf);
- if n > 0 {
- let s = String::from_utf16_lossy(&buf[..n as usize]);
- if s == "SALSUBFRAME" || s == "SALMENU" || s == "SALTMPSUBFRAME" {
- Some(s)
- } else { None }
- } else { None }
+ n > 0 && String::from_utf16_lossy(&buf[..n as usize]).starts_with("SAL")
};
- if let Some(class) = sal_class {
- let stub = format!(
- "- Window <{class} — SAL/VCL target, UIA walk skipped>\n\
- (SAL's UIA provider hangs on TreeScope.Subtree BuildUpdatedCache \
- when the daemon is running in MTA; the cua-driver fast-path \
- returns immediately rather than wait 4 s. Use one of: \
- (a) `screenshot(pid, window_id)` + pixel `click(x, y)`; \
- (b) `press_key` with `dispatch:\"foreground\"` (Esc / Enter / Y / N); \
- (c) `get_window_state` with `capture_mode:\"vision\"`.)\n"
- );
- return UiaTreeResult { tree_markdown: stub, nodes: Vec::new() };
+ if sal_class {
+ return crate::msaa::walk_msaa_tree(hwnd);
}
// Two-call sequence (ElementFromHandle + BuildUpdatedCache) instead of
@@ -408,7 +401,7 @@ unsafe fn walk_cached(
let node = if is_actionable {
let idx = *counter;
*counter += 1;
- let (center_x, center_y) = read_cached_bounding_rect(element);
+ let (center_x, center_y, rect) = read_cached_bounding_rect_full(element);
UiaNode {
element_index: Some(idx),
control_type: control_type.clone(),
@@ -420,6 +413,8 @@ unsafe fn walk_cached(
element_ptr: ptr,
center_x,
center_y,
+ rect,
+ msaa_role: None,
}
} else {
UiaNode {
@@ -433,6 +428,8 @@ unsafe fn walk_cached(
element_ptr: ptr,
center_x: 0,
center_y: 0,
+ rect: None,
+ msaa_role: None,
}
};
@@ -496,11 +493,19 @@ fn read_cached_bool(element: &IUIAutomationElement, property_id: windows::Win32:
}
}
-fn read_cached_bounding_rect(element: &IUIAutomationElement) -> (i32, i32) {
+/// Read bounding rect as (center_x, center_y, Some((l,t,r,b))). Returns
+/// rect=None when the element has no meaningful BoundingRectangle (offscreen
+/// containers, structure-only elements).
+fn read_cached_bounding_rect_full(element: &IUIAutomationElement) -> (i32, i32, Option<(i32, i32, i32, i32)>) {
unsafe {
- element.CachedBoundingRectangle()
- .map(|r| ((r.left + r.right) / 2, (r.top + r.bottom) / 2))
- .unwrap_or((0, 0))
+ match element.CachedBoundingRectangle() {
+ Ok(r) if r.right > r.left && r.bottom > r.top => (
+ (r.left + r.right) / 2,
+ (r.top + r.bottom) / 2,
+ Some((r.left, r.top, r.right, r.bottom)),
+ ),
+ _ => (0, 0, None),
+ }
}
}
@@ -587,7 +592,7 @@ fn control_type_name(id: i32) -> String {
}.into()
}
-fn format_node_line(node: &UiaNode) -> String {
+pub(crate) fn format_node_line(node: &UiaNode) -> String {
let mut s = String::new();
if let Some(idx) = node.element_index {
s.push_str(&format!("- [{}] {}", idx, node.control_type));