A personal assistant that lives on your desktop, remembers what matters, and can use your Mac for you.
TypeScript
feat(desktop): Windows app identity, app launching rules and UI Automation mapping
Marcelito8f984424ec64
diff --git a/apps/desktop/src-tauri/src/computer/mod.rs b/apps/desktop/src-tauri/src/computer/mod.rsindex 1f171b0..1a66ddb 100644--- a/apps/desktop/src-tauri/src/computer/mod.rs+++ b/apps/desktop/src-tauri/src/computer/mod.rs@@ -22,8 +22,13 @@ pub mod risk;pub mod server;pub mod session;pub mod windows;-// What Windows posts for each input event; tested everywhere. The Windows desktop backend that-// posts it widens this to `any(windows, test)`.+// The Windows desktop's pure parts, tested everywhere: UI Automation element mapping, which+// processes are apps, and what Windows posts for each input event. The Windows desktop backend+// that uses them widens these to `any(windows, test)`.+#[cfg(test)]+pub mod winaccess;+#[cfg(test)]+pub mod winapps;#[cfg(test)]pub mod winput;pub mod working;diff --git a/apps/desktop/src-tauri/src/computer/winaccess.rs b/apps/desktop/src-tauri/src/computer/winaccess.rsnew file mode 100644index 0000000..2730ca3--- /dev/null+++ b/apps/desktop/src-tauri/src/computer/winaccess.rs@@ -0,0 +1,152 @@+//! What UI Automation reports about an element, in the Accessibility terms the risk check reads+//! (S24.4 on Windows). Plain mapping, tested on every OS; `win` reads the elements.+//!+//! A control type becomes the macOS role with the same meaning (a Button is `AXButton`, a+//! Hyperlink `AXLink`, an Edit `AXTextField`). The element's name is its title, its help text+//! its description, and its value pattern's value its value. A dialog's default button is the+//! one whose legacy accessible state says so (`STATE_SYSTEM_DEFAULT`).++use super::risk::AxElement;++pub const UIA_BUTTON: i32 = 50000;+pub const UIA_CHECK_BOX: i32 = 50002;+pub const UIA_COMBO_BOX: i32 = 50003;+pub const UIA_EDIT: i32 = 50004;+pub const UIA_HYPERLINK: i32 = 50005;+pub const UIA_LIST_ITEM: i32 = 50007;+pub const UIA_MENU_ITEM: i32 = 50011;+pub const UIA_RADIO_BUTTON: i32 = 50013;+pub const UIA_TAB_ITEM: i32 = 50019;+pub const UIA_TEXT: i32 = 50020;+pub const UIA_GROUP: i32 = 50026;+pub const UIA_DOCUMENT: i32 = 50030;+pub const UIA_SPLIT_BUTTON: i32 = 50031;+pub const UIA_WINDOW: i32 = 50032;+pub const UIA_PANE: i32 = 50033;++/// `STATE_SYSTEM_DEFAULT`: the button Return presses in its dialog.+pub const STATE_SYSTEM_DEFAULT: u32 = 0x100;++/// The Accessibility role for a UI Automation control type.+pub fn role(control_type: i32) -> &'static str {+ match control_type {+ UIA_BUTTON => "AXButton",+ UIA_SPLIT_BUTTON => "AXMenuButton",+ UIA_MENU_ITEM => "AXMenuItem",+ UIA_HYPERLINK => "AXLink",+ UIA_EDIT => "AXTextField",+ UIA_DOCUMENT => "AXTextArea",+ UIA_COMBO_BOX => "AXComboBox",+ UIA_CHECK_BOX => "AXCheckBox",+ UIA_RADIO_BUTTON => "AXRadioButton",+ UIA_TAB_ITEM => "AXTab",+ UIA_LIST_ITEM => "AXRow",+ UIA_TEXT => "AXStaticText",+ UIA_GROUP | UIA_PANE => "AXGroup",+ UIA_WINDOW => "AXWindow",+ _ => "AXUnknown",+ }+}++/// Whether a legacy accessible state marks the default button.+pub fn is_default(state: u32) -> bool {+ state & STATE_SYSTEM_DEFAULT != 0+}++/// An element from what UI Automation reports; blank strings count as absent.+pub fn element(control_type: i32, name: Option<String>, help: Option<String>, value: Option<String>) -> AxElement {+ let present = |text: Option<String>| text.filter(|t| !t.trim().is_empty());+ AxElement { role: role(control_type).into(), title: present(name), description: present(help), value: present(value) }+}++/// What the risk check gets for an element UI Automation does not report, or Gentle Dot's own+/// (its click-through glow) while another app owns the window there: an element that cannot be+/// read stands in for it and asks for a confirmation, so a risky action is never let through unseen.+pub fn unreadable() -> AxElement {+ AxElement {+ role: role(UIA_BUTTON).into(),+ title: Some("an element Gentle Dot cannot read; confirm to act on it".into()),+ description: None,+ value: None,+ }+}++#[cfg(test)]+mod tests {+ use super::*;+ use crate::computer::risk::{assess, Act, Focus};++ fn named(control_type: i32, name: &str) -> AxElement {+ element(control_type, Some(name.into()), None, None)+ }++ #[test]+ fn pressable_controls_get_the_roles_a_click_can_press() {+ assert_eq!(role(UIA_BUTTON), "AXButton");+ assert_eq!(role(UIA_SPLIT_BUTTON), "AXMenuButton");+ assert_eq!(role(UIA_MENU_ITEM), "AXMenuItem");+ assert_eq!(role(UIA_HYPERLINK), "AXLink");+ }++ #[test]+ fn text_inputs_get_the_field_roles() {+ assert_eq!(role(UIA_EDIT), "AXTextField");+ assert_eq!(role(UIA_DOCUMENT), "AXTextArea");+ assert_eq!(role(UIA_COMBO_BOX), "AXComboBox");+ }++ #[test]+ fn other_controls_keep_roles_that_are_neither_pressable_nor_fields() {+ for (control_type, expected) in [+ (UIA_CHECK_BOX, "AXCheckBox"), (UIA_RADIO_BUTTON, "AXRadioButton"), (UIA_TAB_ITEM, "AXTab"),+ (UIA_LIST_ITEM, "AXRow"), (UIA_TEXT, "AXStaticText"), (UIA_GROUP, "AXGroup"), (UIA_PANE, "AXGroup"),+ (UIA_WINDOW, "AXWindow"), (12345, "AXUnknown"),+ ] {+ assert_eq!(role(control_type), expected, "{control_type}");+ }+ }++ #[test]+ fn blank_strings_are_absent_and_the_rest_is_kept_as_reported() {+ let e = element(UIA_BUTTON, Some(" Send ".into()), Some(" ".into()), Some(String::new()));+ assert_eq!(e, AxElement { role: "AXButton".into(), title: Some(" Send ".into()), description: None, value: None });+ let e = element(UIA_EDIT, None, Some("Type a message".into()), Some("hola".into()));+ assert_eq!((e.title, e.description.as_deref(), e.value.as_deref()), (None, Some("Type a message"), Some("hola")));+ }++ #[test]+ fn only_the_default_state_bit_marks_the_default_button() {+ assert!(is_default(STATE_SYSTEM_DEFAULT));+ assert!(is_default(STATE_SYSTEM_DEFAULT | 0x4 | 0x100000));+ assert!(!is_default(0));+ assert!(!is_default(0x4 | 0x100000));+ }++ #[test]+ fn clicking_a_windows_send_button_or_buy_link_needs_a_confirmation() {+ assert!(assess(Act::Click(&[named(UIA_BUTTON, "Send")]), None).is_some());+ assert!(assess(Act::Click(&[named(UIA_HYPERLINK, "Comprar ahora")]), None).is_some());+ // The words inside a button count, as on macOS: text under the pointer, then the button.+ let chain = [named(UIA_TEXT, "Delete"), element(UIA_BUTTON, None, None, None)];+ assert!(assess(Act::Click(&chain), None).is_some());+ assert!(assess(Act::Click(&[named(UIA_BUTTON, "Open")]), None).is_none());+ assert!(assess(Act::Click(&[named(UIA_CHECK_BOX, "Send me news")]), None).is_none(), "a check box is not pressed");+ }++ #[test]+ fn clicking_what_cannot_be_read_under_the_glow_needs_a_confirmation() {+ let reason = assess(Act::Click(&[unreadable()]), None).expect("a confirmation");+ assert!(reason.contains("Gentle Dot cannot read"), "{reason}");+ assert!(assess(Act::Drag(&[named(UIA_TEXT, "x")], &[unreadable()]), None).is_some());+ }++ #[test]+ fn return_in_a_windows_chat_box_or_on_a_default_button_needs_a_confirmation() {+ let chat = Focus { element: named(UIA_EDIT, "Message #general"), ..Focus::default() };+ assert!(assess(Act::Return(Some(&chat)), None).is_some());+ let default = Focus { element: named(UIA_BUTTON, "OK"), is_default_button: true, default_button: None };+ assert!(assess(Act::Return(Some(&default)), None).is_some());+ let note = Focus { element: named(UIA_DOCUMENT, "Untitled - Notepad"), ..Focus::default() };+ assert!(assess(Act::Return(Some(¬e)), None).is_none());+ }+}diff --git a/apps/desktop/src-tauri/src/computer/winapps.rs b/apps/desktop/src-tauri/src/computer/winapps.rsnew file mode 100644index 0000000..85be7aa--- /dev/null+++ b/apps/desktop/src-tauri/src/computer/winapps.rs@@ -0,0 +1,269 @@+//! Which windows are apps, which app a window is, and what `open_app` may launch, on Windows+//! (S24.2, S24.4). Plain rules over what `win` reads from the system, tested on every OS.++/// A top-level window as `win` reads it, front to back.+#[derive(Debug, Clone, PartialEq, Eq)]+pub struct TopWindow {+ pub pid: i32,+ pub title: String,+ pub visible: bool,+ /// Hidden by the shell (a suspended Store app keeps a cloaked window).+ pub cloaked: bool,+ /// `WS_EX_TOOLWINDOW`: palettes and floating toolbars, not apps.+ pub tool: bool,+ /// `WS_EX_APPWINDOW`: shown on the taskbar even when owned.+ pub app_window: bool,+ /// Owned by another window (a dialog), so it is part of that window's app.+ pub owned: bool,+}++impl TopWindow {+ /// The taskbar's rule: a visible, titled window that is neither a tool window nor owned,+ /// unless it asks to be on the taskbar.+ pub fn is_app(&self) -> bool {+ self.visible && !self.cloaked && !self.title.trim().is_empty() && (self.app_window || (!self.tool && !self.owned))+ }+}++/// The processes with an app window, front to back, each once.+pub fn app_pids(windows: &[TopWindow]) -> Vec<i32> {+ let mut pids = Vec::new();+ for window in windows.iter().filter(|w| w.is_app()) {+ if !pids.contains(&window.pid) {+ pids.push(window.pid);+ }+ }+ pids+}++/// What `open_app` was asked for: an app's name, or a local program or shortcut by path.+#[derive(Debug, Clone, PartialEq, Eq)]+pub enum Request {+ Name(String),+ Path(String),+}++/// `text` without `suffix`, compared ASCII case-insensitively.+pub fn strip_ci<'a>(text: &'a str, suffix: &str) -> Option<&'a str> {+ let cut = text.len().checked_sub(suffix.len())?;+ (text.is_char_boundary(cut) && text[cut..].eq_ignore_ascii_case(suffix)).then(|| &text[..cut])+}++/// `X:\` at the start: a path on one of this PC's drives.+fn drive_path(text: &str) -> bool {+ let b = text.as_bytes();+ b.len() > 3 && b[0].is_ascii_alphabetic() && b[1] == b':' && b[2] == b'\\'+}++fn program_or_shortcut(text: &str) -> bool {+ strip_ci(text, ".exe").is_some() || strip_ci(text, ".lnk").is_some()+}++/// Reads what `open_app` was asked for. Only an app name, or the path of a program (.exe) or a+/// shortcut (.lnk) on a local drive; never a URL, a protocol (`ms-settings:`), a command line, a+/// network share, a script, or a document.+pub fn request(name: &str) -> Result<Request, String> {+ let name = name.trim();+ if name.is_empty() {+ return Err("`open_app` needs an app name.".into());+ }+ if name.chars().any(char::is_control) || name.ends_with('.') {+ return Err(format!("`{name}` is not an app name."));+ }+ let colons = name.matches(':').count();+ if colons > 1 || (colons == 1 && !drive_path(name)) {+ return Err(format!("`{name}` is not an app name: URLs and protocols cannot be opened."));+ }+ if name.contains(['"', '<', '>', '|', '&', '^', '%', '*', '?']) || name.contains(" /") || name.contains(" -") {+ return Err(format!("`{name}` is not an app name: command lines cannot be run."));+ }+ if name.contains(['\\', '/']) {+ let local = drive_path(name) && !name.contains('/') && !name.split('\\').any(|part| part == "..");+ if !local {+ return Err(format!("`{name}` is not an app: only programs on this PC's drives can be opened."));+ }+ if !program_or_shortcut(name) {+ return Err(format!("`{name}` is not an app: only programs (.exe) and shortcuts (.lnk) can be opened."));+ }+ return Ok(Request::Path(name.to_string()));+ }+ // A bare name with an extension must be a program or a shortcut (not `notes.txt` or `run.cmd`).+ if name.contains('.') && !program_or_shortcut(name) && name.rsplit('.').next().is_some_and(|ext| !ext.contains(' ')) {+ return Err(format!("`{name}` is not an app: only programs (.exe) and shortcuts (.lnk) can be opened."));+ }+ Ok(Request::Name(name.to_string()))+}++/// The Start menu shortcut named `name` (its file name without `.lnk`, any case).+pub fn shortcut_named<'a>(name: &str, shortcuts: &'a [String]) -> Option<&'a String> {+ let wanted = strip_ci(name, ".lnk").unwrap_or(name);+ shortcuts.iter().find(|path| {+ let file = path.rsplit(['\\', '/']).next().unwrap_or(path);+ strip_ci(file, ".lnk").is_some_and(|stem| stem.eq_ignore_ascii_case(wanted))+ })+}++/// What a shortcut, an App Paths entry, or the search resolved to, when it is a program on a+/// local drive started with no arguments. Anything else (a script, a document, a command line,+/// a network share) is refused, so `open_app` only ever starts a program, as `open -a` does.+pub fn executable_target(path: &str, arguments: &str) -> Result<String, String> {+ let path = path.trim().trim_matches('"').trim();+ if !arguments.trim().is_empty() {+ return Err(format!("`{path}` is started with a command line, so it was not opened."));+ }+ let local = drive_path(path) && path.matches(':').count() == 1 && !path.contains('/') && !path.split('\\').any(|part| part == "..");+ if !local || strip_ci(path, ".exe").is_none() || path.ends_with(['.', ' ']) {+ return Err(format!("`{path}` is not a program on this PC's drives, so it was not opened."));+ }+ Ok(path.to_string())+}++/// Whether a running app answers to `name`: its name or its executable (with or without `.exe`).+pub fn answers_to(name: &str, app_name: &str, executable: Option<&str>) -> bool {+ let name = name.trim();+ let bare = |exe: &str| strip_ci(exe, ".exe").unwrap_or(exe).to_string();+ app_name.eq_ignore_ascii_case(name)+ || executable.is_some_and(|exe| exe.eq_ignore_ascii_case(name) || bare(exe).eq_ignore_ascii_case(&bare(name)))+}++/// The titles of the Control Panel page that holds saved credentials, which File Explorer hosts.+const CREDENTIAL_MANAGER_TITLES: &[&str] = &["Credential Manager", "Administrador de credenciales"];++/// The blocked page a File Explorer window shows, by its title: such a window is that page,+/// not File Explorer.+pub fn hosted_page(executable: Option<&str>, title: &str) -> Option<&'static str> {+ let explorer = executable.is_some_and(|exe| exe.eq_ignore_ascii_case("explorer.exe"));+ let title = title.trim();+ (explorer && CREDENTIAL_MANAGER_TITLES.iter().any(|t| t.eq_ignore_ascii_case(title))).then_some("Credential Manager")+}++/// Whether the window in front stands for an app window being brought forward: the same process+/// (`front_pid`, `pid`) and the same root owner (`front_owner`, `owner`, as addresses), as when the+/// app shows its own dialog. A window that vanished reads as process 0 and owner 0, so two vanished+/// windows never match (that would report an app in front with nothing there).+pub fn fronts_same_app(front_pid: i32, pid: i32, front_owner: usize, owner: usize) -> bool {+ front_pid != 0 && front_owner != 0 && front_pid == pid && front_owner == owner+}++#[cfg(test)]+mod tests {+ use super::*;++ fn window(pid: i32, title: &str) -> TopWindow {+ TopWindow { pid, title: title.into(), visible: true, cloaked: false, tool: false, app_window: false, owned: false }+ }++ #[test]+ fn a_window_counts_as_in_front_only_with_a_real_process() {+ // The window itself, or another window of its process under the same owner (its dialog).+ assert!(fronts_same_app(7, 7, 100, 100));+ assert!(!fronts_same_app(7, 8, 100, 100));+ assert!(!fronts_same_app(7, 7, 100, 200));+ // Both windows gone: no process (0) and no owner (0) on either side is not a match.+ assert!(!fronts_same_app(0, 0, 0, 0));+ assert!(!fronts_same_app(0, 0, 100, 100));+ }++ #[test]+ fn a_visible_titled_window_is_an_app() {+ assert!(window(1, "Notepad").is_app());+ }++ #[test]+ fn hidden_cloaked_untitled_tool_and_owned_windows_are_not_apps() {+ assert!(!TopWindow { visible: false, ..window(1, "x") }.is_app());+ assert!(!TopWindow { cloaked: true, ..window(1, "x") }.is_app());+ assert!(!window(1, " ").is_app());+ assert!(!TopWindow { tool: true, ..window(1, "x") }.is_app());+ assert!(!TopWindow { owned: true, ..window(1, "x") }.is_app());+ }++ #[test]+ fn an_app_window_flag_puts_even_tool_or_owned_windows_on_the_list() {+ assert!(TopWindow { tool: true, app_window: true, ..window(1, "x") }.is_app());+ assert!(TopWindow { owned: true, app_window: true, ..window(1, "x") }.is_app());+ assert!(!TopWindow { app_window: true, visible: false, ..window(1, "x") }.is_app());+ }++ #[test]+ fn each_app_is_listed_once_front_to_back() {+ let windows = [window(7, "Chrome"), TopWindow { tool: true, ..window(9, "Palette") }, window(3, "Notepad"), window(7, "Chrome 2")];+ assert_eq!(app_pids(&windows), [7, 3]);+ }++ #[test]+ fn app_names_and_local_program_or_shortcut_paths_are_accepted() {+ for name in ["Notepad", "chrome.exe", "Visual Studio Code", "Notepad++", "Tool.LNK", "Microsoft Edge"] {+ assert_eq!(request(name), Ok(Request::Name(name.into())), "{name}");+ }+ for path in [r"C:\Program Files\App\app.exe", r"C:\Users\me\Desktop\Tool.LNK", r"D:\Tools\x.EXE"] {+ assert_eq!(request(path), Ok(Request::Path(path.into())), "{path}");+ }+ assert_eq!(request(" Notepad "), Ok(Request::Name("Notepad".into())));+ // Any Unicode name is read without panicking.+ assert_eq!(request("a€"), Ok(Request::Name("a€".into())));+ assert_eq!(request("€:"), Err("`€:` is not an app name: URLs and protocols cannot be opened.".into()));+ }++ #[test]+ fn urls_protocols_command_lines_scripts_documents_and_shares_are_refused() {+ for name in [+ "", " ", "ms-settings:", "ms-settings:privacy", "https://example.com", "file:///C:/x.exe", "cmd /c del x",+ "powershell -c x", "a & b", "a | b", "%COMSPEC%", "x*.exe", "notes.txt", "payload.cmd", "run.bat", "x.vbs",+ "chrome.exe.", "a\u{7}b", r"C:\x\notes.txt", r"\\server\share\app.exe", r"C:\a:b.exe", r"C:\x\..\Windows\cmd.exe",+ "C:/x/app.exe", r"tools\app.exe",+ ] {+ assert!(request(name).is_err(), "{name:?} should be refused");+ }+ assert!(request("ms-settings:").unwrap_err().contains("protocols"));+ assert!(request("cmd /c del x").unwrap_err().contains("command lines"));+ assert!(request("payload.cmd").unwrap_err().contains("only programs"));+ }++ #[test]+ fn a_start_menu_shortcut_is_found_by_its_name_in_any_case() {+ let shortcuts = [r"C:\ProgramData\Microsoft\Windows\Start Menu\Programs\Google Chrome.lnk".to_string(), r"C:\Users\me\Start Menu\Programs\Notes\NOTES.LNK".to_string()];+ assert_eq!(shortcut_named("google chrome", &shortcuts), Some(&shortcuts[0]));+ assert_eq!(shortcut_named("Notes.lnk", &shortcuts), Some(&shortcuts[1]));+ assert_eq!(shortcut_named("Chrome", &shortcuts), None);+ }++ #[test]+ fn only_a_local_program_started_without_arguments_is_launched() {+ let chrome = r"C:\Program Files\Google\Chrome\Application\chrome.exe";+ assert_eq!(executable_target(chrome, ""), Ok(chrome.into()));+ assert_eq!(executable_target(&format!("\"{chrome}\""), " "), Ok(chrome.into()));+ assert!(executable_target(r"C:\Windows\System32\cmd.exe", "/c del x").unwrap_err().contains("command line"));+ for target in [r"C:\x\run.cmd", r"C:\x\notes.txt", r"\\server\share\app.exe", r"C:\x\a.exe:stream", r"C:\x\..\y\a.exe", "app.exe", r"C:\x\a.exe.", ""] {+ assert!(executable_target(target, "").is_err(), "{target:?} should be refused");+ }+ }++ #[test]+ fn suffixes_compare_case_insensitively_and_safely() {+ assert_eq!(strip_ci("Tool.LnK", ".lnk"), Some("Tool"));+ assert_eq!(strip_ci("a.Exe", ".exe"), Some("a"));+ assert_eq!(strip_ci("exe", ".exe"), None);+ assert_eq!(strip_ci("€exe", ".exe"), None);+ }++ #[test]+ fn a_file_explorer_window_showing_credential_manager_is_that_page() {+ assert_eq!(hosted_page(Some("explorer.exe"), "Credential Manager"), Some("Credential Manager"));+ assert_eq!(hosted_page(Some("EXPLORER.EXE"), " administrador de credenciales "), Some("Credential Manager"));+ assert_eq!(hosted_page(Some("explorer.exe"), "Downloads"), None);+ assert_eq!(hosted_page(Some("notepad.exe"), "Credential Manager"), None);+ assert_eq!(hosted_page(None, "Credential Manager"), None);+ }++ #[test]+ fn a_running_app_answers_to_its_name_or_executable() {+ assert!(answers_to("notepad", "Notepad", Some("Notepad.exe")));+ assert!(answers_to("NOTEPAD.EXE", "Notepad", Some("Notepad.exe")));+ assert!(answers_to("Google Chrome", "Google Chrome", Some("chrome.exe")));+ assert!(answers_to("chrome", "Google Chrome", Some("chrome.exe")));+ assert!(!answers_to("chrom", "Google Chrome", Some("chrome.exe")));+ assert!(!answers_to("Notepad", "Paint", None));+ assert!(answers_to("notepad", "Notepad", Some("NOTEPAD.Exe")));+ }+}