Skip to content

A personal assistant that lives on your desktop, remembers what matters, and can use your Mac for you.

TypeScript

feat(desktop): Windows app identity, app launching rules and UI Automation mapping

Marcelito8f984424ec64
diff --git a/apps/desktop/src-tauri/src/computer/mod.rs b/apps/desktop/src-tauri/src/computer/mod.rs
index 1f171b0..1a66ddb 100644
--- a/apps/desktop/src-tauri/src/computer/mod.rs
+++ b/apps/desktop/src-tauri/src/computer/mod.rs
@@ -22,8 +22,13 @@ pub mod risk;
pub mod server;
pub mod session;
pub mod windows;
-// What Windows posts for each input event; tested everywhere. The Windows desktop backend that
-// posts it widens this to `any(windows, test)`.
+// The Windows desktop's pure parts, tested everywhere: UI Automation element mapping, which
+// processes are apps, and what Windows posts for each input event. The Windows desktop backend
+// that uses them widens these to `any(windows, test)`.
+#[cfg(test)]
+pub mod winaccess;
+#[cfg(test)]
+pub mod winapps;
#[cfg(test)]
pub mod winput;
pub mod working;
diff --git a/apps/desktop/src-tauri/src/computer/winaccess.rs b/apps/desktop/src-tauri/src/computer/winaccess.rs
new file mode 100644
index 0000000..2730ca3
--- /dev/null
+++ b/apps/desktop/src-tauri/src/computer/winaccess.rs
@@ -0,0 +1,152 @@
+//! What UI Automation reports about an element, in the Accessibility terms the risk check reads
+//! (S24.4 on Windows). Plain mapping, tested on every OS; `win` reads the elements.
+//!
+//! A control type becomes the macOS role with the same meaning (a Button is `AXButton`, a
+//! Hyperlink `AXLink`, an Edit `AXTextField`). The element's name is its title, its help text
+//! its description, and its value pattern's value its value. A dialog's default button is the
+//! one whose legacy accessible state says so (`STATE_SYSTEM_DEFAULT`).
+
+use super::risk::AxElement;
+
+pub const UIA_BUTTON: i32 = 50000;
+pub const UIA_CHECK_BOX: i32 = 50002;
+pub const UIA_COMBO_BOX: i32 = 50003;
+pub const UIA_EDIT: i32 = 50004;
+pub const UIA_HYPERLINK: i32 = 50005;
+pub const UIA_LIST_ITEM: i32 = 50007;
+pub const UIA_MENU_ITEM: i32 = 50011;
+pub const UIA_RADIO_BUTTON: i32 = 50013;
+pub const UIA_TAB_ITEM: i32 = 50019;
+pub const UIA_TEXT: i32 = 50020;
+pub const UIA_GROUP: i32 = 50026;
+pub const UIA_DOCUMENT: i32 = 50030;
+pub const UIA_SPLIT_BUTTON: i32 = 50031;
+pub const UIA_WINDOW: i32 = 50032;
+pub const UIA_PANE: i32 = 50033;
+
+/// `STATE_SYSTEM_DEFAULT`: the button Return presses in its dialog.
+pub const STATE_SYSTEM_DEFAULT: u32 = 0x100;
+
+/// The Accessibility role for a UI Automation control type.
+pub fn role(control_type: i32) -> &'static str {
+ match control_type {
+ UIA_BUTTON => "AXButton",
+ UIA_SPLIT_BUTTON => "AXMenuButton",
+ UIA_MENU_ITEM => "AXMenuItem",
+ UIA_HYPERLINK => "AXLink",
+ UIA_EDIT => "AXTextField",
+ UIA_DOCUMENT => "AXTextArea",
+ UIA_COMBO_BOX => "AXComboBox",
+ UIA_CHECK_BOX => "AXCheckBox",
+ UIA_RADIO_BUTTON => "AXRadioButton",
+ UIA_TAB_ITEM => "AXTab",
+ UIA_LIST_ITEM => "AXRow",
+ UIA_TEXT => "AXStaticText",
+ UIA_GROUP | UIA_PANE => "AXGroup",
+ UIA_WINDOW => "AXWindow",
+ _ => "AXUnknown",
+ }
+}
+
+/// Whether a legacy accessible state marks the default button.
+pub fn is_default(state: u32) -> bool {
+ state & STATE_SYSTEM_DEFAULT != 0
+}
+
+/// An element from what UI Automation reports; blank strings count as absent.
+pub fn element(control_type: i32, name: Option<String>, help: Option<String>, value: Option<String>) -> AxElement {
+ let present = |text: Option<String>| text.filter(|t| !t.trim().is_empty());
+ AxElement { role: role(control_type).into(), title: present(name), description: present(help), value: present(value) }
+}
+
+/// What the risk check gets for an element UI Automation does not report, or Gentle Dot's own
+/// (its click-through glow) while another app owns the window there: an element that cannot be
+/// read stands in for it and asks for a confirmation, so a risky action is never let through unseen.
+pub fn unreadable() -> AxElement {
+ AxElement {
+ role: role(UIA_BUTTON).into(),
+ title: Some("an element Gentle Dot cannot read; confirm to act on it".into()),
+ description: None,
+ value: None,
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use crate::computer::risk::{assess, Act, Focus};
+
+ fn named(control_type: i32, name: &str) -> AxElement {
+ element(control_type, Some(name.into()), None, None)
+ }
+
+ #[test]
+ fn pressable_controls_get_the_roles_a_click_can_press() {
+ assert_eq!(role(UIA_BUTTON), "AXButton");
+ assert_eq!(role(UIA_SPLIT_BUTTON), "AXMenuButton");
+ assert_eq!(role(UIA_MENU_ITEM), "AXMenuItem");
+ assert_eq!(role(UIA_HYPERLINK), "AXLink");
+ }
+
+ #[test]
+ fn text_inputs_get_the_field_roles() {
+ assert_eq!(role(UIA_EDIT), "AXTextField");
+ assert_eq!(role(UIA_DOCUMENT), "AXTextArea");
+ assert_eq!(role(UIA_COMBO_BOX), "AXComboBox");
+ }
+
+ #[test]
+ fn other_controls_keep_roles_that_are_neither_pressable_nor_fields() {
+ for (control_type, expected) in [
+ (UIA_CHECK_BOX, "AXCheckBox"), (UIA_RADIO_BUTTON, "AXRadioButton"), (UIA_TAB_ITEM, "AXTab"),
+ (UIA_LIST_ITEM, "AXRow"), (UIA_TEXT, "AXStaticText"), (UIA_GROUP, "AXGroup"), (UIA_PANE, "AXGroup"),
+ (UIA_WINDOW, "AXWindow"), (12345, "AXUnknown"),
+ ] {
+ assert_eq!(role(control_type), expected, "{control_type}");
+ }
+ }
+
+ #[test]
+ fn blank_strings_are_absent_and_the_rest_is_kept_as_reported() {
+ let e = element(UIA_BUTTON, Some(" Send ".into()), Some(" ".into()), Some(String::new()));
+ assert_eq!(e, AxElement { role: "AXButton".into(), title: Some(" Send ".into()), description: None, value: None });
+ let e = element(UIA_EDIT, None, Some("Type a message".into()), Some("hola".into()));
+ assert_eq!((e.title, e.description.as_deref(), e.value.as_deref()), (None, Some("Type a message"), Some("hola")));
+ }
+
+ #[test]
+ fn only_the_default_state_bit_marks_the_default_button() {
+ assert!(is_default(STATE_SYSTEM_DEFAULT));
+ assert!(is_default(STATE_SYSTEM_DEFAULT | 0x4 | 0x100000));
+ assert!(!is_default(0));
+ assert!(!is_default(0x4 | 0x100000));
+ }
+
+ #[test]
+ fn clicking_a_windows_send_button_or_buy_link_needs_a_confirmation() {
+ assert!(assess(Act::Click(&[named(UIA_BUTTON, "Send")]), None).is_some());
+ assert!(assess(Act::Click(&[named(UIA_HYPERLINK, "Comprar ahora")]), None).is_some());
+ // The words inside a button count, as on macOS: text under the pointer, then the button.
+ let chain = [named(UIA_TEXT, "Delete"), element(UIA_BUTTON, None, None, None)];
+ assert!(assess(Act::Click(&chain), None).is_some());
+ assert!(assess(Act::Click(&[named(UIA_BUTTON, "Open")]), None).is_none());
+ assert!(assess(Act::Click(&[named(UIA_CHECK_BOX, "Send me news")]), None).is_none(), "a check box is not pressed");
+ }
+
+ #[test]
+ fn clicking_what_cannot_be_read_under_the_glow_needs_a_confirmation() {
+ let reason = assess(Act::Click(&[unreadable()]), None).expect("a confirmation");
+ assert!(reason.contains("Gentle Dot cannot read"), "{reason}");
+ assert!(assess(Act::Drag(&[named(UIA_TEXT, "x")], &[unreadable()]), None).is_some());
+ }
+
+ #[test]
+ fn return_in_a_windows_chat_box_or_on_a_default_button_needs_a_confirmation() {
+ let chat = Focus { element: named(UIA_EDIT, "Message #general"), ..Focus::default() };
+ assert!(assess(Act::Return(Some(&chat)), None).is_some());
+ let default = Focus { element: named(UIA_BUTTON, "OK"), is_default_button: true, default_button: None };
+ assert!(assess(Act::Return(Some(&default)), None).is_some());
+ let note = Focus { element: named(UIA_DOCUMENT, "Untitled - Notepad"), ..Focus::default() };
+ assert!(assess(Act::Return(Some(&note)), None).is_none());
+ }
+}
diff --git a/apps/desktop/src-tauri/src/computer/winapps.rs b/apps/desktop/src-tauri/src/computer/winapps.rs
new file mode 100644
index 0000000..85be7aa
--- /dev/null
+++ b/apps/desktop/src-tauri/src/computer/winapps.rs
@@ -0,0 +1,269 @@
+//! Which windows are apps, which app a window is, and what `open_app` may launch, on Windows
+//! (S24.2, S24.4). Plain rules over what `win` reads from the system, tested on every OS.
+
+/// A top-level window as `win` reads it, front to back.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct TopWindow {
+ pub pid: i32,
+ pub title: String,
+ pub visible: bool,
+ /// Hidden by the shell (a suspended Store app keeps a cloaked window).
+ pub cloaked: bool,
+ /// `WS_EX_TOOLWINDOW`: palettes and floating toolbars, not apps.
+ pub tool: bool,
+ /// `WS_EX_APPWINDOW`: shown on the taskbar even when owned.
+ pub app_window: bool,
+ /// Owned by another window (a dialog), so it is part of that window's app.
+ pub owned: bool,
+}
+
+impl TopWindow {
+ /// The taskbar's rule: a visible, titled window that is neither a tool window nor owned,
+ /// unless it asks to be on the taskbar.
+ pub fn is_app(&self) -> bool {
+ self.visible && !self.cloaked && !self.title.trim().is_empty() && (self.app_window || (!self.tool && !self.owned))
+ }
+}
+
+/// The processes with an app window, front to back, each once.
+pub fn app_pids(windows: &[TopWindow]) -> Vec<i32> {
+ let mut pids = Vec::new();
+ for window in windows.iter().filter(|w| w.is_app()) {
+ if !pids.contains(&window.pid) {
+ pids.push(window.pid);
+ }
+ }
+ pids
+}
+
+/// What `open_app` was asked for: an app's name, or a local program or shortcut by path.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub enum Request {
+ Name(String),
+ Path(String),
+}
+
+/// `text` without `suffix`, compared ASCII case-insensitively.
+pub fn strip_ci<'a>(text: &'a str, suffix: &str) -> Option<&'a str> {
+ let cut = text.len().checked_sub(suffix.len())?;
+ (text.is_char_boundary(cut) && text[cut..].eq_ignore_ascii_case(suffix)).then(|| &text[..cut])
+}
+
+/// `X:\` at the start: a path on one of this PC's drives.
+fn drive_path(text: &str) -> bool {
+ let b = text.as_bytes();
+ b.len() > 3 && b[0].is_ascii_alphabetic() && b[1] == b':' && b[2] == b'\\'
+}
+
+fn program_or_shortcut(text: &str) -> bool {
+ strip_ci(text, ".exe").is_some() || strip_ci(text, ".lnk").is_some()
+}
+
+/// Reads what `open_app` was asked for. Only an app name, or the path of a program (.exe) or a
+/// shortcut (.lnk) on a local drive; never a URL, a protocol (`ms-settings:`), a command line, a
+/// network share, a script, or a document.
+pub fn request(name: &str) -> Result<Request, String> {
+ let name = name.trim();
+ if name.is_empty() {
+ return Err("`open_app` needs an app name.".into());
+ }
+ if name.chars().any(char::is_control) || name.ends_with('.') {
+ return Err(format!("`{name}` is not an app name."));
+ }
+ let colons = name.matches(':').count();
+ if colons > 1 || (colons == 1 && !drive_path(name)) {
+ return Err(format!("`{name}` is not an app name: URLs and protocols cannot be opened."));
+ }
+ if name.contains(['"', '<', '>', '|', '&', '^', '%', '*', '?']) || name.contains(" /") || name.contains(" -") {
+ return Err(format!("`{name}` is not an app name: command lines cannot be run."));
+ }
+ if name.contains(['\\', '/']) {
+ let local = drive_path(name) && !name.contains('/') && !name.split('\\').any(|part| part == "..");
+ if !local {
+ return Err(format!("`{name}` is not an app: only programs on this PC's drives can be opened."));
+ }
+ if !program_or_shortcut(name) {
+ return Err(format!("`{name}` is not an app: only programs (.exe) and shortcuts (.lnk) can be opened."));
+ }
+ return Ok(Request::Path(name.to_string()));
+ }
+ // A bare name with an extension must be a program or a shortcut (not `notes.txt` or `run.cmd`).
+ if name.contains('.') && !program_or_shortcut(name) && name.rsplit('.').next().is_some_and(|ext| !ext.contains(' ')) {
+ return Err(format!("`{name}` is not an app: only programs (.exe) and shortcuts (.lnk) can be opened."));
+ }
+ Ok(Request::Name(name.to_string()))
+}
+
+/// The Start menu shortcut named `name` (its file name without `.lnk`, any case).
+pub fn shortcut_named<'a>(name: &str, shortcuts: &'a [String]) -> Option<&'a String> {
+ let wanted = strip_ci(name, ".lnk").unwrap_or(name);
+ shortcuts.iter().find(|path| {
+ let file = path.rsplit(['\\', '/']).next().unwrap_or(path);
+ strip_ci(file, ".lnk").is_some_and(|stem| stem.eq_ignore_ascii_case(wanted))
+ })
+}
+
+/// What a shortcut, an App Paths entry, or the search resolved to, when it is a program on a
+/// local drive started with no arguments. Anything else (a script, a document, a command line,
+/// a network share) is refused, so `open_app` only ever starts a program, as `open -a` does.
+pub fn executable_target(path: &str, arguments: &str) -> Result<String, String> {
+ let path = path.trim().trim_matches('"').trim();
+ if !arguments.trim().is_empty() {
+ return Err(format!("`{path}` is started with a command line, so it was not opened."));
+ }
+ let local = drive_path(path) && path.matches(':').count() == 1 && !path.contains('/') && !path.split('\\').any(|part| part == "..");
+ if !local || strip_ci(path, ".exe").is_none() || path.ends_with(['.', ' ']) {
+ return Err(format!("`{path}` is not a program on this PC's drives, so it was not opened."));
+ }
+ Ok(path.to_string())
+}
+
+/// Whether a running app answers to `name`: its name or its executable (with or without `.exe`).
+pub fn answers_to(name: &str, app_name: &str, executable: Option<&str>) -> bool {
+ let name = name.trim();
+ let bare = |exe: &str| strip_ci(exe, ".exe").unwrap_or(exe).to_string();
+ app_name.eq_ignore_ascii_case(name)
+ || executable.is_some_and(|exe| exe.eq_ignore_ascii_case(name) || bare(exe).eq_ignore_ascii_case(&bare(name)))
+}
+
+/// The titles of the Control Panel page that holds saved credentials, which File Explorer hosts.
+const CREDENTIAL_MANAGER_TITLES: &[&str] = &["Credential Manager", "Administrador de credenciales"];
+
+/// The blocked page a File Explorer window shows, by its title: such a window is that page,
+/// not File Explorer.
+pub fn hosted_page(executable: Option<&str>, title: &str) -> Option<&'static str> {
+ let explorer = executable.is_some_and(|exe| exe.eq_ignore_ascii_case("explorer.exe"));
+ let title = title.trim();
+ (explorer && CREDENTIAL_MANAGER_TITLES.iter().any(|t| t.eq_ignore_ascii_case(title))).then_some("Credential Manager")
+}
+
+/// Whether the window in front stands for an app window being brought forward: the same process
+/// (`front_pid`, `pid`) and the same root owner (`front_owner`, `owner`, as addresses), as when the
+/// app shows its own dialog. A window that vanished reads as process 0 and owner 0, so two vanished
+/// windows never match (that would report an app in front with nothing there).
+pub fn fronts_same_app(front_pid: i32, pid: i32, front_owner: usize, owner: usize) -> bool {
+ front_pid != 0 && front_owner != 0 && front_pid == pid && front_owner == owner
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ fn window(pid: i32, title: &str) -> TopWindow {
+ TopWindow { pid, title: title.into(), visible: true, cloaked: false, tool: false, app_window: false, owned: false }
+ }
+
+ #[test]
+ fn a_window_counts_as_in_front_only_with_a_real_process() {
+ // The window itself, or another window of its process under the same owner (its dialog).
+ assert!(fronts_same_app(7, 7, 100, 100));
+ assert!(!fronts_same_app(7, 8, 100, 100));
+ assert!(!fronts_same_app(7, 7, 100, 200));
+ // Both windows gone: no process (0) and no owner (0) on either side is not a match.
+ assert!(!fronts_same_app(0, 0, 0, 0));
+ assert!(!fronts_same_app(0, 0, 100, 100));
+ }
+
+ #[test]
+ fn a_visible_titled_window_is_an_app() {
+ assert!(window(1, "Notepad").is_app());
+ }
+
+ #[test]
+ fn hidden_cloaked_untitled_tool_and_owned_windows_are_not_apps() {
+ assert!(!TopWindow { visible: false, ..window(1, "x") }.is_app());
+ assert!(!TopWindow { cloaked: true, ..window(1, "x") }.is_app());
+ assert!(!window(1, " ").is_app());
+ assert!(!TopWindow { tool: true, ..window(1, "x") }.is_app());
+ assert!(!TopWindow { owned: true, ..window(1, "x") }.is_app());
+ }
+
+ #[test]
+ fn an_app_window_flag_puts_even_tool_or_owned_windows_on_the_list() {
+ assert!(TopWindow { tool: true, app_window: true, ..window(1, "x") }.is_app());
+ assert!(TopWindow { owned: true, app_window: true, ..window(1, "x") }.is_app());
+ assert!(!TopWindow { app_window: true, visible: false, ..window(1, "x") }.is_app());
+ }
+
+ #[test]
+ fn each_app_is_listed_once_front_to_back() {
+ let windows = [window(7, "Chrome"), TopWindow { tool: true, ..window(9, "Palette") }, window(3, "Notepad"), window(7, "Chrome 2")];
+ assert_eq!(app_pids(&windows), [7, 3]);
+ }
+
+ #[test]
+ fn app_names_and_local_program_or_shortcut_paths_are_accepted() {
+ for name in ["Notepad", "chrome.exe", "Visual Studio Code", "Notepad++", "Tool.LNK", "Microsoft Edge"] {
+ assert_eq!(request(name), Ok(Request::Name(name.into())), "{name}");
+ }
+ for path in [r"C:\Program Files\App\app.exe", r"C:\Users\me\Desktop\Tool.LNK", r"D:\Tools\x.EXE"] {
+ assert_eq!(request(path), Ok(Request::Path(path.into())), "{path}");
+ }
+ assert_eq!(request(" Notepad "), Ok(Request::Name("Notepad".into())));
+ // Any Unicode name is read without panicking.
+ assert_eq!(request("a€"), Ok(Request::Name("a€".into())));
+ assert_eq!(request("€:"), Err("`€:` is not an app name: URLs and protocols cannot be opened.".into()));
+ }
+
+ #[test]
+ fn urls_protocols_command_lines_scripts_documents_and_shares_are_refused() {
+ for name in [
+ "", " ", "ms-settings:", "ms-settings:privacy", "https://example.com", "file:///C:/x.exe", "cmd /c del x",
+ "powershell -c x", "a & b", "a | b", "%COMSPEC%", "x*.exe", "notes.txt", "payload.cmd", "run.bat", "x.vbs",
+ "chrome.exe.", "a\u{7}b", r"C:\x\notes.txt", r"\\server\share\app.exe", r"C:\a:b.exe", r"C:\x\..\Windows\cmd.exe",
+ "C:/x/app.exe", r"tools\app.exe",
+ ] {
+ assert!(request(name).is_err(), "{name:?} should be refused");
+ }
+ assert!(request("ms-settings:").unwrap_err().contains("protocols"));
+ assert!(request("cmd /c del x").unwrap_err().contains("command lines"));
+ assert!(request("payload.cmd").unwrap_err().contains("only programs"));
+ }
+
+ #[test]
+ fn a_start_menu_shortcut_is_found_by_its_name_in_any_case() {
+ let shortcuts = [r"C:\ProgramData\Microsoft\Windows\Start Menu\Programs\Google Chrome.lnk".to_string(), r"C:\Users\me\Start Menu\Programs\Notes\NOTES.LNK".to_string()];
+ assert_eq!(shortcut_named("google chrome", &shortcuts), Some(&shortcuts[0]));
+ assert_eq!(shortcut_named("Notes.lnk", &shortcuts), Some(&shortcuts[1]));
+ assert_eq!(shortcut_named("Chrome", &shortcuts), None);
+ }
+
+ #[test]
+ fn only_a_local_program_started_without_arguments_is_launched() {
+ let chrome = r"C:\Program Files\Google\Chrome\Application\chrome.exe";
+ assert_eq!(executable_target(chrome, ""), Ok(chrome.into()));
+ assert_eq!(executable_target(&format!("\"{chrome}\""), " "), Ok(chrome.into()));
+ assert!(executable_target(r"C:\Windows\System32\cmd.exe", "/c del x").unwrap_err().contains("command line"));
+ for target in [r"C:\x\run.cmd", r"C:\x\notes.txt", r"\\server\share\app.exe", r"C:\x\a.exe:stream", r"C:\x\..\y\a.exe", "app.exe", r"C:\x\a.exe.", ""] {
+ assert!(executable_target(target, "").is_err(), "{target:?} should be refused");
+ }
+ }
+
+ #[test]
+ fn suffixes_compare_case_insensitively_and_safely() {
+ assert_eq!(strip_ci("Tool.LnK", ".lnk"), Some("Tool"));
+ assert_eq!(strip_ci("a.Exe", ".exe"), Some("a"));
+ assert_eq!(strip_ci("exe", ".exe"), None);
+ assert_eq!(strip_ci("€exe", ".exe"), None);
+ }
+
+ #[test]
+ fn a_file_explorer_window_showing_credential_manager_is_that_page() {
+ assert_eq!(hosted_page(Some("explorer.exe"), "Credential Manager"), Some("Credential Manager"));
+ assert_eq!(hosted_page(Some("EXPLORER.EXE"), " administrador de credenciales "), Some("Credential Manager"));
+ assert_eq!(hosted_page(Some("explorer.exe"), "Downloads"), None);
+ assert_eq!(hosted_page(Some("notepad.exe"), "Credential Manager"), None);
+ assert_eq!(hosted_page(None, "Credential Manager"), None);
+ }
+
+ #[test]
+ fn a_running_app_answers_to_its_name_or_executable() {
+ assert!(answers_to("notepad", "Notepad", Some("Notepad.exe")));
+ assert!(answers_to("NOTEPAD.EXE", "Notepad", Some("Notepad.exe")));
+ assert!(answers_to("Google Chrome", "Google Chrome", Some("chrome.exe")));
+ assert!(answers_to("chrome", "Google Chrome", Some("chrome.exe")));
+ assert!(!answers_to("chrom", "Google Chrome", Some("chrome.exe")));
+ assert!(!answers_to("Notepad", "Paint", None));
+ assert!(answers_to("notepad", "Notepad", Some("NOTEPAD.Exe")));
+ }
+}