A personal assistant that lives on your desktop, remembers what matters, and can use your Mac for you.
TypeScript
fix(desktop): list minimized Store apps by their own names
Marcelitoe8ac7c1ca261
diff --git a/apps/desktop/src-tauri/src/computer/win.rs b/apps/desktop/src-tauri/src/computer/win.rsindex 109da18..04fb3c7 100644--- a/apps/desktop/src-tauri/src/computer/win.rs+++ b/apps/desktop/src-tauri/src/computer/win.rs@@ -29,8 +29,8 @@ use super::risk::{AxElement, Focus};use super::glow::Frame;use super::winaccess::{element, is_default, known_chain, unreadable, UIA_BUTTON, UIA_WINDOW};use super::winapps::{- answers_to, app_pids, executable_target, fronts_same_app, hosted_page, request, shortcut_named, strip_ci, Request,- TopWindow,+ answers_to, executable_target, fronts_same_app, hosted_page, listed_name, listed_windows, request, shortcut_named,+ strip_ci, Request, TopWindow,};use super::winput::{checks_typing_target, releases, send_batches, settles, steps, Step, VirtualScreen};use super::{@@ -915,18 +915,99 @@ pub fn window_owner_at(at: Point) -> Option<AppIdentity> {(!root.is_null()).then(|| target_of(root))}+/// The shell display name for the AppUserModelID stored on a frame. These shell COM interfaces+/// are used only here; all returned objects and strings are released before leaving the apartment.+fn frame_app_name(hwnd: HWND) -> Option<String> {+ use windows_sys::core::GUID;++ #[repr(C)]+ struct PropertyKey { fmtid: GUID, pid: u32 }+ // PROPVARIANT: the value union has room for two pointers on either supported pointer width.+ #[repr(C)]+ struct PropVariant { vt: u16, reserved: [u16; 3], value: [usize; 2] }+ #[repr(C)]+ struct PropertyStoreVtable {+ prefix: [*const c_void; 5],+ get_value: unsafe extern "system" fn(*mut c_void, *const PropertyKey, *mut PropVariant) -> i32,+ }+ #[repr(C)]+ struct ShellItemVtable {+ prefix: [*const c_void; 5],+ get_display_name: unsafe extern "system" fn(*mut c_void, u32, *mut *mut u16) -> i32,+ }+ #[link(name = "shell32")]+ extern "system" {+ fn SHGetPropertyStoreForWindow(hwnd: HWND, iid: *const GUID, out: *mut *mut c_void) -> i32;+ fn SHCreateItemInKnownFolder(folder: *const GUID, flags: u32, path: *const u16, iid: *const GUID, out: *mut *mut c_void) -> i32;+ }+ #[link(name = "ole32")]+ extern "system" {+ fn PropVariantClear(value: *mut PropVariant) -> i32;+ fn CoTaskMemFree(ptr: *mut c_void);+ }+ struct ComPtr(*mut c_void);+ impl Drop for ComPtr {+ fn drop(&mut self) {+ // SAFETY: all ComPtr values hold a live interface with IUnknown as its first vtable.+ unsafe {+ let vtable = *(self.0 as *const *const *const c_void);+ let release: unsafe extern "system" fn(*mut c_void) -> u32 = std::mem::transmute(*vtable.add(2));+ release(self.0);+ }+ }+ }+ const STORE: GUID = GUID::from_u128(0x886d8eeb_8cf2_4446_8d02_cdba1dbdcf99);+ const ITEM: GUID = GUID::from_u128(0x43826d1e_e718_42ee_bc55_a1e261c37bfe);+ const APPS: GUID = GUID::from_u128(0x1e87508d_89c2_42f0_8a7e_645a0f50ca58);+ const AUMID: PropertyKey = PropertyKey { fmtid: GUID::from_u128(0x9f4c2855_9f79_4b39_a8d0_e1d42de1d5f3), pid: 5 };+ let _apartment = Apartment::join(Apartment::SHELL);+ // SAFETY: shell calls fill the indicated out-pointers; successful interfaces are released,+ // the PROPVARIANT is cleared, and the shell-allocated display name is freed.+ unsafe {+ let mut raw = null_mut();+ if SHGetPropertyStoreForWindow(hwnd, &STORE, &mut raw) < 0 || raw.is_null() { return None; }+ let store = ComPtr(raw);+ let vtable = &**(store.0 as *const *const PropertyStoreVtable);+ let mut value: PropVariant = zeroed();+ let result = (vtable.get_value)(store.0, &AUMID, &mut value);+ let mut item_raw = null_mut();+ if result >= 0 && value.vt == 31 && value.value[0] != 0 {+ SHCreateItemInKnownFolder(&APPS, 0, value.value[0] as *const u16, &ITEM, &mut item_raw);+ }+ PropVariantClear(&mut value);+ if item_raw.is_null() { return None; }+ let item = ComPtr(item_raw);+ let vtable = &**(item.0 as *const *const ShellItemVtable);+ let mut name = null_mut();+ if (vtable.get_display_name)(item.0, 0, &mut name) < 0 || name.is_null() { return None; }+ let mut len = 0;+ while *name.add(len) != 0 { len += 1; }+ let result = String::from_utf16_lossy(std::slice::from_raw_parts(name, len));+ CoTaskMemFree(name.cast());+ Some(result)+ }+}+/// The running apps, front to back, or why they cannot be listed (a partial list would read as/// apps that are not running).pub fn list_apps() -> Result<Vec<AppInfo>, String> {let active = frontmost_app().map(|app| app.pid);- let windows: Vec<TopWindow> = top_windows()?.into_iter().map(|(_, window)| window).collect();- Ok(app_pids(&windows)- .into_iter()- .map(|pid| {- let id = identity(pid);- AppInfo { name: id.name, bundle_id: id.bundle_id, active: active == Some(pid) }- })- .collect())+ let windows = top_windows()?;+ let listing: Vec<(TopWindow, bool)> = windows.iter().map(|(hwnd, window)| {+ let detached = class_name(*hwnd).as_deref() == Some(FRAME_CLASS)+ && app_pid(*hwnd).is_none() && is_frame_host(&identity(window.pid));+ (window.clone(), detached)+ }).collect();+ Ok(listed_windows(&listing).into_iter().map(|index| {+ let (hwnd, _) = &windows[index];+ let (window, detached) = &listing[index];+ let id = identity(window.pid);+ // A detached frame is only named for listing; it cannot be acted on until restored and+ // identified. target_of and every blocklist/activation check keep their existing identity.+ let name = listed_name(*detached, if *detached { frame_app_name(*hwnd) } else { None },+ if *detached { Some(window.title.clone()) } else { None }, id.name);+ AppInfo { name, bundle_id: id.bundle_id, active: active == Some(window.pid) }+ }).collect())}/// Brings a window to the front. Windows lets only the process that received the last input@@ -1765,6 +1846,21 @@ impl super::Desktop for WinDesktop {mod tests {use super::*;+ #[test]+ #[ignore = "interactive Windows desktop: minimize Settings before running"]+ fn detached_settings_is_still_an_unidentified_refused_target() {+ let (hwnd, _) = top_windows().unwrap().into_iter().find(|(hwnd, window)| {+ window.is_app() && class_name(*hwnd).as_deref() == Some(FRAME_CLASS)+ && app_pid(*hwnd).is_none() && is_frame_host(&identity(window.pid))+ && listed_name(true, frame_app_name(*hwnd), Some(window.title.clone()), String::new())+ .eq_ignore_ascii_case("Settings")+ }).expect("minimize Settings in an English Windows session first");+ let target = target_of(hwnd);+ assert_eq!(target.name, UNIDENTIFIED);+ assert!(target.bundle_id.is_none());+ assert!(blocked_reason(&target, std::process::id() as i32).is_some());+ }+fn per_monitor_aware() {use windows_sys::Win32::UI::HiDpi::{SetProcessDpiAwarenessContext, DPI_AWARENESS_CONTEXT_PER_MONITOR_AWARE_V2};// SAFETY: plain call; the app itself is already per-monitor aware.diff --git a/apps/desktop/src-tauri/src/computer/winapps.rs b/apps/desktop/src-tauri/src/computer/winapps.rsindex 85be7aa..aa9a8d6 100644--- a/apps/desktop/src-tauri/src/computer/winapps.rs+++ b/apps/desktop/src-tauri/src/computer/winapps.rs@@ -26,6 +26,7 @@ impl TopWindow {}/// The processes with an app window, front to back, each once.+#[cfg(test)]pub fn app_pids(windows: &[TopWindow]) -> Vec<i32> {let mut pids = Vec::new();for window in windows.iter().filter(|w| w.is_app()) {@@ -36,6 +37,31 @@ pub fn app_pids(windows: &[TopWindow]) -> Vec<i32> {pids}+/// Indices to list, given each window and whether its Store frame is detached.+pub fn listed_windows(windows: &[(TopWindow, bool)]) -> Vec<usize> {+ let mut pids = Vec::new();+ let mut indices = Vec::new();+ for (index, (window, detached)) in windows.iter().enumerate() {+ if window.is_app() && (*detached || !pids.contains(&window.pid)) {+ if !detached {+ pids.push(window.pid);+ }+ indices.push(index);+ }+ }+ indices+}++/// Display name for a detached frame; never changes the action identity.+pub fn listed_name(detached: bool, aumid_name: Option<String>, title: Option<String>, original: String) -> String {+ if !detached {+ return original;+ }+ aumid_name.filter(|name| !name.trim().is_empty())+ .or_else(|| title.filter(|name| !name.trim().is_empty()))+ .unwrap_or(original)+}+/// What `open_app` was asked for: an app's name, or a local program or shortcut by path.#[derive(Debug, Clone, PartialEq, Eq)]pub enum Request {@@ -153,6 +179,25 @@ mod tests {TopWindow { pid, title: title.into(), visible: true, cloaked: false, tool: false, app_window: false, owned: false }}+ #[test]+ fn detached_frame_name_prefers_app_name_then_title_then_host() {+ let s = |x: &str| Some(x.to_string());+ assert_eq!(listed_name(true, s("Calculator"), s("Title"), "Application Frame Host".into()), "Calculator");+ assert_eq!(listed_name(true, None, s("Settings"), "Application Frame Host".into()), "Settings");+ assert_eq!(listed_name(true, None, s(" "), "Application Frame Host".into()), "Application Frame Host");+ assert_eq!(listed_name(true, None, None, "Application Frame Host".into()), "Application Frame Host");+ assert_eq!(listed_name(false, s("wrong"), s("wrong"), "Notepad".into()), "Notepad");+ }++ #[test]+ fn detached_frames_are_listed_individually_other_apps_by_pid() {+ let windows = [(window(7, "Calculator"), true), (window(7, "Settings"), true),+ (window(9, "attached"), false), (window(9, "attached again"), false),+ (window(10, "Notepad"), false), (window(10, "Notepad again"), false),+ (TopWindow { visible: false, ..window(7, "hidden") }, true)];+ assert_eq!(listed_windows(&windows), [0, 1, 2, 4]);+ }+#[test]fn a_window_counts_as_in_front_only_with_a_real_process() {// The window itself, or another window of its process under the same owner (its dialog).diff --git a/docs/design.md b/docs/design.mdindex f32cbf3..23213b9 100644--- a/docs/design.md+++ b/docs/design.md@@ -170,7 +170,7 @@ The UI shows one endless chat: no conversations list, no new conversation, and tThe assistant can see the screen and operate apps through a built-in MCP server that the desktop app serves. macOS (L59) and Windows (S24.11); Linux, the web page and a VPS never offer it.- S24.1 Helper: native Rust inside the Tauri app (`apps/desktop/src-tauri`), never in the daemon or the engine. It serves MCP (JSON-RPC 2.0 over streamable HTTP) on `http://127.0.0.1:<random port>/mcp` and requires `Authorization: Bearer <key>`, a fresh random key per app launch.-- S24.2 Tools: `screenshot`, `click`, `move`, `drag`, `scroll`, `type`, `key`, `open_app`, `list_apps`, `wait`, each with an optional `intent`. Screenshots of the display under the pointer are downscaled to a long edge of 1280 px; coordinates are pixels of the latest screenshot; actions return a fresh one; Gentle Dot's own windows are left out.+- S24.2 Tools: `screenshot`, `click`, `move`, `drag`, `scroll`, `type`, `key`, `open_app`, `list_apps`, `wait`, each with an optional `intent`. Screenshots of the display under the pointer are downscaled to a long edge of 1280 px; coordinates are pixels of the latest screenshot; actions return a fresh one; Gentle Dot's own windows are left out. On Windows, a detached Store app frame is listed by its shell display name (or its window title, or the frame host when neither is available), separately from other detached frames. This name is for listing only: the minimized app cannot be acted on until it is restored and identified; the action blocklist is unchanged.- S24.3 Session grant: the first call without a session shows a native dialog ("Allow Gentle Dot to see and control this Mac for 30 minutes?"). A session ends on timeout, Stop, the tray item, or the panic shortcut ⌥⇧Esc, and queued actions are dropped.- S24.4 Guards: a fixed blocklist (Gentle Dot itself, System Settings, SecurityAgent, loginwindow, Keychain Access, password managers), 10 actions per second, and a native per-action confirmation for risky actions (send, pay, buy, delete, submit, confirm, transfer, and their Spanish equivalents, by Accessibility role and title, by `return` on a default button, or by the declared `intent`). Keyboard actions go only to the app that was steadily in front during the latest screenshot; their recipient is fixed when they are checked and confirmed and rechecked before each chunk of text, each Return, and the key, and a change stops the input with the count of characters already sent. After a confirmation, the app in front before the dialog must be back in front within 500 ms, or nothing is done.- S24.5 Permissions: the app reports and requests Accessibility and Screen Recording (`computer_permissions`, `computer_request_permission {kind}`). The Connectors screen shows a built-in "Computer" entry with both statuses, Grant buttons, Check again, the note that debug builds lose the grants on every rebuild (macOS ties them to the code signature), and a note when the current model does not accept images (`ConnectorInfo.noImages`, from the model's `input` in the engine's `get_state` and `set_model` answers).