A personal assistant that lives on your desktop, remembers what matters, and can use your Mac for you.
TypeScript
fix(desktop): tell an unreadable Windows version from one too old for screenshots
Marcelito2c7232cd8572
diff --git a/apps/desktop/src-tauri/src/computer/app.rs b/apps/desktop/src-tauri/src/computer/app.rsindex 0743aaa..bf786dc 100644--- a/apps/desktop/src-tauri/src/computer/app.rs+++ b/apps/desktop/src-tauri/src/computer/app.rs@@ -520,7 +520,7 @@ pub fn computer_permissions(computer: State<'_, Computer>) -> Permissions {#[cfg(not(any(target_os = "macos", windows)))]{let _ = computer;- Permissions { accessibility: false, screen_recording: false, screenshots_supported: false }+ Permissions { accessibility: false, screen_recording: false, screenshots_supported: false, screenshots_unsupported: None }}}@@ -715,7 +715,32 @@ Use Stop in the panel or the tray menu instead."#[test]fn permissions_serialize_with_the_l60_field_names() {- let value = serde_json::to_value(Permissions { accessibility: true, screen_recording: false, screenshots_supported: true }).unwrap();+ let value = serde_json::to_value(Permissions {+ accessibility: true,+ screen_recording: false,+ screenshots_supported: true,+ screenshots_unsupported: None,+ })+ .unwrap();assert_eq!(value, serde_json::json!({"accessibility": true, "screenRecording": false, "screenshotsSupported": true}));}++ /// Why screenshots are unsupported rides next to the bool, which stays as it was for every consumer.+ #[test]+ fn unsupported_screenshots_say_why() {+ let unsupported = |reason| Permissions {+ accessibility: true,+ screen_recording: true,+ screenshots_supported: false,+ screenshots_unsupported: Some(reason),+ };+ assert_eq!(+ serde_json::to_value(unsupported(super::super::ScreenshotsUnsupported::TooOld)).unwrap(),+ serde_json::json!({"accessibility": true, "screenRecording": true, "screenshotsSupported": false, "screenshotsUnsupported": "tooOld"})+ );+ assert_eq!(+ serde_json::to_value(unsupported(super::super::ScreenshotsUnsupported::UnknownVersion)).unwrap(),+ serde_json::json!({"accessibility": true, "screenRecording": true, "screenshotsSupported": false, "screenshotsUnsupported": "unknownVersion"})+ );+ }}diff --git a/apps/desktop/src-tauri/src/computer/fake.rs b/apps/desktop/src-tauri/src/computer/fake.rsindex 834b129..0a13c16 100644--- a/apps/desktop/src-tauri/src/computer/fake.rs+++ b/apps/desktop/src-tauri/src/computer/fake.rs@@ -61,7 +61,12 @@ impl Default for FakeDesktop {FakeDesktop {events: Mutex::new(Vec::new()),typed_posts: Mutex::new(Vec::new()),- permissions: Mutex::new(Permissions { accessibility: true, screen_recording: true, screenshots_supported: true }),+ permissions: Mutex::new(Permissions {+ accessibility: true,+ screen_recording: true,+ screenshots_supported: true,+ screenshots_unsupported: None,+ }),frontmost: Mutex::new(Some(safari)),focused_app: Mutex::new(None),front_after_posts: Mutex::new(None),diff --git a/apps/desktop/src-tauri/src/computer/macos.rs b/apps/desktop/src-tauri/src/computer/macos.rsindex b8220ba..474d4a5 100644--- a/apps/desktop/src-tauri/src/computer/macos.rs+++ b/apps/desktop/src-tauri/src/computer/macos.rs@@ -10,7 +10,7 @@ use super::glow::Frame;use super::keys::KeyCombo;use super::risk::{AxElement, Focus};use super::windows::{hit_test_app, owner_row_at, WindowRow};-use super::{AppInfo, Capture, Desktop, InputEvent, MouseButton, PermissionKind, Permissions};+use super::{AppInfo, Capture, Desktop, InputEvent, MouseButton, PermissionKind, Permissions, ScreenshotsUnsupported};use block2::RcBlock;use objc2::rc::Retained;use objc2::runtime::AnyObject;@@ -460,11 +460,13 @@ fn window_rows() -> Vec<WindowRow> {impl Desktop for MacDesktop {fn permissions(&self) -> Permissions {+ let screenshots_supported = available!(macos = 14.0);Permissions {// SAFETY: plain queries.accessibility: unsafe { AXIsProcessTrusted() },screen_recording: CGPreflightScreenCaptureAccess(),- screenshots_supported: available!(macos = 14.0),+ screenshots_supported,+ screenshots_unsupported: (!screenshots_supported).then_some(ScreenshotsUnsupported::TooOld),}}diff --git a/apps/desktop/src-tauri/src/computer/mcp.rs b/apps/desktop/src-tauri/src/computer/mcp.rsindex 1a588e6..650dda2 100644--- a/apps/desktop/src-tauri/src/computer/mcp.rs+++ b/apps/desktop/src-tauri/src/computer/mcp.rs@@ -26,8 +26,8 @@ use super::rate::{RateLimiter, MAX_ACTIONS_PER_SECOND};use super::risk::{assess, Act};use super::working::{NoPreparation, Preparation};use super::{- Clock, Desktop, Dialogs, InputEvent, KeyboardRecipient, MouseButton, TypedError, KEYS_CHANGED_BEFORE_TYPING,- KEYS_CHANGED_WHILE_TYPING,+ Clock, Desktop, Dialogs, InputEvent, KeyboardRecipient, MouseButton, ScreenshotsUnsupported, TypedError,+ KEYS_CHANGED_BEFORE_TYPING, KEYS_CHANGED_WHILE_TYPING,};use base64::Engine as _;use serde_json::{json, Map, Value};@@ -78,6 +78,23 @@ const SCREENSHOTS_UNSUPPORTED: &str = "Screenshots need macOS 14 or later, so co#[cfg(windows)]const SCREENSHOTS_UNSUPPORTED: &str ="Screenshots need Windows 10 version 2004 or later, so computer control is unavailable on this PC.";+/// Why computer control is unavailable when Windows' build cannot be read.+#[cfg(windows)]+const SCREENSHOTS_UNKNOWN_VERSION: &str = "Gentle Dot cannot tell which Windows version this is, so it cannot \+keep its own windows out of screenshots and computer control is unavailable on this PC.";++/// The error for screenshots the system cannot take; one without a reason counts as too old.+#[cfg(not(windows))]+fn screenshots_unsupported(_: Option<ScreenshotsUnsupported>) -> &'static str {+ SCREENSHOTS_UNSUPPORTED+}+#[cfg(windows)]+fn screenshots_unsupported(reason: Option<ScreenshotsUnsupported>) -> &'static str {+ match reason {+ Some(ScreenshotsUnsupported::UnknownVersion) => SCREENSHOTS_UNKNOWN_VERSION,+ Some(ScreenshotsUnsupported::TooOld) | None => SCREENSHOTS_UNSUPPORTED,+ }+}/// Why a key was not sent when the app receiving it changed after it was checked.const KEY_RECIPIENT_CHANGED: &str = "The app receiving the keys changed before the key was sent, so it was not sent.";@@ -536,7 +553,7 @@ impl Helper {| Tool::Key { .. });if sees && !permissions.screenshots_supported {- return Err(SCREENSHOTS_UNSUPPORTED.into());+ return Err(screenshots_unsupported(permissions.screenshots_unsupported).into());}if sees && !permissions.screen_recording {return Err(format!("{NEEDS_PERMISSION} Screen Recording permission. {GRANT_IT}"));@@ -925,7 +942,7 @@ mod tests {use super::super::keys::parse_combo;use super::super::risk::{AxElement, Focus};use super::super::session::{Reason, StateEvent, SESSION_MS};- use super::super::{InputEvent, MouseButton};+ use super::super::{InputEvent, MouseButton, ScreenshotsUnsupported};use super::*;// Blocked apps of the platform the tests run on (each has its own list): bundle id or@@ -1380,6 +1397,47 @@ mod tests {);}+ #[test]+ fn screenshots_on_a_system_too_old_say_so() {+ let rig = rig();+ {+ let mut permissions = rig.desktop.permissions.lock().unwrap();+ permissions.screenshots_supported = false;+ permissions.screenshots_unsupported = Some(ScreenshotsUnsupported::TooOld);+ }+ let result = rig.call("screenshot", json!({}));+ assert!(is_error(&result));+ #[cfg(not(windows))]+ assert_eq!(first_text(&result), "Screenshots need macOS 14 or later, so computer control is unavailable on this Mac.");+ #[cfg(windows)]+ assert_eq!(+ first_text(&result),+ "Screenshots need Windows 10 version 2004 or later, so computer control is unavailable on this PC."+ );+ }++ /// An unreadable Windows build is not called too old; screenshots stay off all the same.+ #[test]+ fn screenshots_on_an_unknown_windows_version_say_so() {+ let rig = rig();+ {+ let mut permissions = rig.desktop.permissions.lock().unwrap();+ permissions.screenshots_supported = false;+ permissions.screenshots_unsupported = Some(ScreenshotsUnsupported::UnknownVersion);+ }+ let result = rig.call("screenshot", json!({}));+ assert!(is_error(&result));+ #[cfg(not(windows))]+ assert_eq!(first_text(&result), "Screenshots need macOS 14 or later, so computer control is unavailable on this Mac.");+ #[cfg(windows)]+ assert_eq!(+ first_text(&result),+ "Gentle Dot cannot tell which Windows version this is, so it cannot keep its own windows out of \+screenshots and computer control is unavailable on this PC."+ );+ assert_eq!(rig.desktop.captures.load(Ordering::SeqCst), 0);+ }+// --- Blocklist and rate limit (S24.4) ---#[test]diff --git a/apps/desktop/src-tauri/src/computer/mod.rs b/apps/desktop/src-tauri/src/computer/mod.rsindex 98c43e5..faf584c 100644--- a/apps/desktop/src-tauri/src/computer/mod.rs+++ b/apps/desktop/src-tauri/src/computer/mod.rs@@ -186,13 +186,26 @@ pub struct AppInfo {/// The `computer_permissions` reply (L60), plus `screenshotsSupported`: false before macOS 14,/// where ScreenCaptureKit cannot take screenshots, and before Windows 10 version 2004, which-/// cannot leave Gentle Dot's windows out of them.+/// cannot leave Gentle Dot's windows out of them. `screenshotsUnsupported` says why, only when+/// they are unsupported.#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]#[serde(rename_all = "camelCase")]pub struct Permissions {pub accessibility: bool,pub screen_recording: bool,pub screenshots_supported: bool,+ #[serde(skip_serializing_if = "Option::is_none")]+ pub screenshots_unsupported: Option<ScreenshotsUnsupported>,+}++/// Why screenshots are unsupported. Either way there are none.+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]+#[serde(rename_all = "camelCase")]+pub enum ScreenshotsUnsupported {+ /// Before macOS 14, or before Windows 10 version 2004.+ TooOld,+ /// Windows' build number could not be read, so it is not trusted to be recent enough.+ UnknownVersion,}/// Which macOS permission the user is asked for (`computer_request_permission {kind}`, L60).diff --git a/apps/desktop/src-tauri/src/computer/win.rs b/apps/desktop/src-tauri/src/computer/win.rsindex 05332ed..e853cc4 100644--- a/apps/desktop/src-tauri/src/computer/win.rs+++ b/apps/desktop/src-tauri/src/computer/win.rs@@ -37,8 +37,8 @@ use super::winput::{VirtualScreen,};use super::{- runs_as_administrator, AppInfo, Capture, InputEvent, KeyboardRecipient, MouseButton, TypedError,- KEYS_CHANGED_WHILE_TYPING,+ runs_as_administrator, AppInfo, Capture, InputEvent, KeyboardRecipient, MouseButton, ScreenshotsUnsupported,+ TypedError, KEYS_CHANGED_WHILE_TYPING,};use std::ffi::c_void;use std::mem::{size_of, zeroed};@@ -1767,11 +1767,16 @@ fn windows_build() -> Option<u32> {String::from_utf16_lossy(units).trim_end_matches('\0').trim().parse().ok()}-/// Whether screenshots can leave Gentle Dot's windows out. A build that cannot be read counts as-/// unable: before Windows 10 version 2004 the exclusion does not fail, it just leaves a black box-/// (`WDA_MONITOR`), so a failed exclusion cannot be relied on to say so.-fn supports_capture_exclusion(build: Option<u32>) -> bool {- build.is_some_and(|build| build >= CAPTURE_EXCLUSION_BUILD)+/// Why screenshots cannot leave Gentle Dot's windows out, or `None` when they can. A build that+/// cannot be read counts as unable too: before Windows 10 version 2004 the exclusion does not+/// fail, it just leaves a black box (`WDA_MONITOR`), so a failed exclusion cannot be relied on to+/// say so.+fn capture_exclusion_unsupported(build: Option<u32>) -> Option<ScreenshotsUnsupported> {+ match build {+ None => Some(ScreenshotsUnsupported::UnknownVersion),+ Some(build) if build < CAPTURE_EXCLUSION_BUILD => Some(ScreenshotsUnsupported::TooOld),+ Some(_) => None,+ }}/// The Windows desktop the helper drives.@@ -1785,10 +1790,12 @@ impl super::Desktop for WinDesktop {fn permissions(&self) -> super::Permissions {// Windows asks for no permission to send input or take screenshots; leaving Gentle Dot's// windows out of them needs a recent enough build.+ let screenshots_unsupported = capture_exclusion_unsupported(windows_build());super::Permissions {accessibility: true,screen_recording: true,- screenshots_supported: supports_capture_exclusion(windows_build()),+ screenshots_supported: screenshots_unsupported.is_none(),+ screenshots_unsupported,}}@@ -1988,19 +1995,27 @@ mod tests {assert_eq!(desktop.keyboard_refusal(&ghost), keyboard_refusal(&ghost));let permissions = desktop.permissions();assert!(permissions.accessibility && permissions.screen_recording);- assert_eq!(permissions.screenshots_supported, supports_capture_exclusion(windows_build()));+ assert_eq!(permissions.screenshots_unsupported, capture_exclusion_unsupported(windows_build()));+ assert_eq!(permissions.screenshots_supported, permissions.screenshots_unsupported.is_none());}/// The machine's own answer follows its build, so the test holds before 2004 too.#[test]fn screenshots_need_windows_10_2004() {- assert!(!supports_capture_exclusion(Some(19040)));- assert!(supports_capture_exclusion(Some(19041)));- assert!(supports_capture_exclusion(Some(26100)));- assert!(!supports_capture_exclusion(None), "an unreadable build is not trusted");+ assert_eq!(capture_exclusion_unsupported(Some(19040)), Some(ScreenshotsUnsupported::TooOld));+ assert_eq!(capture_exclusion_unsupported(Some(19041)), None);+ assert_eq!(capture_exclusion_unsupported(Some(26100)), None);+ assert_eq!(+ capture_exclusion_unsupported(None),+ Some(ScreenshotsUnsupported::UnknownVersion),+ "an unreadable build is not trusted, nor called too old"+ );let build = windows_build().expect("the build number is readable");use super::super::Desktop;- assert_eq!(WinDesktop.permissions().screenshots_supported, build >= CAPTURE_EXCLUSION_BUILD, "{build}");+ let permissions = WinDesktop.permissions();+ assert_eq!(permissions.screenshots_supported, build >= CAPTURE_EXCLUSION_BUILD, "{build}");+ let too_old = (build < CAPTURE_EXCLUSION_BUILD).then_some(ScreenshotsUnsupported::TooOld);+ assert_eq!(permissions.screenshots_unsupported, too_old, "{build}");}/// Windows that could not be listed may include ours: the capture is not clean. Nothingdiff --git a/docs/design.md b/docs/design.mdindex e5c2256..28879a3 100644--- a/docs/design.md+++ b/docs/design.md@@ -189,7 +189,7 @@ The assistant can see the screen and operate apps through a built-in MCP server- S24.11 Windows: the helper uses the same session and guards, with Windows-specific input, screenshots, app discovery and UI Automation.- The panic shortcut is Alt+Shift+Esc; the panel says "Controlling your PC", and Connectors shows no macOS permission rows.- - Screenshots require Windows 10 version 2004 (build 19041) or newer for `WDA_EXCLUDEFROMCAPTURE`. Each screenshot leaves out every visible Gentle Dot window and, right after it is taken, lets the ones it excluded show again, so the user's own screenshots still show the Dot and the panel; windows that are always left out (the glow, the approval dialogs) stay out. A dialog that cannot be left out holds screenshots off while it is open, and a build that cannot be read counts as too old.+ - Screenshots require Windows 10 version 2004 (build 19041) or newer for `WDA_EXCLUDEFROMCAPTURE`. Each screenshot leaves out every visible Gentle Dot window and, right after it is taken, lets the ones it excluded show again, so the user's own screenshots still show the Dot and the panel; windows that are always left out (the glow, the approval dialogs) stay out. A dialog that cannot be left out holds screenshots off while it is open. On an older build `computer_permissions` answers `screenshotsSupported: false` with `screenshotsUnsupported: "tooOld"`; a build that cannot be read gets no screenshots either, with `screenshotsUnsupported: "unknownVersion"`, so the helper's error and the Connectors note say that Gentle Dot cannot tell the Windows version instead of calling it too old. `screenshotsUnsupported` is omitted when screenshots are supported (macOS before 14 sends `"tooOld"`), and a `false` without it reads as too old.- Keyboard input to an elevated app is refused: "{app} runs as administrator, so Gentle Dot cannot send it keys."- `open_app` resolves names through Start menu shortcuts and App Paths, not PATH; it refuses unsafe targets and checks the resolved executable against the blocklist.- UI Automation failures are fail-closed: if any element in an ancestor chain is unreadable, the entire chain is unreadable and asks for confirmation. A default-button scan cut short is also unreadable.diff --git a/packages/ui/src/components/Computer.tsx b/packages/ui/src/components/Computer.tsxindex 8b154df..405808c 100644--- a/packages/ui/src/components/Computer.tsx+++ b/packages/ui/src/components/Computer.tsx@@ -33,6 +33,10 @@ export const DEBUG_BUILD_NOTE ="Debug builds lose these permissions on every rebuild because macOS ties them to the app's signature; grant them again after updating.";const yoloNote = (windows: boolean) =>`Yolo mode lets the assistant send, pay, delete, and submit without asking you first. It turns itself off after 1 hour or when the app quits; Stop and ${windows ? "Alt+Shift+Esc" : "⌥⇧Esc"} still work.`;+const WINDOWS_TOO_OLD_NOTE =+ "Computer control needs Windows 10 version 2004 (build 19041) or newer for screenshots.";+const WINDOWS_UNKNOWN_VERSION_NOTE =+ "Gentle Dot cannot tell which Windows version this is, so computer control is unavailable: screenshots need Windows 10 version 2004 (build 19041) or newer.";const NO_IMAGES_NOTE ="The current model does not accept images, so the assistant cannot see your screen. Choose a model that does in Profiles.";@@ -137,7 +141,9 @@ export function ComputerEntry({ info, controls }: ComputerEntryProps) {)}{windows && granted?.screenshotsSupported === false ? (<p className="connector-note computer-warning">- Computer control needs Windows 10 version 2004 (build 19041) or newer for screenshots.+ {granted.screenshotsUnsupported === "unknownVersion"+ ? WINDOWS_UNKNOWN_VERSION_NOTE+ : WINDOWS_TOO_OLD_NOTE}</p>) : null}<div className="computer-yolo">diff --git a/packages/ui/src/computer.ts b/packages/ui/src/computer.tsindex 5d00457..c490bd2 100644--- a/packages/ui/src/computer.ts+++ b/packages/ui/src/computer.ts@@ -15,6 +15,8 @@ export interface ComputerPermissions {accessibility: boolean;screenRecording: boolean;screenshotsSupported?: boolean;+ /** Why screenshots are unsupported, only when they are; an older app sends none. */+ screenshotsUnsupported?: "tooOld" | "unknownVersion";}export type ComputerPermission = "accessibility" | "screenRecording";diff --git a/packages/ui/test/computer.test.tsx b/packages/ui/test/computer.test.tsxindex d613d0a..2ede02a 100644--- a/packages/ui/test/computer.test.tsx+++ b/packages/ui/test/computer.test.tsx@@ -4,17 +4,24 @@ import { APP_REQUIRED, type ConnectorInfo } from "@gentle-dot/protocol";import { act, render, screen, waitFor, within } from "@testing-library/react";import userEvent from "@testing-library/user-event";import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";+import type { ComputerPermissions } from "../src/computer.ts";import { type DotState, initialState } from "../src/store.ts";const HELPER = "http://127.0.0.1:51234/mcp";const KEY = "s3cr3t-Computer-Key-0123456789abcdef";const WINDOWS_2004_NOTE ="Computer control needs Windows 10 version 2004 (build 19041) or newer for screenshots.";+const UNKNOWN_WINDOWS_NOTE =+ "Gentle Dot cannot tell which Windows version this is, so computer control is unavailable: screenshots need Windows 10 version 2004 (build 19041) or newer.";const h = vi.hoisted(() => ({tauri: true,endpoint: null as unknown,- permissions: { accessibility: false, screenRecording: true, screenshotsSupported: true },+ permissions: {+ accessibility: false,+ screenRecording: true,+ screenshotsSupported: true,+ } as ComputerPermissions,status: { active: false } as unknown,invoke: vi.fn(),send: vi.fn(),@@ -297,6 +304,25 @@ describe("on Windows", () => {h.state = connectorsOpen([computerEntry()]);await renderApp("?surface=panel");expect(await screen.findByText(WINDOWS_2004_NOTE)).toBeInTheDocument();+ expect(screen.queryByText(UNKNOWN_WINDOWS_NOTE)).toBeNull();+ });++ it("says this Windows is too old when the app says so", async () => {+ h.permissions.screenshotsSupported = false;+ h.permissions.screenshotsUnsupported = "tooOld";+ h.state = connectorsOpen([computerEntry()]);+ await renderApp("?surface=panel");+ expect(await screen.findByText(WINDOWS_2004_NOTE)).toBeInTheDocument();+ expect(screen.queryByText(UNKNOWN_WINDOWS_NOTE)).toBeNull();+ });++ it("says the Windows version is unknown, not too old, when the app cannot tell it", async () => {+ h.permissions.screenshotsSupported = false;+ h.permissions.screenshotsUnsupported = "unknownVersion";+ h.state = connectorsOpen([computerEntry()]);+ await renderApp("?surface=panel");+ expect(await screen.findByText(UNKNOWN_WINDOWS_NOTE)).toBeInTheDocument();+ expect(screen.queryByText(WINDOWS_2004_NOTE)).toBeNull();});it("says nothing about Windows 10 version 2004 when screenshots are supported", async () => {@@ -305,6 +331,7 @@ describe("on Windows", () => {const entry = await screen.findByRole("listitem", { name: "Computer" });await within(entry).findByText("Windows asks for no permission for computer control.");expect(within(entry).queryByText(WINDOWS_2004_NOTE)).toBeNull();+ expect(within(entry).queryByText(UNKNOWN_WINDOWS_NOTE)).toBeNull();});it("names the PC in the session banner", async () => {@@ -324,12 +351,14 @@ describe("on macOS", () => {it("shows none of the Windows texts, even when screenshots are reported unsupported", async () => {h.permissions.screenshotsSupported = false;+ h.permissions.screenshotsUnsupported = "unknownVersion";h.state = connectorsOpen([computerEntry()]);await renderApp("?surface=panel");const entry = await screen.findByRole("listitem", { name: "Computer" });await within(entry).findByText("Accessibility");expect(within(entry).queryByText("Windows asks for no permission for computer control.")).toBeNull();expect(within(entry).queryByText(WINDOWS_2004_NOTE)).toBeNull();+ expect(within(entry).queryByText(UNKNOWN_WINDOWS_NOTE)).toBeNull();expect(within(entry).queryByText(/Alt\+Shift\+Esc/)).toBeNull();expect(within(entry).getByText(/Stop and ⌥⇧Esc still work/)).toBeInTheDocument();});