A personal assistant that lives on your desktop, remembers what matters, and can use your Mac for you.
TypeScript
feat(desktop): add the native macOS computer-control helper
Alan Buscaglia176f547c4530
diff --git a/apps/desktop/src-tauri/Cargo.lock b/apps/desktop/src-tauri/Cargo.lockindex 822455d..ccf1e16 100644--- a/apps/desktop/src-tauri/Cargo.lock+++ b/apps/desktop/src-tauri/Cargo.lock@@ -47,6 +47,12 @@ version = "1.0.104"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470"+[[package]]+name = "ascii"+version = "1.1.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "d92bec98840b8f03a5ff5413de5293bfcd8bf96467cf5452609f939ec6f5de16"+[[package]]name = "async-broadcast"version = "0.7.2"@@ -498,6 +504,12 @@ dependencies = ["windows-link",]+[[package]]+name = "chunked_transfer"+version = "1.5.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "6e4de3bc4ea267985becf712dc6d9eed8b04c953b3fcfb339ebc87acd9804901"+[[package]]name = "combine"version = "4.6.8"@@ -1263,7 +1275,18 @@ dependencies = [name = "gentle-dot"version = "0.1.0"dependencies = [+ "base64 0.22.1",+ "block2",+ "getrandom 0.3.4",+ "image","libc",+ "objc2",+ "objc2-app-kit",+ "objc2-application-services",+ "objc2-core-foundation",+ "objc2-core-graphics",+ "objc2-foundation",+ "objc2-screen-capture-kit","serde","serde_json","tauri",@@ -1273,6 +1296,7 @@ dependencies = ["tauri-plugin-global-shortcut","tauri-plugin-opener","tauri-plugin-single-instance",+ "tiny_http",][[package]]@@ -1559,6 +1583,12 @@ version = "1.10.1"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87"+[[package]]+name = "httpdate"+version = "1.0.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9"+[[package]]name = "hyper"version = "1.12.0"@@ -1757,6 +1787,8 @@ dependencies = ["moxcms","num-traits","png 0.18.1",+ "zune-core",+ "zune-jpeg",][[package]]@@ -2279,12 +2311,23 @@ checksum = "d49e936b501e5c5bf01fda3a9452ff86dc3ea98ad5f283e1455153142d97518c"dependencies = ["bitflags 2.13.2","block2",+ "libc","objc2","objc2-core-foundation","objc2-foundation","objc2-quartz-core",]+[[package]]+name = "objc2-application-services"+version = "0.3.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "69282c2b5bc58fba07cb9de2113619532eb551e98efe3d8d695509ef45fbd53b"+dependencies = [+ "bitflags 2.13.2",+ "objc2-core-foundation",+]+[[package]]name = "objc2-cloud-kit"version = "0.3.2"@@ -2413,6 +2456,20 @@ dependencies = ["objc2-foundation",]+[[package]]+name = "objc2-screen-capture-kit"+version = "0.3.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "74b7c5390f477482f001bc354d6571a70db7e4f8d5288e860c45521fbce11394"+dependencies = [+ "block2",+ "libc",+ "objc2",+ "objc2-core-foundation",+ "objc2-core-graphics",+ "objc2-foundation",+]+[[package]]name = "objc2-security"version = "0.3.2"@@ -3994,6 +4051,18 @@ dependencies = ["time-core",]+[[package]]+name = "tiny_http"+version = "0.12.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "389915df6413a2e74fb181895f933386023c71110878cd0825588928e64cdc82"+dependencies = [+ "ascii",+ "chunked_transfer",+ "httpdate",+ "log",+]+[[package]]name = "tinystr"version = "0.8.4"@@ -5246,6 +5315,21 @@ version = "1.0.23"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b"+[[package]]+name = "zune-core"+version = "0.5.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "d56377fd46368984a170bc5aac5567e52ca5da874caa60bea39fcbca78fb658b"++[[package]]+name = "zune-jpeg"+version = "0.5.15"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "27bc9d5b815bc103f142aa054f561d9187d191692ec7c2d1e2b4737f8dbd7296"+dependencies = [+ "zune-core",+]+[[package]]name = "zvariant"version = "5.15.0"diff --git a/apps/desktop/src-tauri/Cargo.toml b/apps/desktop/src-tauri/Cargo.tomlindex 62e2d86..3222dc7 100644--- a/apps/desktop/src-tauri/Cargo.toml+++ b/apps/desktop/src-tauri/Cargo.toml@@ -22,3 +22,18 @@ tauri-plugin-single-instance = "2"serde = { version = "1", features = ["derive"] }serde_json = "1"libc = "0.2"+getrandom = "0.3"+base64 = "0.22"+tiny_http = "0.12"+image = { version = "0.25", default-features = false, features = ["jpeg"] }++# The computer-control helper (S24) is macOS only.+[target.'cfg(target_os = "macos")'.dependencies]+objc2 = "0.6"+block2 = "0.6"+objc2-foundation = { version = "0.3", default-features = false, features = ["std", "NSArray", "NSDictionary", "NSError", "NSString", "NSValue"] }+objc2-core-foundation = { version = "0.3", default-features = false, features = ["std", "CFArray", "CFCGTypes", "CFData", "CFDictionary", "CFNumber", "CFString"] }+objc2-core-graphics = { version = "0.3", default-features = false, features = ["std", "CGDataProvider", "CGDirectDisplay", "CGError", "CGEvent", "CGEventTypes", "CGImage", "CGRemoteOperation", "CGWindow"] }+objc2-app-kit = { version = "0.3", default-features = false, features = ["std", "libc", "NSRunningApplication", "NSWorkspace"] }+objc2-application-services = { version = "0.3", default-features = false, features = ["std", "AXError", "AXUIElement", "HIServices"] }+objc2-screen-capture-kit = { version = "0.3", default-features = false, features = ["std", "block2", "libc", "objc2-core-foundation", "objc2-core-graphics", "SCScreenshotManager", "SCShareableContent", "SCStream"] }diff --git a/apps/desktop/src-tauri/build.rs b/apps/desktop/src-tauri/build.rsindex 93015e8..aba1be9 100644--- a/apps/desktop/src-tauri/build.rs+++ b/apps/desktop/src-tauri/build.rs@@ -2,7 +2,17 @@ use std::env;use std::path::{Path, PathBuf};/// Commands the UI may invoke; each one gets an `allow-<command>` permission.-const COMMANDS: &[&str] = &["connection_info", "toggle_panel", "hide_panel", "set_dot_state"];+const COMMANDS: &[&str] = &[+ "connection_info",+ "toggle_panel",+ "hide_panel",+ "set_dot_state",+ "computer_endpoint",+ "computer_permissions",+ "computer_request_permission",+ "computer_stop",+ "computer_status",+];fn find_in_path(program: &str, path: &str) -> Option<PathBuf> {env::split_paths(path)diff --git a/apps/desktop/src-tauri/capabilities/default.json b/apps/desktop/src-tauri/capabilities/default.jsonindex d3d5053..4f141f3 100644--- a/apps/desktop/src-tauri/capabilities/default.json+++ b/apps/desktop/src-tauri/capabilities/default.json@@ -1,7 +1,7 @@{"$schema": "../gen/schemas/desktop-schema.json","identifier": "default",- "description": "What the Dot and panel UI may call: event listening, Dot dragging, opening web pages (sign-in), and the four shell commands.",+ "description": "What the Dot and panel UI may call: event listening, Dot dragging, opening web pages (sign-in), the four shell commands, and the computer-control commands.","windows": ["dot", "panel"],"permissions": ["core:event:allow-listen",@@ -11,6 +11,11 @@"allow-connection-info","allow-toggle-panel","allow-hide-panel",- "allow-set-dot-state"+ "allow-set-dot-state",+ "allow-computer-endpoint",+ "allow-computer-permissions",+ "allow-computer-request-permission",+ "allow-computer-stop",+ "allow-computer-status"]}diff --git a/apps/desktop/src-tauri/permissions/autogenerated/computer_endpoint.toml b/apps/desktop/src-tauri/permissions/autogenerated/computer_endpoint.tomlnew file mode 100644index 0000000..e88dcd1--- /dev/null+++ b/apps/desktop/src-tauri/permissions/autogenerated/computer_endpoint.toml@@ -0,0 +1,11 @@+# Automatically generated - DO NOT EDIT!++[[permission]]+identifier = "allow-computer-endpoint"+description = "Enables the computer_endpoint command without any pre-configured scope."+commands.allow = ["computer_endpoint"]++[[permission]]+identifier = "deny-computer-endpoint"+description = "Denies the computer_endpoint command without any pre-configured scope."+commands.deny = ["computer_endpoint"]diff --git a/apps/desktop/src-tauri/permissions/autogenerated/computer_permissions.toml b/apps/desktop/src-tauri/permissions/autogenerated/computer_permissions.tomlnew file mode 100644index 0000000..86d6ba1--- /dev/null+++ b/apps/desktop/src-tauri/permissions/autogenerated/computer_permissions.toml@@ -0,0 +1,11 @@+# Automatically generated - DO NOT EDIT!++[[permission]]+identifier = "allow-computer-permissions"+description = "Enables the computer_permissions command without any pre-configured scope."+commands.allow = ["computer_permissions"]++[[permission]]+identifier = "deny-computer-permissions"+description = "Denies the computer_permissions command without any pre-configured scope."+commands.deny = ["computer_permissions"]diff --git a/apps/desktop/src-tauri/permissions/autogenerated/computer_request_permission.toml b/apps/desktop/src-tauri/permissions/autogenerated/computer_request_permission.tomlnew file mode 100644index 0000000..6811c27--- /dev/null+++ b/apps/desktop/src-tauri/permissions/autogenerated/computer_request_permission.toml@@ -0,0 +1,11 @@+# Automatically generated - DO NOT EDIT!++[[permission]]+identifier = "allow-computer-request-permission"+description = "Enables the computer_request_permission command without any pre-configured scope."+commands.allow = ["computer_request_permission"]++[[permission]]+identifier = "deny-computer-request-permission"+description = "Denies the computer_request_permission command without any pre-configured scope."+commands.deny = ["computer_request_permission"]diff --git a/apps/desktop/src-tauri/permissions/autogenerated/computer_status.toml b/apps/desktop/src-tauri/permissions/autogenerated/computer_status.tomlnew file mode 100644index 0000000..83f2379--- /dev/null+++ b/apps/desktop/src-tauri/permissions/autogenerated/computer_status.toml@@ -0,0 +1,11 @@+# Automatically generated - DO NOT EDIT!++[[permission]]+identifier = "allow-computer-status"+description = "Enables the computer_status command without any pre-configured scope."+commands.allow = ["computer_status"]++[[permission]]+identifier = "deny-computer-status"+description = "Denies the computer_status command without any pre-configured scope."+commands.deny = ["computer_status"]diff --git a/apps/desktop/src-tauri/permissions/autogenerated/computer_stop.toml b/apps/desktop/src-tauri/permissions/autogenerated/computer_stop.tomlnew file mode 100644index 0000000..2bcbf41--- /dev/null+++ b/apps/desktop/src-tauri/permissions/autogenerated/computer_stop.toml@@ -0,0 +1,11 @@+# Automatically generated - DO NOT EDIT!++[[permission]]+identifier = "allow-computer-stop"+description = "Enables the computer_stop command without any pre-configured scope."+commands.allow = ["computer_stop"]++[[permission]]+identifier = "deny-computer-stop"+description = "Denies the computer_stop command without any pre-configured scope."+commands.deny = ["computer_stop"]diff --git a/apps/desktop/src-tauri/src/computer/app.rs b/apps/desktop/src-tauri/src/computer/app.rsnew file mode 100644index 0000000..c7855cc--- /dev/null+++ b/apps/desktop/src-tauri/src/computer/app.rs@@ -0,0 +1,182 @@+//! The app side of computer control (L60): the `computer_*` commands, the `computer://state`+//! event, the panic shortcut, the timeout watcher, and the native dialogs. Off macOS the+//! commands answer "unavailable" and nothing starts.++use super::control::Control;+use super::server::Endpoint;+use super::session::{Reason, StateEvent};+use super::{Clock, PermissionKind, Permissions, SystemClock};+use std::sync::Arc;+use tauri::{AppHandle, Emitter, Manager, State};++pub const STATE_EVENT: &str = "computer://state";+/// ⌥⇧Esc ends the session from anywhere (S24.3).+pub const PANIC_SHORTCUT: &str = "Alt+Shift+Escape";+pub const GRANT_TEXT: &str = "Allow Gentle Dot to see and control this Mac for 30 minutes?";+pub const GRANT_BUTTONS: (&str, &str) = ("Allow", "Deny");+pub const CONFIRM_BUTTONS: (&str, &str) = ("Allow", "Cancel");++pub struct Computer {+ control: Arc<Control>,+ endpoint: Option<Endpoint>,+ #[cfg(target_os = "macos")]+ desktop: Arc<dyn super::Desktop>,+}++/// Creates the session and, on macOS, starts the MCP helper and the timeout watcher.+pub fn start(app: &AppHandle) -> Computer {+ let events = app.clone();+ let control = Arc::new(Control::new(move |event: &StateEvent| {+ if let Err(error) = events.emit(STATE_EVENT, event) {+ eprintln!("gentle-dot: cannot report the computer-control state: {error}");+ }+ }));+ #[cfg(target_os = "macos")]+ {+ let desktop: Arc<dyn super::Desktop> = Arc::new(super::MacDesktop::default());+ let endpoint = start_helper(app, control.clone(), desktop.clone())+ .map_err(|error| eprintln!("gentle-dot: computer control is unavailable: {error}"))+ .ok();+ watch_timeout(control.clone());+ Computer { control, endpoint, desktop }+ }+ #[cfg(not(target_os = "macos"))]+ Computer { control, endpoint: None }+}++#[cfg(target_os = "macos")]+fn start_helper(+ app: &AppHandle,+ control: Arc<Control>,+ desktop: Arc<dyn super::Desktop>,+) -> std::io::Result<Endpoint> {+ let token = super::server::new_token();+ let helper = super::mcp::Helper::new(+ desktop,+ Arc::new(NativeDialogs { app: app.clone() }),+ Arc::new(SystemClock),+ control,+ token.clone(),+ std::process::id() as i32,+ );+ super::server::start(Arc::new(helper), token)+}++/// Reports the timeout when it happens, not only at the next tool call.+#[cfg(target_os = "macos")]+fn watch_timeout(control: Arc<Control>) {+ std::thread::spawn(move || loop {+ std::thread::sleep(std::time::Duration::from_secs(1));+ control.expire(SystemClock.now_ms());+ });+}++/// Ends the session (Stop button, tray item, panic shortcut). Queued actions are dropped.+pub fn stop(app: &AppHandle, reason: Reason) {+ if let Some(computer) = app.try_state::<Computer>() {+ computer.control.end(reason);+ }+}++#[tauri::command]+pub fn computer_endpoint(computer: State<'_, Computer>) -> Option<Endpoint> {+ computer.endpoint.clone()+}++#[tauri::command]+pub fn computer_permissions(computer: State<'_, Computer>) -> Permissions {+ #[cfg(target_os = "macos")]+ return computer.desktop.permissions();+ #[cfg(not(target_os = "macos"))]+ {+ let _ = computer;+ Permissions { accessibility: false, screen_recording: false, screenshots_supported: false }+ }+}++#[tauri::command]+pub fn computer_request_permission(kind: PermissionKind) -> Result<(), String> {+ #[cfg(target_os = "macos")]+ return super::request_permission(kind);+ #[cfg(not(target_os = "macos"))]+ {+ let _ = kind;+ Err("Computer control is only available on macOS.".into())+ }+}++#[tauri::command]+pub fn computer_stop(app: AppHandle) {+ stop(&app, Reason::Stopped);+}++#[tauri::command]+pub fn computer_status(computer: State<'_, Computer>) -> StateEvent {+ computer.control.status(SystemClock.now_ms())+}++/// Dialogs shown by the app itself (NSAlert through the dialog plugin), never the webview.+#[cfg(target_os = "macos")]+struct NativeDialogs {+ app: AppHandle,+}++#[cfg(target_os = "macos")]+impl NativeDialogs {+ fn ask(&self, message: &str, (allow, refuse): (&str, &str)) -> bool {+ use tauri_plugin_dialog::{DialogExt, MessageDialogButtons, MessageDialogKind};+ self.app+ .dialog()+ .message(message)+ .title("Gentle Dot")+ .kind(MessageDialogKind::Warning)+ .buttons(MessageDialogButtons::OkCancelCustom(allow.into(), refuse.into()))+ .blocking_show()+ }+}++#[cfg(target_os = "macos")]+impl super::Dialogs for NativeDialogs {+ fn ask_grant(&self) -> bool {+ self.ask(GRANT_TEXT, GRANT_BUTTONS)+ }++ fn confirm(&self, message: &str) -> bool {+ self.ask(message, CONFIRM_BUTTONS)+ }+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn the_grant_dialog_asks_exactly_what_the_spec_says() {+ assert_eq!(GRANT_TEXT, "Allow Gentle Dot to see and control this Mac for 30 minutes?");+ assert_eq!(GRANT_BUTTONS, ("Allow", "Deny"));+ }++ #[test]+ fn the_panic_shortcut_is_option_shift_escape() {+ let shortcut: tauri_plugin_global_shortcut::Shortcut = PANIC_SHORTCUT.parse().unwrap();+ let expected = tauri_plugin_global_shortcut::Shortcut::new(+ Some(tauri_plugin_global_shortcut::Modifiers::ALT | tauri_plugin_global_shortcut::Modifiers::SHIFT),+ tauri_plugin_global_shortcut::Code::Escape,+ );+ assert_eq!(shortcut, expected);+ }++ #[test]+ fn permission_kinds_read_the_l60_names() {+ let kind: PermissionKind = serde_json::from_str("\"screenRecording\"").unwrap();+ assert_eq!(kind, PermissionKind::ScreenRecording);+ let kind: PermissionKind = serde_json::from_str("\"accessibility\"").unwrap();+ assert_eq!(kind, PermissionKind::Accessibility);+ }++ #[test]+ fn permissions_serialize_with_the_l60_field_names() {+ let value = serde_json::to_value(Permissions { accessibility: true, screen_recording: false, screenshots_supported: true }).unwrap();+ assert_eq!(value, serde_json::json!({"accessibility": true, "screenRecording": false, "screenshotsSupported": true}));+ }+}diff --git a/apps/desktop/src-tauri/src/computer/blocklist.rs b/apps/desktop/src-tauri/src/computer/blocklist.rsnew file mode 100644index 0000000..2c64e94--- /dev/null+++ b/apps/desktop/src-tauri/src/computer/blocklist.rs@@ -0,0 +1,153 @@+//! Apps no action may target (S24.4): Gentle Dot itself, System Settings, the login and+//! authorization prompts, Keychain Access, and the known password managers.++/// The app an action targets: the frontmost app for keyboard actions, the owner of the+/// window under the point for pointer actions.+#[derive(Debug, Clone, PartialEq, Eq)]+pub struct AppIdentity {+ pub pid: i32,+ pub bundle_id: Option<String>,+ pub name: String,+}++struct Entry {+ label: &'static str,+ /// A bundle id also blocks its helpers (`com.apple.Passwords` blocks `com.apple.Passwords.MenuBarExtra`).+ bundles: &'static [&'static str],+ names: &'static [&'static str],+}++pub const GENTLE_DOT_BUNDLE: &str = "dev.gentleman.gentle-dot";++const BLOCKED: &[Entry] = &[+ Entry { label: "Gentle Dot", bundles: &[GENTLE_DOT_BUNDLE], names: &["Gentle Dot"] },+ Entry {+ label: "System Settings",+ bundles: &["com.apple.systempreferences"],+ names: &["System Settings", "System Preferences"],+ },+ Entry { label: "SecurityAgent", bundles: &["com.apple.SecurityAgent"], names: &["SecurityAgent"] },+ Entry { label: "loginwindow", bundles: &["com.apple.loginwindow"], names: &["loginwindow"] },+ Entry { label: "Keychain Access", bundles: &["com.apple.keychainaccess"], names: &["Keychain Access"] },+ Entry {+ label: "1Password",+ bundles: &["com.1password", "com.agilebits"],+ names: &["1Password", "1Password 7", "1Password 8"],+ },+ Entry { label: "Bitwarden", bundles: &["com.bitwarden"], names: &["Bitwarden"] },+ Entry { label: "Dashlane", bundles: &["com.dashlane"], names: &["Dashlane"] },+ Entry { label: "LastPass", bundles: &["com.lastpass"], names: &["LastPass"] },+ Entry { label: "Passwords", bundles: &["com.apple.Passwords"], names: &["Passwords"] },+];++fn bundle_matches(bundle: &str, blocked: &str) -> bool {+ let (bundle, blocked) = (bundle.to_lowercase(), blocked.to_lowercase());+ bundle == blocked || bundle.strip_prefix(&blocked).is_some_and(|rest| rest.starts_with('.'))+}++/// Why `app` may not be targeted, or `None` when it may. `own_pid` catches Gentle Dot+/// when it runs unbundled (a development build has no bundle id).+pub fn blocked_reason(app: &AppIdentity, own_pid: i32) -> Option<&'static str> {+ if app.pid == own_pid {+ return Some(BLOCKED[0].label);+ }+ match &app.bundle_id {+ Some(bundle) => BLOCKED.iter().find(|e| e.bundles.iter().any(|b| bundle_matches(bundle, b))).map(|e| e.label),+ None => blocked_name(&app.name),+ }+}++/// The same check for `open_app`, which only has a name or a bundle id (`Keychain Access.app` too).+pub fn blocked_name(name_or_bundle: &str) -> Option<&'static str> {+ let name = name_or_bundle.trim().trim_end_matches('/');+ let name = name.rsplit('/').next().unwrap_or(name);+ let name = name.strip_suffix(".app").unwrap_or(name);+ BLOCKED+ .iter()+ .find(|e| {+ e.names.iter().any(|n| n.eq_ignore_ascii_case(name)) || e.bundles.iter().any(|b| bundle_matches(name, b))+ })+ .map(|e| e.label)+}++#[cfg(test)]+mod tests {+ use super::*;++ const OWN_PID: i32 = 4242;++ fn app(bundle: Option<&str>, name: &str) -> AppIdentity {+ AppIdentity { pid: 100, bundle_id: bundle.map(str::to_string), name: name.into() }+ }++ #[test]+ fn every_listed_bundle_is_blocked() {+ let cases = [+ ("com.apple.systempreferences", "System Settings"),+ ("com.apple.SecurityAgent", "SecurityAgent"),+ ("com.apple.loginwindow", "loginwindow"),+ ("com.apple.keychainaccess", "Keychain Access"),+ ("com.1password.1password", "1Password"),+ ("com.agilebits.onepassword7", "1Password"),+ ("com.bitwarden.desktop", "Bitwarden"),+ ("com.dashlane.dashlanephonefinal", "Dashlane"),+ ("com.lastpass.LastPass", "LastPass"),+ ("com.apple.Passwords", "Passwords"),+ ("dev.gentleman.gentle-dot", "Gentle Dot"),+ ];+ for (bundle, label) in cases {+ assert_eq!(blocked_reason(&app(Some(bundle), "x"), OWN_PID), Some(label), "{bundle}");+ }+ }++ #[test]+ fn helpers_of_a_blocked_bundle_are_blocked() {+ assert_eq!(blocked_reason(&app(Some("com.apple.Passwords.MenuBarExtra"), "x"), OWN_PID), Some("Passwords"));+ }++ #[test]+ fn bundle_ids_compare_case_insensitively() {+ assert_eq!(blocked_reason(&app(Some("COM.APPLE.KEYCHAINACCESS"), "x"), OWN_PID), Some("Keychain Access"));+ }++ #[test]+ fn similar_prefixes_are_not_blocked() {+ assert_eq!(blocked_reason(&app(Some("com.apple.PasswordsX"), "x"), OWN_PID), None);+ assert_eq!(blocked_reason(&app(Some("com.lastpassword.app"), "x"), OWN_PID), None);+ }++ #[test]+ fn ordinary_apps_are_allowed() {+ assert_eq!(blocked_reason(&app(Some("com.apple.Safari"), "Safari"), OWN_PID), None);+ assert_eq!(blocked_reason(&app(None, "Some Tool"), OWN_PID), None);+ }++ #[test]+ fn gentle_dot_is_blocked_by_its_own_pid() {+ let unbundled = AppIdentity { pid: OWN_PID, bundle_id: None, name: "gentle-dot".into() };+ assert_eq!(blocked_reason(&unbundled, OWN_PID), Some("Gentle Dot"));+ }++ #[test]+ fn an_app_without_a_bundle_id_is_checked_by_name() {+ assert_eq!(blocked_reason(&app(None, "SecurityAgent"), OWN_PID), Some("SecurityAgent"));+ }++ #[test]+ fn open_app_names_and_bundle_ids_are_checked() {+ assert_eq!(blocked_name("System Settings"), Some("System Settings"));+ assert_eq!(blocked_name("system preferences"), Some("System Settings"));+ assert_eq!(blocked_name("Keychain Access.app"), Some("Keychain Access"));+ assert_eq!(blocked_name("com.bitwarden.desktop"), Some("Bitwarden"));+ assert_eq!(blocked_name(" 1password "), Some("1Password"));+ assert_eq!(blocked_name("Safari"), None);+ assert_eq!(blocked_name("com.apple.Safari"), None);+ }++ #[test]+ fn open_app_paths_are_checked_by_their_app_name() {+ assert_eq!(blocked_name("/System/Applications/System Settings.app"), Some("System Settings"));+ assert_eq!(blocked_name("/Applications/1Password.app/"), Some("1Password"));+ assert_eq!(blocked_name("/Applications/Safari.app"), None);+ }+}diff --git a/apps/desktop/src-tauri/src/computer/control.rs b/apps/desktop/src-tauri/src/computer/control.rsnew file mode 100644index 0000000..d0ede33--- /dev/null+++ b/apps/desktop/src-tauri/src/computer/control.rs@@ -0,0 +1,55 @@+//! The session shared by the MCP server and the app (Stop button, tray item, panic shortcut,+//! timeout watcher). Every transition is reported to `notify` (the `computer://state` event).++use super::session::{Reason, Session, StateEvent};+use std::sync::Mutex;++type Notify = Box<dyn Fn(&StateEvent) + Send + Sync>;++pub struct Control {+ session: Mutex<Session>,+ notify: Notify,+}++impl Control {+ pub fn new(notify: impl Fn(&StateEvent) + Send + Sync + 'static) -> Self {+ Control { session: Mutex::new(Session::default()), notify: Box::new(notify) }+ }++ fn with<T>(&self, f: impl FnOnce(&mut Session) -> T) -> T {+ f(&mut self.session.lock().unwrap_or_else(|poisoned| poisoned.into_inner()))+ }++ fn report(&self, event: Option<StateEvent>) {+ if let Some(event) = event {+ (self.notify)(&event);+ }+ }++ pub fn status(&self, now_ms: u64) -> StateEvent {+ self.with(|s| s.status(now_ms))+ }++ pub fn is_active(&self, now_ms: u64) -> bool {+ self.with(|s| s.is_active(now_ms))+ }++ pub fn epoch(&self) -> u64 {+ self.with(|s| s.epoch())+ }++ pub fn grant(&self, now_ms: u64) {+ let event = self.with(|s| s.grant(now_ms));+ self.report(Some(event));+ }++ pub fn end(&self, reason: Reason) {+ let event = self.with(|s| s.end(reason));+ self.report(event);+ }++ pub fn expire(&self, now_ms: u64) {+ let event = self.with(|s| s.expire(now_ms));+ self.report(event);+ }+}diff --git a/apps/desktop/src-tauri/src/computer/coords.rs b/apps/desktop/src-tauri/src/computer/coords.rsnew file mode 100644index 0000000..dd51581--- /dev/null+++ b/apps/desktop/src-tauri/src/computer/coords.rs@@ -0,0 +1,137 @@+//! Screenshot geometry (S24.2): the size a capture is downscaled to, and how a pixel of the+//! latest screenshot maps back to a global point (the unit macOS input events use).++/// Long edge of every screenshot, in pixels.+pub const MAX_LONG_EDGE: u32 = 1280;++#[derive(Debug, Clone, Copy, PartialEq)]+pub struct Point {+ pub x: f64,+ pub y: f64,+}++/// The size that fits `width × height` within a long edge of `max`, keeping the aspect+/// ratio. Never upscales and never returns a zero side.+pub fn fit_long_edge(width: u32, height: u32, max: u32) -> (u32, u32) {+ let long = width.max(height).max(1);+ if long <= max {+ return (width.max(1), height.max(1));+ }+ let fit = |side: u32| ((u64::from(side) * u64::from(max) + u64::from(long) / 2) / u64::from(long)).max(1) as u32;+ (fit(width), fit(height))+}++/// Where a screenshot came from: its size in pixels, the display's top-left corner in+/// global points, and `scale`, the screenshot pixels per point.+#[derive(Debug, Clone, Copy, PartialEq)]+pub struct ScreenshotGeometry {+ pub image_width: u32,+ pub image_height: u32,+ pub scale: f64,+ pub origin: Point,+}++#[derive(Debug, Clone, PartialEq)]+pub enum CoordError {+ NotFinite,+ OutOfBounds { width: u32, height: u32 },+}++impl std::fmt::Display for CoordError {+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {+ match self {+ CoordError::NotFinite => write!(f, "coordinates must be numbers"),+ CoordError::OutOfBounds { width, height } => {+ write!(f, "coordinates are outside the latest screenshot ({width} × {height} pixels)")+ }+ }+ }+}++impl ScreenshotGeometry {+ /// The geometry of a display `width_pt` points wide captured at `image_width × image_height`.+ pub fn for_display(origin: Point, width_pt: f64, image_width: u32, image_height: u32) -> Self {+ ScreenshotGeometry { image_width, image_height, scale: f64::from(image_width) / width_pt, origin }+ }++ /// Maps a screenshot pixel to a global point, rounded to a whole point.+ pub fn to_point(&self, x: f64, y: f64) -> Result<Point, CoordError> {+ if !x.is_finite() || !y.is_finite() {+ return Err(CoordError::NotFinite);+ }+ let (width, height) = (self.image_width, self.image_height);+ if x < 0.0 || y < 0.0 || x >= f64::from(width) || y >= f64::from(height) {+ return Err(CoordError::OutOfBounds { width, height });+ }+ Ok(Point { x: (self.origin.x + x / self.scale).round(), y: (self.origin.y + y / self.scale).round() })+ }+}++#[cfg(test)]+mod tests {+ use super::*;++ fn retina_laptop() -> ScreenshotGeometry {+ // A 1512 × 982 pt display captured at 1280 × 831 px.+ ScreenshotGeometry::for_display(Point { x: 0.0, y: 0.0 }, 1512.0, 1280, 831)+ }++ #[test]+ fn landscape_and_portrait_fit_the_long_edge() {+ assert_eq!(fit_long_edge(3024, 1964, 1280), (1280, 831));+ assert_eq!(fit_long_edge(1080, 1920, 1280), (720, 1280));+ }++ #[test]+ fn small_captures_are_never_upscaled() {+ assert_eq!(fit_long_edge(1024, 768, 1280), (1024, 768));+ assert_eq!(fit_long_edge(1280, 1280, 1280), (1280, 1280));+ }++ #[test]+ fn extreme_ratios_keep_at_least_one_pixel() {+ assert_eq!(fit_long_edge(100_000, 10, 1280), (1280, 1));+ assert_eq!(fit_long_edge(0, 0, 1280), (1, 1));+ }++ #[test]+ fn scale_is_screenshot_pixels_per_point() {+ let geometry = retina_laptop();+ assert!((geometry.scale - 1280.0 / 1512.0).abs() < 1e-9);+ assert_eq!((geometry.image_width, geometry.image_height), (1280, 831));+ }++ #[test]+ fn pixels_map_to_points_with_the_scale() {+ let geometry = ScreenshotGeometry::for_display(Point { x: 0.0, y: 0.0 }, 2560.0, 1280, 720);+ assert_eq!(geometry.to_point(640.0, 360.0), Ok(Point { x: 1280.0, y: 720.0 }));+ assert_eq!(geometry.to_point(0.0, 0.0), Ok(Point { x: 0.0, y: 0.0 }));+ }++ #[test]+ fn the_display_origin_is_added() {+ // A second display left of and above the main one.+ let geometry = ScreenshotGeometry::for_display(Point { x: -1920.0, y: -300.0 }, 1920.0, 1280, 720);+ assert_eq!(geometry.to_point(640.0, 360.0), Ok(Point { x: -960.0, y: 240.0 }));+ }++ #[test]+ fn points_are_rounded_to_whole_points() {+ // 100 px / (1280/1512) = 118.125 pt; 101 px = 119.30625 pt; 1279 px = 1510.81875 pt.+ let geometry = retina_laptop();+ assert_eq!(geometry.to_point(100.0, 101.0), Ok(Point { x: 118.0, y: 119.0 }));+ assert_eq!(geometry.to_point(1279.0, 0.5), Ok(Point { x: 1511.0, y: 1.0 }));+ }++ #[test]+ fn pixels_outside_the_screenshot_are_rejected() {+ let geometry = retina_laptop();+ let outside = Err(CoordError::OutOfBounds { width: 1280, height: 831 });+ assert_eq!(geometry.to_point(1280.0, 10.0), outside);+ assert_eq!(geometry.to_point(10.0, 831.0), outside);+ assert_eq!(geometry.to_point(-1.0, 10.0), outside);+ assert_eq!(geometry.to_point(10.0, -0.5), outside);+ assert_eq!(geometry.to_point(f64::NAN, 10.0), Err(CoordError::NotFinite));+ assert_eq!(geometry.to_point(10.0, f64::INFINITY), Err(CoordError::NotFinite));+ }+}diff --git a/apps/desktop/src-tauri/src/computer/encode.rs b/apps/desktop/src-tauri/src/computer/encode.rsnew file mode 100644index 0000000..3e22061--- /dev/null+++ b/apps/desktop/src-tauri/src/computer/encode.rs@@ -0,0 +1,66 @@+//! Screenshot encoding: ScreenCaptureKit hands back BGRA rows; the model gets a JPEG.++pub const JPEG_QUALITY: u8 = 80;++/// Encodes a BGRA bitmap whose rows are `bytes_per_row` long (rows may be padded) as JPEG.+pub fn bgra_to_jpeg(+ bgra: &[u8],+ width: u32,+ height: u32,+ bytes_per_row: usize,+ quality: u8,+) -> Result<Vec<u8>, String> {+ let (w, h) = (width as usize, height as usize);+ if w == 0 || h == 0 || bytes_per_row < w * 4 || bgra.len() < bytes_per_row * (h - 1) + w * 4 {+ let size = bgra.len();+ return Err(format!("unexpected bitmap: {width} × {height}, {bytes_per_row} bytes per row, {size} bytes"));+ }+ let mut rgb = Vec::with_capacity(w * h * 3);+ for row in bgra.chunks(bytes_per_row).take(h) {+ for i in (0..w * 4).step_by(4) {+ rgb.extend_from_slice(&[row[i + 2], row[i + 1], row[i]]);+ }+ }+ let mut jpeg = Vec::new();+ image::codecs::jpeg::JpegEncoder::new_with_quality(&mut jpeg, quality)+ .encode(&rgb, width, height, image::ExtendedColorType::Rgb8)+ .map_err(|e| format!("cannot encode the screenshot: {e}"))?;+ Ok(jpeg)+}++#[cfg(test)]+mod tests {+ use super::*;++ fn decode(jpeg: &[u8]) -> image::RgbImage {+ image::load_from_memory_with_format(jpeg, image::ImageFormat::Jpeg).unwrap().to_rgb8()+ }++ #[test]+ fn bgra_becomes_a_jpeg_of_the_same_size_and_colors() {+ // 8 × 8 pure red in BGRA.+ let bgra: Vec<u8> = [0u8, 0, 255, 255].repeat(64);+ let jpeg = bgra_to_jpeg(&bgra, 8, 8, 32, 90).unwrap();+ assert_eq!(&jpeg[..2], &[0xFF, 0xD8]);+ let image = decode(&jpeg);+ assert_eq!(image.dimensions(), (8, 8));+ let [r, g, b] = image.get_pixel(4, 4).0;+ assert!(r > 240 && g < 20 && b < 20, "{r} {g} {b}");+ }++ #[test]+ fn padded_rows_are_skipped() {+ // 8 × 8 blue, each row padded with 16 bytes of white.+ let row: Vec<u8> = [[255u8, 0, 0, 255].repeat(8), vec![255; 16]].concat();+ let jpeg = bgra_to_jpeg(&row.repeat(8), 8, 8, 48, 90).unwrap();+ let [r, g, b] = decode(&jpeg).get_pixel(7, 7).0;+ assert!(b > 240 && r < 20 && g < 20, "{r} {g} {b}");+ }++ #[test]+ fn short_or_empty_bitmaps_are_rejected() {+ assert!(bgra_to_jpeg(&[0; 10], 2, 2, 8, 80).is_err());+ assert!(bgra_to_jpeg(&[], 0, 0, 0, 80).is_err());+ assert!(bgra_to_jpeg(&[0; 64], 4, 4, 8, 80).is_err());+ }+}diff --git a/apps/desktop/src-tauri/src/computer/fake.rs b/apps/desktop/src-tauri/src/computer/fake.rsnew file mode 100644index 0000000..5e887ff--- /dev/null+++ b/apps/desktop/src-tauri/src/computer/fake.rs@@ -0,0 +1,174 @@+//! Test doubles: a Mac that records events, scripted dialogs, and a clock that sleeps instantly.++use super::blocklist::AppIdentity;+use super::coords::{Point, ScreenshotGeometry};+use super::risk::{AxElement, Focus};+use super::{AppInfo, Capture, Clock, Desktop, Dialogs, InputEvent, Permissions};+use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering};+use std::sync::Mutex;++pub fn app(pid: i32, bundle: &str, name: &str) -> AppIdentity {+ AppIdentity { pid, bundle_id: Some(bundle.into()), name: name.into() }+}++type OwnerAt = Box<dyn Fn(Point) -> Option<AppIdentity> + Send + Sync>;++pub struct FakeDesktop {+ pub events: Mutex<Vec<InputEvent>>,+ pub permissions: Mutex<Permissions>,+ pub frontmost: Mutex<Option<AppIdentity>>,+ pub owner_at: Mutex<OwnerAt>,+ pub elements: Mutex<Vec<AxElement>>,+ pub focus: Mutex<Option<Focus>>,+ pub opened: Mutex<Vec<String>>,+ pub activated: Mutex<Vec<i32>>,+ pub captures: AtomicUsize,+ /// A 2560 × 1600 pt display at the origin, captured at 1280 × 800 (scale 0.5).+ pub geometry: ScreenshotGeometry,+}++impl Default for FakeDesktop {+ fn default() -> Self {+ let safari = app(200, "com.apple.Safari", "Safari");+ let owner = safari.clone();+ FakeDesktop {+ events: Mutex::new(Vec::new()),+ permissions: Mutex::new(Permissions { accessibility: true, screen_recording: true, screenshots_supported: true }),+ frontmost: Mutex::new(Some(safari)),+ owner_at: Mutex::new(Box::new(move |_| Some(owner.clone()))),+ elements: Mutex::new(Vec::new()),+ focus: Mutex::new(None),+ opened: Mutex::new(Vec::new()),+ activated: Mutex::new(Vec::new()),+ captures: AtomicUsize::new(0),+ geometry: ScreenshotGeometry::for_display(Point { x: 0.0, y: 0.0 }, 2560.0, 1280, 800),+ }+ }+}++impl FakeDesktop {+ pub fn events(&self) -> Vec<InputEvent> {+ self.events.lock().unwrap().clone()+ }+}++impl Desktop for FakeDesktop {+ fn permissions(&self) -> Permissions {+ *self.permissions.lock().unwrap()+ }++ fn capture(&self) -> Result<Capture, String> {+ self.captures.fetch_add(1, Ordering::SeqCst);+ Ok(Capture { data: b"jpeg".to_vec(), mime_type: "image/jpeg", geometry: self.geometry })+ }++ fn post(&self, event: &InputEvent) -> Result<(), String> {+ self.events.lock().unwrap().push(event.clone());+ Ok(())+ }++ fn list_apps(&self) -> Result<Vec<AppInfo>, String> {+ Ok(vec![+ AppInfo { name: "Safari".into(), bundle_id: Some("com.apple.Safari".into()), active: true },+ AppInfo { name: "Mail".into(), bundle_id: Some("com.apple.mail".into()), active: false },+ ])+ }++ fn open_app(&self, name: &str) -> Result<(), String> {+ self.opened.lock().unwrap().push(name.into());+ Ok(())+ }++ fn element_at(&self, _at: Point) -> Vec<AxElement> {+ self.elements.lock().unwrap().clone()+ }++ fn focused_element(&self) -> Option<Focus> {+ self.focus.lock().unwrap().clone()+ }++ fn frontmost_app(&self) -> Option<AppIdentity> {+ self.frontmost.lock().unwrap().clone()+ }++ fn window_owner_at(&self, at: Point) -> Option<AppIdentity> {+ (self.owner_at.lock().unwrap())(at)+ }++ fn activate(&self, pid: i32) {+ self.activated.lock().unwrap().push(pid);+ }+}++type Hook = Box<dyn Fn() + Send + Sync>;++/// Answers dialogs with fixed choices and records what was asked.+pub struct FakeDialogs {+ pub allow: Mutex<bool>,+ pub confirm: Mutex<bool>,+ pub grants: AtomicUsize,+ pub confirmations: Mutex<Vec<String>>,+ /// Runs while a dialog is open (for example, to move the focus to Gentle Dot).+ pub during: Mutex<Option<Hook>>,+}++impl FakeDialogs {+ pub fn new(allow: bool, confirm: bool) -> Self {+ FakeDialogs {+ allow: Mutex::new(allow),+ confirm: Mutex::new(confirm),+ grants: AtomicUsize::new(0),+ confirmations: Mutex::new(Vec::new()),+ during: Mutex::new(None),+ }+ }++ fn run_hook(&self) {+ if let Some(hook) = self.during.lock().unwrap().as_ref() {+ hook();+ }+ }+}++impl Dialogs for FakeDialogs {+ fn ask_grant(&self) -> bool {+ self.grants.fetch_add(1, Ordering::SeqCst);+ self.run_hook();+ *self.allow.lock().unwrap()+ }++ fn confirm(&self, message: &str) -> bool {+ self.confirmations.lock().unwrap().push(message.into());+ self.run_hook();+ *self.confirm.lock().unwrap()+ }+}++/// Starts at a fixed time; `sleep_ms` advances it and runs `on_sleep`.+pub struct FakeClock {+ pub now: AtomicU64,+ pub on_sleep: Mutex<Option<Hook>>,+}++impl FakeClock {+ pub fn new(now: u64) -> Self {+ FakeClock { now: AtomicU64::new(now), on_sleep: Mutex::new(None) }+ }++ pub fn advance(&self, ms: u64) {+ self.now.fetch_add(ms, Ordering::SeqCst);+ }+}++impl Clock for FakeClock {+ fn now_ms(&self) -> u64 {+ self.now.load(Ordering::SeqCst)+ }++ fn sleep_ms(&self, ms: u64) {+ self.advance(ms);+ if let Some(hook) = self.on_sleep.lock().unwrap().as_ref() {+ hook();+ }+ }+}diff --git a/apps/desktop/src-tauri/src/computer/keys.rs b/apps/desktop/src-tauri/src/computer/keys.rsnew file mode 100644index 0000000..8792e98--- /dev/null+++ b/apps/desktop/src-tauri/src/computer/keys.rs@@ -0,0 +1,137 @@+//! Key combos for the `key` tool (S24.2), such as `cmd+shift+t` or `return`, parsed into+//! modifiers and one macOS virtual key code (ANSI layout).++#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]+pub struct Modifiers {+ pub cmd: bool,+ pub shift: bool,+ pub alt: bool,+ pub ctrl: bool,+ pub fn_key: bool,+}++#[derive(Debug, Clone, Copy, PartialEq, Eq)]+pub struct KeyCombo {+ pub modifiers: Modifiers,+ /// Canonical key name, for messages and the risk check.+ pub key: &'static str,+ /// macOS virtual key code (`kVK_*`).+ pub code: u16,+}++impl KeyCombo {+ /// Return (or Enter) with any modifiers: the key that submits forms and sends messages.+ pub fn is_return(&self) -> bool {+ self.key == "return"+ }+}++/// Named keys and their virtual key codes. Single characters use the ANSI layout.+const KEYS: &[(&str, u16)] = &[+ ("a", 0), ("s", 1), ("d", 2), ("f", 3), ("h", 4), ("g", 5), ("z", 6), ("x", 7), ("c", 8), ("v", 9),+ ("b", 11), ("q", 12), ("w", 13), ("e", 14), ("r", 15), ("y", 16), ("t", 17), ("1", 18), ("2", 19),+ ("3", 20), ("4", 21), ("6", 22), ("5", 23), ("=", 24), ("9", 25), ("7", 26), ("-", 27), ("8", 28),+ ("0", 29), ("]", 30), ("o", 31), ("u", 32), ("[", 33), ("i", 34), ("p", 35), ("return", 36), ("l", 37),+ ("j", 38), ("'", 39), ("k", 40), (";", 41), ("\\", 42), (",", 43), ("/", 44), ("n", 45), ("m", 46),+ (".", 47), ("tab", 48), ("space", 49), ("`", 50), ("delete", 51), ("escape", 53), ("f5", 96), ("f6", 97),+ ("f7", 98), ("f3", 99), ("f8", 100), ("f9", 101), ("f11", 103), ("f10", 109), ("f12", 111), ("home", 115),+ ("pageup", 116), ("forwarddelete", 117), ("f4", 118), ("end", 119), ("f2", 120), ("pagedown", 121),+ ("f1", 122), ("left", 123), ("right", 124), ("down", 125), ("up", 126),+];++/// Other spellings of the named keys.+const ALIASES: &[(&str, &str)] = &[+ ("enter", "return"), ("esc", "escape"), ("backspace", "delete"), ("del", "forwarddelete"),+ ("page_up", "pageup"), ("page_down", "pagedown"), ("arrowleft", "left"), ("arrowright", "right"),+ ("arrowup", "up"), ("arrowdown", "down"), ("minus", "-"), ("equal", "="),+ ("comma", ","), ("period", "."), ("slash", "/"), ("backslash", "\\"),+];++/// Parses `mod+mod+key`: case-insensitive, parts joined by `+`, any number of modifiers+/// (`cmd`, `shift`, `alt`/`option`, `ctrl`/`control`, `fn`) and exactly one key.+pub fn parse_combo(combo: &str) -> Result<KeyCombo, String> {+ let mut modifiers = Modifiers::default();+ let mut key = None;+ for part in combo.split('+') {+ let part = part.trim().to_lowercase();+ let flag = match part.as_str() {+ "cmd" | "command" => &mut modifiers.cmd,+ "shift" => &mut modifiers.shift,+ "alt" | "option" | "opt" => &mut modifiers.alt,+ "ctrl" | "control" => &mut modifiers.ctrl,+ "fn" => &mut modifiers.fn_key,+ _ => {+ if key.is_some() {+ return Err(format!("`{combo}` names more than one key"));+ }+ let name = ALIASES.iter().find(|(alias, _)| *alias == part).map_or(part.as_str(), |(_, name)| name);+ let found = KEYS.iter().find(|(known, _)| *known == name);+ key = Some(*found.ok_or_else(|| format!("unknown key `{part}` in `{combo}`"))?);+ continue;+ }+ };+ *flag = true;+ }+ let (key, code) = key.ok_or_else(|| format!("`{combo}` names no key"))?;+ Ok(KeyCombo { modifiers, key, code })+}++#[cfg(test)]+mod tests {+ use super::*;++ fn mods(cmd: bool, shift: bool, alt: bool, ctrl: bool) -> Modifiers {+ Modifiers { cmd, shift, alt, ctrl, fn_key: false }+ }++ #[test]+ fn a_plain_named_key() {+ let combo = parse_combo("return").unwrap();+ assert_eq!((combo.key, combo.code, combo.modifiers), ("return", 36, Modifiers::default()));+ }++ #[test]+ fn modifiers_and_a_letter() {+ let combo = parse_combo("cmd+shift+t").unwrap();+ assert_eq!((combo.key, combo.code, combo.modifiers), ("t", 17, mods(true, true, false, false)));+ }++ #[test]+ fn modifier_spellings_and_case_are_accepted() {+ let combo = parse_combo(" Command + Option + Control + A ").unwrap();+ assert_eq!((combo.key, combo.code, combo.modifiers), ("a", 0, mods(true, false, true, true)));+ assert_eq!(parse_combo("opt+ctrl+left").unwrap().modifiers, mods(false, false, true, true));+ assert!(parse_combo("fn+f1").unwrap().modifiers.fn_key);+ }++ #[test]+ fn aliases_name_the_same_key() {+ assert_eq!(parse_combo("enter").unwrap().code, 36);+ assert_eq!(parse_combo("esc").unwrap().code, 53);+ assert_eq!(parse_combo("backspace").unwrap().code, 51);+ assert_eq!(parse_combo("cmd+minus").unwrap().code, 27);+ }++ #[test]+ fn digits_punctuation_and_function_keys() {+ assert_eq!(parse_combo("cmd+1").unwrap().code, 18);+ assert_eq!(parse_combo("cmd+,").unwrap().code, 43);+ assert_eq!(parse_combo("f12").unwrap().code, 111);+ assert_eq!(parse_combo("space").unwrap().code, 49);+ }++ #[test]+ fn invalid_combos_are_rejected() {+ for combo in ["", "cmd+", "cmd", "cmd+shift", "cmd+a+b", "hyper+a", "f13", "cmd++a"] {+ assert!(parse_combo(combo).is_err(), "{combo:?} should be rejected");+ }+ }++ #[test]+ fn return_is_recognized_with_any_modifiers() {+ assert!(parse_combo("return").unwrap().is_return());+ assert!(parse_combo("cmd+return").unwrap().is_return());+ assert!(parse_combo("ctrl+enter").unwrap().is_return());+ assert!(!parse_combo("tab").unwrap().is_return());+ }+}diff --git a/apps/desktop/src-tauri/src/computer/macos.rs b/apps/desktop/src-tauri/src/computer/macos.rsnew file mode 100644index 0000000..698871f--- /dev/null+++ b/apps/desktop/src-tauri/src/computer/macos.rs@@ -0,0 +1,460 @@+//! The real Mac behind `Desktop`: ScreenCaptureKit screenshots (macOS 14+), Quartz events,+//! the Accessibility API, and NSWorkspace. Nothing here decides policy; `mcp` does.++use super::blocklist::AppIdentity;+use super::coords::{fit_long_edge, Point, ScreenshotGeometry, MAX_LONG_EDGE};+use super::encode::{bgra_to_jpeg, JPEG_QUALITY};+use super::keys::KeyCombo;+use super::risk::{AxElement, Focus};+use super::{AppInfo, Capture, Desktop, InputEvent, MouseButton, PermissionKind, Permissions};+use block2::RcBlock;+use objc2::rc::Retained;+use objc2::{available, AnyThread};+use objc2_app_kit::{NSApplicationActivationOptions, NSApplicationActivationPolicy, NSRunningApplication, NSWorkspace};+use objc2_application_services::{AXError, AXIsProcessTrusted, AXIsProcessTrustedWithOptions, AXUIElement};+use objc2_core_foundation::{CFBoolean, CFDictionary, CFRetained, CFString, CFType, CGPoint, CGRect, CGSize};+use objc2_core_graphics::{+ kCGNullWindowID, kCGWindowAlpha, kCGWindowBounds, kCGWindowOwnerName, kCGWindowOwnerPID, CGDataProvider, CGDirectDisplayID, CGError, CGEvent, CGEventField, CGEventFlags,+ CGEventTapLocation, CGEventType, CGGetDisplaysWithPoint, CGImage, CGMouseButton,+ CGPreflightScreenCaptureAccess, CGRequestScreenCaptureAccess, CGScrollEventUnit, CGWindowListCopyWindowInfo,+ CGWindowListOption,+};+use objc2_foundation::{NSArray, NSDictionary, NSError, NSNumber, NSString};+use objc2_screen_capture_kit::{+ SCContentFilter, SCScreenshotManager, SCShareableContent, SCStreamConfiguration, SCWindow,+};+use std::ptr::NonNull;+use std::sync::mpsc;+use std::thread;+use std::time::Duration;++const CAPTURE_TIMEOUT: Duration = Duration::from_secs(5);+/// Pause between the events of one action, so apps see distinct presses.+const EVENT_GAP: Duration = Duration::from_millis(15);+const DRAG_STEPS: u32 = 12;+/// How many ancestors of the element under a point are read (text inside a button).+const AX_ANCESTORS: usize = 4;+/// `kCVPixelFormatType_32BGRA`.+const PIXEL_FORMAT_BGRA: u32 = u32::from_be_bytes(*b"BGRA");++/// Values crossing from a ScreenCaptureKit completion handler to the waiting thread.+/// The objects are immutable snapshots, used by one thread at a time.+struct Handoff<T>(T);+unsafe impl<T> Send for Handoff<T> {}++pub struct MacDesktop {+ own_pid: i32,+}++impl Default for MacDesktop {+ fn default() -> Self {+ MacDesktop { own_pid: std::process::id() as i32 }+ }+}++fn identity(app: &NSRunningApplication) -> AppIdentity {+ AppIdentity {+ pid: app.processIdentifier(),+ bundle_id: app.bundleIdentifier().map(|b| b.to_string()),+ name: app.localizedName().map(|n| n.to_string()).unwrap_or_default(),+ }+}++fn post_event(event: Option<CFRetained<CGEvent>>) -> Result<(), String> {+ let event = event.ok_or("macOS could not create the input event")?;+ CGEvent::post(CGEventTapLocation::HIDEventTap, Some(&event));+ thread::sleep(EVENT_GAP);+ Ok(())+}++fn mouse(kind: CGEventType, at: Point, button: CGMouseButton) -> Option<CFRetained<CGEvent>> {+ CGEvent::new_mouse_event(None, kind, CGPoint { x: at.x, y: at.y }, button)+}++/// The held modifiers of a combo: their key codes (`kVK_Command`, ...) and flags.+fn modifiers(combo: &KeyCombo) -> Vec<(u16, CGEventFlags)> {+ let m = combo.modifiers;+ [+ (m.cmd, 55, CGEventFlags::MaskCommand),+ (m.shift, 56, CGEventFlags::MaskShift),+ (m.alt, 58, CGEventFlags::MaskAlternate),+ (m.ctrl, 59, CGEventFlags::MaskControl),+ (m.fn_key, 63, CGEventFlags::MaskSecondaryFn),+ ]+ .into_iter()+ .filter(|(on, ..)| *on)+ .map(|(_, code, flag)| (code, flag))+ .collect()+}++fn key_event(code: u16, down: bool, flags: CGEventFlags) -> Result<(), String> {+ let event = CGEvent::new_keyboard_event(None, code, down);+ if let Some(event) = &event {+ CGEvent::set_flags(Some(event), flags);+ }+ post_event(event)+}++fn click(at: Point, button: MouseButton, count: u8) -> Result<(), String> {+ let (down, up, cg_button) = match button {+ MouseButton::Left => (CGEventType::LeftMouseDown, CGEventType::LeftMouseUp, CGMouseButton::Left),+ MouseButton::Right => (CGEventType::RightMouseDown, CGEventType::RightMouseUp, CGMouseButton::Right),+ };+ post_event(mouse(CGEventType::MouseMoved, at, CGMouseButton::Left))?;+ for state in 1..=i64::from(count) {+ for kind in [down, up] {+ let event = mouse(kind, at, cg_button);+ if let Some(event) = &event {+ CGEvent::set_integer_value_field(Some(event), CGEventField::MouseEventClickState, state);+ }+ post_event(event)?;+ }+ }+ Ok(())+}++fn drag(from: Point, to: Point) -> Result<(), String> {+ post_event(mouse(CGEventType::MouseMoved, from, CGMouseButton::Left))?;+ post_event(mouse(CGEventType::LeftMouseDown, from, CGMouseButton::Left))?;+ for step in 1..=DRAG_STEPS {+ let t = f64::from(step) / f64::from(DRAG_STEPS);+ let at = Point { x: from.x + (to.x - from.x) * t, y: from.y + (to.y - from.y) * t };+ post_event(mouse(CGEventType::LeftMouseDragged, at, CGMouseButton::Left))?;+ }+ post_event(mouse(CGEventType::LeftMouseUp, to, CGMouseButton::Left))+}++fn scroll(at: Point, dx: i32, dy: i32) -> Result<(), String> {+ post_event(mouse(CGEventType::MouseMoved, at, CGMouseButton::Left))?;+ // Positive wheel values scroll up and left; the tool's dy > 0 scrolls down.+ post_event(CGEvent::new_scroll_wheel_event2(None, CGScrollEventUnit::Pixel, 2, -dy, -dx, 0))+}++/// Presses the modifiers, then the key, then releases them in reverse, like a person does.+fn key(combo: &KeyCombo) -> Result<(), String> {+ let held = modifiers(combo);+ let mut flags = CGEventFlags::empty();+ for (code, flag) in &held {+ flags |= *flag;+ key_event(*code, true, flags)?;+ }+ key_event(combo.code, true, flags)?;+ key_event(combo.code, false, flags)?;+ for (code, flag) in held.iter().rev() {+ flags &= !*flag;+ key_event(*code, false, flags)?;+ }+ Ok(())+}++fn text(chunk: &str) -> Result<(), String> {+ let units: Vec<u16> = chunk.encode_utf16().collect();+ for down in [true, false] {+ let event = CGEvent::new_keyboard_event(None, 0, down);+ if let Some(event) = &event {+ CGEvent::set_flags(Some(event), CGEventFlags::empty());+ // SAFETY: `units` outlives the call and holds `units.len()` UTF-16 units.+ unsafe { CGEvent::keyboard_set_unicode_string(Some(event), units.len() as _, units.as_ptr()) };+ }+ post_event(event)?;+ }+ Ok(())+}++fn pointer_location() -> Point {+ let event = CGEvent::new(None);+ let at = CGEvent::location(event.as_deref());+ Point { x: at.x, y: at.y }+}++fn display_at(at: Point) -> Option<CGDirectDisplayID> {+ let mut display: CGDirectDisplayID = 0;+ let mut count = 0u32;+ // SAFETY: both out-pointers are valid for one display.+ let error = unsafe { CGGetDisplaysWithPoint(CGPoint { x: at.x, y: at.y }, 1, &mut display, &mut count) };+ (error == CGError::Success && count == 1).then_some(display)+}++fn shareable_content() -> Result<Retained<SCShareableContent>, String> {+ let (tx, rx) = mpsc::channel();+ let handler = RcBlock::new(move |content: *mut SCShareableContent, error: *mut NSError| {+ // SAFETY: ScreenCaptureKit passes valid (or null) objects to the handler.+ let content = unsafe { Retained::retain(content) };+ let error = unsafe { error.as_ref() }.map(|e| e.localizedDescription().to_string());+ let _ = tx.send(Handoff(content.ok_or_else(|| error.unwrap_or_else(|| "no shareable content".into()))));+ });+ // SAFETY: the handler matches the documented signature.+ unsafe { SCShareableContent::getShareableContentWithCompletionHandler(&handler) };+ let Handoff(content) = rx.recv_timeout(CAPTURE_TIMEOUT).map_err(|_| "the screen capture timed out")?;+ content.map_err(|e| format!("cannot list the screen content: {e}"))+}++fn capture_image(filter: &SCContentFilter, config: &SCStreamConfiguration) -> Result<CFRetained<CGImage>, String> {+ let (tx, rx) = mpsc::channel();+ let handler = RcBlock::new(move |image: *mut CGImage, error: *mut NSError| {+ // SAFETY: the image is valid for the handler's duration; retaining keeps it alive.+ let image = NonNull::new(image).map(|image| unsafe { CFRetained::retain(image) });+ let error = unsafe { error.as_ref() }.map(|e| e.localizedDescription().to_string());+ let _ = tx.send(Handoff(image.ok_or_else(|| error.unwrap_or_else(|| "no image".into()))));+ });+ // SAFETY: the handler matches the documented signature.+ unsafe {+ SCScreenshotManager::captureImageWithFilter_configuration_completionHandler(filter, config, Some(&handler))+ };+ let Handoff(image) = rx.recv_timeout(CAPTURE_TIMEOUT).map_err(|_| "the screenshot timed out")?;+ image.map_err(|e| format!("cannot take the screenshot: {e}"))+}++fn jpeg_of(image: &CGImage) -> Result<(Vec<u8>, u32, u32), String> {+ let (width, height) = (CGImage::width(Some(image)), CGImage::height(Some(image)));+ if CGImage::bits_per_pixel(Some(image)) != 32 {+ return Err("unexpected screenshot pixel format".into());+ }+ let data = CGImage::data_provider(Some(image)).and_then(|p| CGDataProvider::data(Some(&p)));+ let bytes = data.ok_or("cannot read the screenshot pixels")?.to_vec();+ let jpeg = bgra_to_jpeg(&bytes, width as u32, height as u32, CGImage::bytes_per_row(Some(image)), JPEG_QUALITY)?;+ Ok((jpeg, width as u32, height as u32))+}++fn ax_string(element: &AXUIElement, attribute: &str) -> Option<String> {+ let value = ax_value(element, attribute)?;+ value.downcast_ref::<CFString>().map(|s| s.to_string()).filter(|s| !s.trim().is_empty())+}++fn ax_value(element: &AXUIElement, attribute: &str) -> Option<CFRetained<CFType>> {+ let attribute = CFString::from_str(attribute);+ let mut value: *const CFType = std::ptr::null();+ // SAFETY: `value` is a valid out-pointer; on success it holds a +1 reference.+ let error = unsafe { element.copy_attribute_value(&attribute, NonNull::from(&mut value)) };+ if error != AXError::Success {+ return None;+ }+ NonNull::new(value.cast_mut()).map(|value| unsafe { CFRetained::from_raw(value) })+}++fn ax_element(element: &AXUIElement, attribute: &str) -> Option<CFRetained<AXUIElement>> {+ ax_value(element, attribute)?.downcast::<AXUIElement>().ok()+}++fn describe(element: &AXUIElement) -> AxElement {+ AxElement {+ role: ax_string(element, "AXRole").unwrap_or_default(),+ title: ax_string(element, "AXTitle"),+ description: ax_string(element, "AXDescription"),+ value: ax_string(element, "AXValue"),+ }+}++fn system_wide() -> CFRetained<AXUIElement> {+ // SAFETY: plain constructor.+ let element = unsafe { AXUIElement::new_system_wide() };+ // A hung app must not stall the helper.+ unsafe { element.set_messaging_timeout(0.5) };+ element+}++type WindowInfo = NSDictionary<NSString, objc2::runtime::AnyObject>;++/// A window-list key (a `CFString` constant) as the bridged `NSString`.+fn info_key(key: &'static CFString) -> &'static NSString {+ // SAFETY: CFString is toll-free bridged to NSString.+ unsafe { &*(key as *const CFString).cast::<NSString>() }+}++/// `{X, Y, Width, Height}` of a window-list entry, in global points.+fn window_bounds(entry: &WindowInfo) -> Option<CGRect> {+ // SAFETY: extern constants provided by CoreGraphics.+ let bounds = entry.objectForKey(info_key(unsafe { kCGWindowBounds }))?;+ let bounds = bounds.downcast::<NSDictionary>().ok()?;+ let number = |key: &str| -> Option<f64> {+ let value = bounds.objectForKey(&NSString::from_str(key))?;+ Some(value.downcast::<NSNumber>().ok()?.doubleValue())+ };+ let origin = CGPoint::new(number("X")?, number("Y")?);+ Some(CGRect::new(origin, CGSize::new(number("Width")?, number("Height")?)))+}++impl Desktop for MacDesktop {+ fn permissions(&self) -> Permissions {+ Permissions {+ // SAFETY: plain queries.+ accessibility: unsafe { AXIsProcessTrusted() },+ screen_recording: CGPreflightScreenCaptureAccess(),+ screenshots_supported: available!(macos = 14.0),+ }+ }++ fn capture(&self) -> Result<Capture, String> {+ if !available!(macos = 14.0) {+ return Err("screenshots need macOS 14 or later".into());+ }+ let display_id = display_at(pointer_location()).ok_or("no display under the pointer")?;+ let content = shareable_content()?;+ // SAFETY: reading properties of the snapshot.+ let display = unsafe { content.displays() }+ .iter()+ .find(|d| unsafe { d.displayID() } == display_id)+ .ok_or("the display under the pointer cannot be captured")?;+ let own: Vec<Retained<SCWindow>> = unsafe { content.windows() }+ .iter()+ .filter(|w| unsafe { w.owningApplication() }.is_some_and(|app| unsafe { app.processID() } == self.own_pid))+ .collect();+ let excluded = NSArray::from_retained_slice(&own);+ let frame = unsafe { display.frame() };+ let (width, height) = fit_long_edge(frame.size.width as u32, frame.size.height as u32, MAX_LONG_EDGE);+ let filter = unsafe {+ SCContentFilter::initWithDisplay_excludingWindows(SCContentFilter::alloc(), &display, &excluded)+ };+ let config = unsafe { SCStreamConfiguration::new() };+ unsafe {+ config.setWidth(width as usize);+ config.setHeight(height as usize);+ config.setPixelFormat(PIXEL_FORMAT_BGRA);+ config.setShowsCursor(true);+ }+ let image = capture_image(&filter, &config)?;+ let (data, image_width, image_height) = jpeg_of(&image)?;+ let origin = Point { x: frame.origin.x, y: frame.origin.y };+ let geometry = ScreenshotGeometry::for_display(origin, frame.size.width, image_width, image_height);+ Ok(Capture { data, mime_type: "image/jpeg", geometry })+ }++ fn post(&self, event: &InputEvent) -> Result<(), String> {+ match event {+ InputEvent::Move(at) => post_event(mouse(CGEventType::MouseMoved, *at, CGMouseButton::Left)),+ InputEvent::Click { at, button, count } => click(*at, *button, *count),+ InputEvent::Drag { from, to } => drag(*from, *to),+ InputEvent::Scroll { at, dx, dy } => scroll(*at, *dx, *dy),+ InputEvent::Key(combo) => key(combo),+ InputEvent::Text(chunk) => text(chunk),+ }+ }++ fn list_apps(&self) -> Result<Vec<AppInfo>, String> {+ let apps = NSWorkspace::sharedWorkspace().runningApplications();+ Ok(apps+ .iter()+ .filter(|app| app.activationPolicy() == NSApplicationActivationPolicy::Regular)+ .map(|app| {+ let id = identity(&app);+ AppInfo { name: id.name, bundle_id: id.bundle_id, active: app.isActive() }+ })+ .collect())+ }++ fn open_app(&self, name: &str) -> Result<(), String> {+ // A bundle id has dots and no spaces; anything else is an app name.+ let by_bundle = name.contains('.') && !name.contains(' ') && !name.ends_with(".app");+ let output = std::process::Command::new("/usr/bin/open")+ .arg(if by_bundle { "-b" } else { "-a" })+ .arg(name)+ .output()+ .map_err(|e| format!("cannot run `open`: {e}"))?;+ if output.status.success() {+ Ok(())+ } else {+ Err(format!("cannot open `{name}`: {}", String::from_utf8_lossy(&output.stderr).trim()))+ }+ }++ fn element_at(&self, at: Point) -> Vec<AxElement> {+ let mut found: *const AXUIElement = std::ptr::null();+ // SAFETY: `found` is a valid out-pointer; on success it holds a +1 reference.+ let out = NonNull::from(&mut found);+ let error = unsafe { system_wide().copy_element_at_position(at.x as f32, at.y as f32, out) };+ let Some(found) = NonNull::new(found.cast_mut()).filter(|_| error == AXError::Success) else {+ return Vec::new();+ };+ let mut element = unsafe { CFRetained::from_raw(found) };+ let mut chain = vec![describe(&element)];+ while chain.len() <= AX_ANCESTORS {+ let Some(parent) = ax_element(&element, "AXParent") else { break };+ chain.push(describe(&parent));+ element = parent;+ }+ chain+ }++ fn focused_element(&self) -> Option<Focus> {+ let focused = ax_element(&system_wide(), "AXFocusedUIElement")?;+ let default = ax_element(&focused, "AXWindow").and_then(|window| ax_element(&window, "AXDefaultButton"));+ let is_default_button = default.as_ref().is_some_and(|d| **d == *focused);+ Some(Focus { element: describe(&focused), is_default_button, default_button: default.as_deref().map(describe) })+ }++ fn frontmost_app(&self) -> Option<AppIdentity> {+ NSWorkspace::sharedWorkspace().frontmostApplication().map(|app| identity(&app))+ }++ fn window_owner_at(&self, at: Point) -> Option<AppIdentity> {+ let options = CGWindowListOption::OptionOnScreenOnly | CGWindowListOption::ExcludeDesktopElements;+ let list = CGWindowListCopyWindowInfo(options, kCGNullWindowID)?;+ // SAFETY: CFArray of CFDictionary is toll-free bridged to NSArray of NSDictionary.+ let list: &NSArray<WindowInfo> = unsafe { &*(CFRetained::as_ptr(&list).as_ptr() as *const _) };+ // SAFETY: extern constants provided by CoreGraphics.+ let (alpha_key, pid_key, name_key) =+ unsafe { (info_key(kCGWindowAlpha), info_key(kCGWindowOwnerPID), info_key(kCGWindowOwnerName)) };+ let point = CGPoint { x: at.x, y: at.y };+ // Front to back: the first visible window containing the point owns it.+ let entry = list.iter().find(|entry| {+ let alpha = entry+ .objectForKey(alpha_key)+ .and_then(|a| a.downcast::<NSNumber>().ok())+ .map_or(1.0, |a| a.doubleValue());+ let bounds = window_bounds(entry);+ alpha > 0.0+ && bounds.is_some_and(|b| {+ point.x >= b.origin.x+ && point.y >= b.origin.y+ && point.x < b.origin.x + b.size.width+ && point.y < b.origin.y + b.size.height+ })+ })?;+ let pid = entry.objectForKey(pid_key)?.downcast::<NSNumber>().ok()?.intValue();+ match NSRunningApplication::runningApplicationWithProcessIdentifier(pid) {+ Some(app) => Some(identity(&app)),+ None => {+ let name = entry+ .objectForKey(name_key)+ .and_then(|n| n.downcast::<NSString>().ok())+ .map(|n| n.to_string())+ .unwrap_or_default();+ Some(AppIdentity { pid, bundle_id: None, name })+ }+ }+ }++ fn activate(&self, pid: i32) {+ if let Some(app) = NSRunningApplication::runningApplicationWithProcessIdentifier(pid) {+ app.activateWithOptions(NSApplicationActivationOptions::empty());+ }+ }+}++/// Shows the system prompt for `kind` (macOS shows it once per app) and opens its pane in System Settings.+pub fn request_permission(kind: PermissionKind) -> Result<(), String> {+ match kind {+ PermissionKind::Accessibility => {+ // SAFETY: the extern static is a valid CFString provided by HIServices.+ let key: &CFString = unsafe { objc2_application_services::kAXTrustedCheckOptionPrompt };+ let options = CFDictionary::<CFString, CFBoolean>::from_slices(&[key], &[CFBoolean::new(true)]);+ // SAFETY: `options` is a valid dictionary with the documented key.+ unsafe { AXIsProcessTrustedWithOptions(Some(options.as_opaque())) };+ }+ PermissionKind::ScreenRecording => {+ CGRequestScreenCaptureAccess();+ }+ }+ open_privacy_pane(kind)+}++/// Opens the Privacy & Security pane for `kind` in System Settings.+pub fn open_privacy_pane(kind: PermissionKind) -> Result<(), String> {+ let pane = match kind {+ PermissionKind::Accessibility => "Privacy_Accessibility",+ PermissionKind::ScreenRecording => "Privacy_ScreenCapture",+ };+ let url = format!("x-apple.systempreferences:com.apple.preference.security?{pane}");+ let status = std::process::Command::new("/usr/bin/open").arg(url).status().map_err(|e| e.to_string())?;+ status.success().then_some(()).ok_or_else(|| "cannot open System Settings".into())+}diff --git a/apps/desktop/src-tauri/src/computer/mcp.rs b/apps/desktop/src-tauri/src/computer/mcp.rsnew file mode 100644index 0000000..58b0f84--- /dev/null+++ b/apps/desktop/src-tauri/src/computer/mcp.rs@@ -0,0 +1,1085 @@+//! The MCP endpoint (S24.1): JSON-RPC 2.0 over streamable HTTP with `initialize`,+//! `tools/list`, and `tools/call`, behind a per-launch bearer token. Each call passes the+//! guards in order: rate limit, permissions, session grant, blocklist, risky-action+//! confirmation; then it acts and returns a fresh screenshot (S24.2–S24.4).++use super::blocklist::{blocked_name, blocked_reason, AppIdentity};+use super::control::Control;+use super::coords::{Point, ScreenshotGeometry};+use super::keys::{parse_combo, KeyCombo};+use super::rate::{RateLimiter, MAX_ACTIONS_PER_SECOND};+use super::risk::{assess, Act};+use super::{Clock, Desktop, Dialogs, InputEvent, MouseButton};+use base64::Engine as _;+use serde_json::{json, Map, Value};+use std::sync::{Arc, Mutex, MutexGuard};++/// Supported MCP revisions, newest first; the newest answers an unknown request.+const PROTOCOL_VERSIONS: &[&str] = &["2025-06-18", "2025-03-26", "2024-11-05"];+/// Time for the screen to settle before the screenshot that follows an action.+const SETTLE_MS: u64 = 300;+const OPEN_APP_SETTLE_MS: u64 = 1500;+const MAX_WAIT_MS: u64 = 5000;+const WAIT_SLICE_MS: u64 = 100;+/// macOS posts at most 20 UTF-16 units of text per keyboard event.+const TEXT_CHUNK_UNITS: usize = 20;++const STOPPED: &str = "Computer control was stopped; this action was dropped.";+const NOT_ALLOWED: &str = "Computer control was not allowed by the user. Nothing was done.";+const NEEDS_PERMISSION: &str = "Gentle Dot needs the";+const GRANT_IT: &str = "Ask the user to grant it in Connectors → Computer.";+const NEEDS_SCREENSHOT: &str = "Take a screenshot first: coordinates are pixels of the latest screenshot.";++const INSTRUCTIONS: &str = "Controls this Mac. Take a screenshot first; x and y are pixels of the latest \+screenshot. Every action returns a fresh screenshot. The user must allow a session, risky actions ask \+the user to confirm, and some apps (System Settings, password managers) are off limits.";++pub struct HttpRequest<'a> {+ pub method: &'a str,+ pub path: &'a str,+ pub authorization: Option<&'a str>,+ pub body: &'a str,+}++#[derive(Debug, Clone, PartialEq)]+pub struct HttpReply {+ pub status: u16,+ pub body: Option<Value>,+}++enum Tool {+ Screenshot,+ Click { x: f64, y: f64, button: MouseButton, count: u8 },+ Move { x: f64, y: f64 },+ Drag { from: (f64, f64), to: (f64, f64) },+ Scroll { x: f64, y: f64, dx: i32, dy: i32 },+ Type { text: String },+ Key { combo: KeyCombo },+ OpenApp { name: String },+ ListApps,+ Wait { ms: u64 },+}++struct Call {+ tool: Tool,+ intent: Option<String>,+}++type Content = Vec<Value>;++fn text(text: impl Into<String>) -> Value {+ json!({"type": "text", "text": text.into()})+}++fn tool_error(message: &str) -> Value {+ json!({"content": [text(message)], "isError": true})+}++fn lock<T>(mutex: &Mutex<T>) -> MutexGuard<'_, T> {+ mutex.lock().unwrap_or_else(|poisoned| poisoned.into_inner())+}++fn constant_time_eq(a: &[u8], b: &[u8]) -> bool {+ a.len() == b.len() && a.iter().zip(b).fold(0u8, |acc, (x, y)| acc | (x ^ y)) == 0+}++fn rpc_reply(status: u16, id: Value, outcome: Result<Value, (i64, String)>) -> HttpReply {+ let body = match outcome {+ Ok(result) => json!({"jsonrpc": "2.0", "id": id, "result": result}),+ Err((code, message)) => json!({"jsonrpc": "2.0", "id": id, "error": {"code": code, "message": message}}),+ };+ HttpReply { status, body: Some(body) }+}++fn schema(properties: Value, required: &[&str]) -> Value {+ let mut properties = properties.as_object().cloned().unwrap_or_default();+ properties.insert(+ "intent".into(),+ json!({"type": "string", "description": "What this step is for, in a few words. Shown to the user when the action needs a confirmation."}),+ );+ json!({"type": "object", "properties": properties, "required": required, "additionalProperties": false})+}++fn coordinate(description: &str) -> Value {+ json!({"type": "number", "minimum": 0, "description": description})+}++/// The tool catalog (S24.2).+fn tools() -> Value {+ let x = coordinate("Pixels from the left edge of the latest screenshot.");+ let y = coordinate("Pixels from the top edge of the latest screenshot.");+ json!([+ {"name": "screenshot", "description": "Take a screenshot of the display under the pointer. Returns the image and its size.",+ "inputSchema": schema(json!({}), &[])},+ {"name": "click", "description": "Click at a point of the latest screenshot.",+ "inputSchema": schema(json!({"x": x, "y": y,+ "button": {"type": "string", "enum": ["left", "right"], "default": "left"},+ "count": {"type": "integer", "enum": [1, 2, 3], "default": 1, "description": "2 for a double click, 3 for a triple click."}}), &["x", "y"])},+ {"name": "move", "description": "Move the pointer to a point of the latest screenshot.",+ "inputSchema": schema(json!({"x": x, "y": y}), &["x", "y"])},+ {"name": "drag", "description": "Press at one point, move to another, and release.",+ "inputSchema": schema(json!({"fromX": x, "fromY": y, "toX": x, "toY": y}), &["fromX", "fromY", "toX", "toY"])},+ {"name": "scroll", "description": "Scroll at a point. dy > 0 scrolls down and dx > 0 scrolls right, in pixels.",+ "inputSchema": schema(json!({"x": x, "y": y, "dx": {"type": "integer"}, "dy": {"type": "integer"}}), &["x", "y", "dx", "dy"])},+ {"name": "type", "description": "Type text into the frontmost app. A newline presses Return.",+ "inputSchema": schema(json!({"text": {"type": "string"}}), &["text"])},+ {"name": "key", "description": "Press a key combo in the frontmost app, for example `cmd+shift+t`, `return`, or `esc`.",+ "inputSchema": schema(json!({"combo": {"type": "string"}}), &["combo"])},+ {"name": "open_app", "description": "Open or bring forward an app by name or bundle id.",+ "inputSchema": schema(json!({"name": {"type": "string"}}), &["name"])},+ {"name": "list_apps", "description": "List the running apps.",+ "inputSchema": schema(json!({}), &[])},+ {"name": "wait", "description": "Wait, then take a screenshot.",+ "inputSchema": schema(json!({"ms": {"type": "integer", "minimum": 0, "maximum": MAX_WAIT_MS}}), &["ms"])}+ ])+}++fn number(args: &Map<String, Value>, key: &str) -> Result<f64, String> {+ args.get(key).and_then(Value::as_f64).ok_or_else(|| format!("`{key}` must be a number"))+}++fn integer(args: &Map<String, Value>, key: &str) -> Result<i64, String> {+ let value = number(args, key)?;+ if value.fract() != 0.0 || value.abs() > 1e9 {+ return Err(format!("`{key}` must be a whole number"));+ }+ Ok(value as i64)+}++fn string(args: &Map<String, Value>, key: &str) -> Result<String, String> {+ args.get(key).and_then(Value::as_str).map(str::to_string).ok_or_else(|| format!("`{key}` must be a string"))+}++/// The call for `name`, `None` for an unknown tool, or why the arguments are invalid.+fn parse_call(name: &str, args: &Value) -> Option<Result<Call, String>> {+ let empty = Map::new();+ let args = args.as_object().unwrap_or(&empty);+ let point = |kx: &str, ky: &str| -> Result<(f64, f64), String> { Ok((number(args, kx)?, number(args, ky)?)) };+ let tool = || -> Result<Tool, String> {+ Ok(match name {+ "screenshot" => Tool::Screenshot,+ "click" => {+ let (x, y) = point("x", "y")?;+ let button = match args.get("button").map(|b| b.as_str()) {+ None | Some(Some("left")) => MouseButton::Left,+ Some(Some("right")) => MouseButton::Right,+ _ => return Err("`button` must be \"left\" or \"right\"".into()),+ };+ let count = if args.contains_key("count") { integer(args, "count")? } else { 1 };+ if !(1..=3).contains(&count) {+ return Err("`count` must be 1, 2, or 3".into());+ }+ Tool::Click { x, y, button, count: count as u8 }+ }+ "move" => {+ let (x, y) = point("x", "y")?;+ Tool::Move { x, y }+ }+ "drag" => Tool::Drag { from: point("fromX", "fromY")?, to: point("toX", "toY")? },+ "scroll" => {+ let (x, y) = point("x", "y")?;+ let delta = |key| integer(args, key).and_then(|v| i32::try_from(v).map_err(|e| e.to_string()));+ Tool::Scroll { x, y, dx: delta("dx")?, dy: delta("dy")? }+ }+ "type" => Tool::Type { text: string(args, "text")? },+ "key" => Tool::Key { combo: parse_combo(&string(args, "combo")?)? },+ "open_app" => {+ let name = string(args, "name")?.trim().to_string();+ if name.is_empty() {+ return Err("`name` must not be empty".into());+ }+ Tool::OpenApp { name }+ }+ "list_apps" => Tool::ListApps,+ "wait" => {+ let ms = integer(args, "ms")?;+ if !(0..=MAX_WAIT_MS as i64).contains(&ms) {+ return Err(format!("`ms` must be between 0 and {MAX_WAIT_MS}"));+ }+ Tool::Wait { ms: ms as u64 }+ }+ other => return Err(format!("unknown tool `{other}`")),+ })+ };+ const NAMES: &[&str] =+ &["screenshot", "click", "move", "drag", "scroll", "type", "key", "open_app", "list_apps", "wait"];+ if !NAMES.contains(&name) {+ return None;+ }+ let intent = args.get("intent").and_then(Value::as_str).map(str::to_string);+ Some(tool().map(|tool| Call { tool, intent }))+}++/// Splits text into chunks of at most `TEXT_CHUNK_UNITS` UTF-16 units, never inside a character.+fn chunks(text: &str) -> Vec<String> {+ let mut out = Vec::new();+ let mut current = String::new();+ for c in text.chars() {+ if current.encode_utf16().count() + c.len_utf16() > TEXT_CHUNK_UNITS {+ out.push(std::mem::take(&mut current));+ }+ current.push(c);+ }+ if !current.is_empty() {+ out.push(current);+ }+ out+}++pub struct Helper {+ desktop: Arc<dyn Desktop>,+ dialogs: Arc<dyn Dialogs>,+ clock: Arc<dyn Clock>,+ control: Arc<Control>,+ token: String,+ own_pid: i32,+ limiter: Mutex<RateLimiter>,+ /// Serializes actions and holds the geometry of the latest screenshot.+ actions: Mutex<Option<ScreenshotGeometry>>,+}++impl Helper {+ pub fn new(+ desktop: Arc<dyn Desktop>,+ dialogs: Arc<dyn Dialogs>,+ clock: Arc<dyn Clock>,+ control: Arc<Control>,+ token: String,+ own_pid: i32,+ ) -> Self {+ Helper {+ desktop,+ dialogs,+ clock,+ control,+ token,+ own_pid,+ limiter: Mutex::new(RateLimiter::new(MAX_ACTIONS_PER_SECOND, 1000)),+ actions: Mutex::new(None),+ }+ }++ fn authorized(&self, header: Option<&str>) -> bool {+ let Some((scheme, token)) = header.and_then(|h| h.trim().split_once(' ')) else {+ return false;+ };+ scheme.eq_ignore_ascii_case("bearer") && constant_time_eq(token.trim().as_bytes(), self.token.as_bytes())+ }++ pub fn handle(&self, request: &HttpRequest<'_>) -> HttpReply {+ if !self.authorized(request.authorization) {+ return rpc_reply(401, Value::Null, Err((-32001, "Unauthorized".into())));+ }+ if request.path != "/mcp" {+ return HttpReply { status: 404, body: None };+ }+ if request.method != "POST" {+ return HttpReply { status: 405, body: None };+ }+ let Ok(message) = serde_json::from_str::<Value>(request.body) else {+ return rpc_reply(400, Value::Null, Err((-32700, "Parse error".into())));+ };+ if !message.is_object() {+ return rpc_reply(400, Value::Null, Err((-32600, "Invalid request".into())));+ }+ // Notifications and responses get no reply.+ let (Some(method), Some(id)) = (message["method"].as_str(), message.get("id").cloned()) else {+ return HttpReply { status: 202, body: None };+ };+ let params = message.get("params").cloned().unwrap_or(Value::Null);+ let outcome = match method {+ "initialize" => Ok(self.initialize(¶ms)),+ "ping" => Ok(json!({})),+ "tools/list" => Ok(json!({"tools": tools()})),+ "tools/call" => self.tools_call(¶ms),+ _ => Err((-32601, format!("Method not found: {method}"))),+ };+ rpc_reply(200, id, outcome)+ }++ fn initialize(&self, params: &Value) -> Value {+ let requested = params["protocolVersion"].as_str();+ let version = requested.filter(|v| PROTOCOL_VERSIONS.contains(v)).unwrap_or(PROTOCOL_VERSIONS[0]);+ json!({+ "protocolVersion": version,+ "capabilities": {"tools": {"listChanged": false}},+ "serverInfo": {"name": "gentle-dot-computer", "version": env!("CARGO_PKG_VERSION")},+ "instructions": INSTRUCTIONS,+ })+ }++ fn tools_call(&self, params: &Value) -> Result<Value, (i64, String)> {+ let name = params["name"].as_str().ok_or((-32602, "Invalid params: `name` is required".to_string()))?;+ let args = params.get("arguments").cloned().unwrap_or_else(|| json!({}));+ self.control.expire(self.clock.now_ms());+ let epoch = self.control.epoch();+ self.call_tool_from(epoch, name, &args).ok_or((-32602, format!("Unknown tool: {name}")))+ }++ /// A `tools/call` that arrived while the session was at `epoch`.+ fn call_tool_from(&self, epoch: u64, name: &str, args: &Value) -> Option<Value> {+ let outcome = parse_call(name, args)?.and_then(|call| self.run(epoch, call));+ Some(match outcome {+ Ok(content) => json!({"content": content}),+ Err(message) => tool_error(&message),+ })+ }++ fn run(&self, epoch: u64, call: Call) -> Result<Content, String> {+ if !lock(&self.limiter).try_acquire(self.clock.now_ms()) {+ return Err(format!("Too many actions: at most {MAX_ACTIONS_PER_SECOND} per second. Wait a moment."));+ }+ self.check_permissions(&call.tool)?;+ let mut latest = lock(&self.actions);+ self.control.expire(self.clock.now_ms());+ if self.control.epoch() != epoch {+ return Err(STOPPED.into());+ }+ if !self.control.is_active(self.clock.now_ms()) {+ if !self.with_dialog(|| self.dialogs.ask_grant()) {+ return Err(NOT_ALLOWED.into());+ }+ self.control.grant(self.clock.now_ms());+ }+ self.act(epoch, &mut latest, call)+ }++ fn check_permissions(&self, tool: &Tool) -> Result<(), String> {+ let permissions = self.desktop.permissions();+ let sees = !matches!(tool, Tool::ListApps);+ let acts = matches!(+ tool,+ Tool::Click { .. }+ | Tool::Move { .. }+ | Tool::Drag { .. }+ | Tool::Scroll { .. }+ | Tool::Type { .. }+ | Tool::Key { .. }+ );+ if sees && !permissions.screenshots_supported {+ return Err("Screenshots need macOS 14 or later, so computer control is unavailable on this Mac.".into());+ }+ if sees && !permissions.screen_recording {+ return Err(format!("{NEEDS_PERMISSION} Screen Recording permission. {GRANT_IT}"));+ }+ if acts && !permissions.accessibility {+ return Err(format!("{NEEDS_PERMISSION} Accessibility permission. {GRANT_IT}"));+ }+ Ok(())+ }++ /// Runs a dialog, then brings back the app that was in front (a dialog activates Gentle Dot).+ fn with_dialog(&self, dialog: impl FnOnce() -> bool) -> bool {+ let before = self.desktop.frontmost_app();+ let answer = dialog();+ if let Some(before) = before {+ if self.desktop.frontmost_app().map(|app| app.pid) != Some(before.pid) {+ self.desktop.activate(before.pid);+ }+ }+ answer+ }++ fn still_running(&self, epoch: u64) -> Result<(), String> {+ if self.control.epoch() != epoch || !self.control.is_active(self.clock.now_ms()) {+ return Err(STOPPED.into());+ }+ Ok(())+ }++ fn post(&self, epoch: u64, event: InputEvent) -> Result<(), String> {+ self.still_running(epoch)?;+ self.desktop.post(&event)+ }++ fn refuse_blocked(&self, app: &AppIdentity) -> Result<(), String> {+ match blocked_reason(app, self.own_pid) {+ Some(label) => Err(format!("Gentle Dot does not control {label}. The action was refused.")),+ None => Ok(()),+ }+ }++ /// The app owning the window under `at`, refused when blocked. Its name, for messages.+ fn pointer_target(&self, at: Point) -> Result<String, String> {+ match self.desktop.window_owner_at(at) {+ Some(app) => self.refuse_blocked(&app).map(|()| app.name),+ None => Ok("the desktop".into()),+ }+ }++ /// The frontmost app, refused when blocked. Its name, for messages.+ fn keyboard_target(&self) -> Result<String, String> {+ let app = self.desktop.frontmost_app().ok_or("Cannot tell which app is in front.")?;+ self.refuse_blocked(&app).map(|()| app.name)+ }++ fn confirm_if_risky(&self, risk: Option<String>, action: &str, target: &str) -> Result<(), String> {+ let Some(reason) = risk else {+ return Ok(());+ };+ let message = format!("{action} in {target}.\n\nThis needs your OK because of {reason}.");+ if self.with_dialog(|| self.dialogs.confirm(&message)) {+ Ok(())+ } else {+ Err(format!("The user declined: {action} in {target}. Nothing was done."))+ }+ }++ fn screenshot(&self, latest: &mut Option<ScreenshotGeometry>) -> Result<Content, String> {+ let capture = self.desktop.capture()?;+ let geometry = capture.geometry;+ *latest = Some(geometry);+ let front = self.desktop.frontmost_app().map(|app| app.name);+ let meta = json!({+ "imageWidth": geometry.image_width,+ "imageHeight": geometry.image_height,+ "scale": geometry.scale,+ "frontApp": front,+ });+ let data = base64::engine::general_purpose::STANDARD.encode(&capture.data);+ Ok(vec![json!({"type": "image", "data": data, "mimeType": capture.mime_type}), text(meta.to_string())])+ }++ /// What was done, then a fresh screenshot once the screen settled.+ fn after(&self, latest: &mut Option<ScreenshotGeometry>, done: String, settle_ms: u64) -> Result<Content, String> {+ self.clock.sleep_ms(settle_ms);+ let mut content = vec![text(done)];+ match self.screenshot(latest) {+ Ok(blocks) => content.extend(blocks),+ Err(error) => content.push(text(format!("No screenshot: {error}"))),+ }+ Ok(content)+ }++ fn act(&self, epoch: u64, latest: &mut Option<ScreenshotGeometry>, call: Call) -> Result<Content, String> {+ let intent = call.intent.as_deref();+ let map = |latest: &Option<ScreenshotGeometry>, (x, y): (f64, f64)| -> Result<Point, String> {+ latest.ok_or(NEEDS_SCREENSHOT)?.to_point(x, y).map_err(|e| e.to_string())+ };+ match call.tool {+ Tool::Screenshot => self.screenshot(latest),+ Tool::Click { x, y, button, count } => {+ let at = map(latest, (x, y))?;+ let target = self.pointer_target(at)?;+ let chain = self.desktop.element_at(at);+ let action = match (button, count) {+ (MouseButton::Right, _) => format!("Right-click at ({x}, {y})"),+ (_, 2) => format!("Double-click at ({x}, {y})"),+ (_, 3) => format!("Triple-click at ({x}, {y})"),+ _ => format!("Click at ({x}, {y})"),+ };+ self.confirm_if_risky(assess(Act::Click(&chain), intent), &action, &target)?;+ self.post(epoch, InputEvent::Click { at, button, count })?;+ self.after(latest, format!("{action} in {target}: done."), SETTLE_MS)+ }+ Tool::Move { x, y } => {+ let at = map(latest, (x, y))?;+ let target = self.pointer_target(at)?;+ let action = format!("Move the pointer to ({x}, {y})");+ self.confirm_if_risky(assess(Act::Other, intent), &action, &target)?;+ self.post(epoch, InputEvent::Move(at))?;+ self.after(latest, format!("{action}: done."), SETTLE_MS)+ }+ Tool::Drag { from, to } => {+ let (start, end) = (map(latest, from)?, map(latest, to)?);+ let source = self.pointer_target(start)?;+ let destination = self.pointer_target(end)?;+ let action = format!("Drag from ({}, {}) to ({}, {})", from.0, from.1, to.0, to.1);+ let target = if source == destination { source } else { format!("{source} and {destination}") };+ self.confirm_if_risky(assess(Act::Other, intent), &action, &target)?;+ self.post(epoch, InputEvent::Drag { from: start, to: end })?;+ self.after(latest, format!("{action}: done."), SETTLE_MS)+ }+ Tool::Scroll { x, y, dx, dy } => {+ let at = map(latest, (x, y))?;+ let target = self.pointer_target(at)?;+ let action = format!("Scroll by ({dx}, {dy}) at ({x}, {y})");+ self.confirm_if_risky(assess(Act::Other, intent), &action, &target)?;+ self.post(epoch, InputEvent::Scroll { at, dx, dy })?;+ self.after(latest, format!("{action}: done."), SETTLE_MS)+ }+ Tool::Type { text: typed } => {+ let target = self.keyboard_target()?;+ let typed = typed.replace("\r\n", "\n").replace('\r', "\n");+ let focus = self.desktop.focused_element();+ let act = if typed.contains('\n') { Act::Return(focus.as_ref()) } else { Act::Other };+ let action = format!("Type {} characters", typed.chars().count());+ self.confirm_if_risky(assess(act, intent), &action, &target)?;+ let return_key = parse_combo("return")?;+ for (i, line) in typed.split('\n').enumerate() {+ if i > 0 {+ self.post(epoch, InputEvent::Key(return_key))?;+ }+ for chunk in chunks(line) {+ self.post(epoch, InputEvent::Text(chunk))?;+ }+ }+ self.after(latest, format!("{action} in {target}: done."), SETTLE_MS)+ }+ Tool::Key { combo } => {+ let target = self.keyboard_target()?;+ let focus = self.desktop.focused_element();+ let act = if combo.is_return() { Act::Return(focus.as_ref()) } else { Act::Other };+ let action = format!("Press {}", combo.key);+ self.confirm_if_risky(assess(act, intent), &action, &target)?;+ self.post(epoch, InputEvent::Key(combo))?;+ self.after(latest, format!("{action} in {target}: done."), SETTLE_MS)+ }+ Tool::OpenApp { name } => {+ if let Some(label) = blocked_name(&name) {+ return Err(format!("Gentle Dot does not control {label}. The app was not opened."));+ }+ let action = format!("Open {name}");+ self.confirm_if_risky(assess(Act::Other, intent), &action, "macOS")?;+ self.still_running(epoch)?;+ self.desktop.open_app(&name)?;+ self.after(latest, format!("{action}: done."), OPEN_APP_SETTLE_MS)+ }+ Tool::ListApps => {+ let apps = self.desktop.list_apps()?;+ Ok(vec![text(serde_json::to_string(&apps).map_err(|e| e.to_string())?)])+ }+ Tool::Wait { ms } => {+ let mut left = ms;+ while left > 0 {+ let slice = left.min(WAIT_SLICE_MS);+ self.clock.sleep_ms(slice);+ left -= slice;+ self.still_running(epoch)?;+ }+ self.after(latest, format!("Waited {ms} ms."), 0)+ }+ }+ }+}++#[cfg(test)]+mod tests {+ use super::super::blocklist::AppIdentity;+ use super::super::coords::Point;+ use super::super::fake::{app, FakeClock, FakeDesktop, FakeDialogs};+ use super::super::keys::parse_combo;+ use super::super::risk::{AxElement, Focus};+ use super::super::session::{Reason, StateEvent, SESSION_MS};+ use super::super::{InputEvent, MouseButton};+ use super::*;+ use serde_json::json;+ use std::sync::atomic::Ordering;+ use std::sync::Mutex;++ const TOKEN: &str = "secret-token";+ const OWN_PID: i32 = 4242;+ const NOW: u64 = 1_700_000_000_000;++ struct Rig {+ helper: Helper,+ desktop: Arc<FakeDesktop>,+ dialogs: Arc<FakeDialogs>,+ clock: Arc<FakeClock>,+ control: Arc<Control>,+ states: Arc<Mutex<Vec<StateEvent>>>,+ }++ fn rig_with(allow: bool, confirm: bool) -> Rig {+ let desktop = Arc::new(FakeDesktop::default());+ let dialogs = Arc::new(FakeDialogs::new(allow, confirm));+ let clock = Arc::new(FakeClock::new(NOW));+ let states = Arc::new(Mutex::new(Vec::new()));+ let sink = states.clone();+ let control = Arc::new(Control::new(move |event| sink.lock().unwrap().push(event.clone())));+ let helper =+ Helper::new(desktop.clone(), dialogs.clone(), clock.clone(), control.clone(), TOKEN.into(), OWN_PID);+ Rig { helper, desktop, dialogs, clock, control, states }+ }++ fn rig() -> Rig {+ rig_with(true, true)+ }++ fn post(rig: &Rig, authorization: Option<&str>, body: &str) -> HttpReply {+ rig.helper.handle(&HttpRequest { method: "POST", path: "/mcp", authorization, body })+ }++ impl Rig {+ fn rpc(&self, method: &str, params: Value) -> Value {+ let body = json!({"jsonrpc": "2.0", "id": 7, "method": method, "params": params}).to_string();+ let reply = post(self, Some(&format!("Bearer {TOKEN}")), &body);+ assert_eq!(reply.status, 200, "{reply:?}");+ let body = reply.body.expect("a JSON-RPC reply");+ assert_eq!((body["jsonrpc"].as_str(), body["id"].as_i64()), (Some("2.0"), Some(7)));+ body+ }++ fn call(&self, name: &str, arguments: Value) -> Value {+ let body = self.rpc("tools/call", json!({"name": name, "arguments": arguments}));+ body["result"].clone()+ }++ /// Grants a session and takes the screenshot that coordinates refer to.+ fn ready(&self) {+ assert!(!is_error(&self.call("screenshot", json!({}))));+ }+ }++ fn is_error(result: &Value) -> bool {+ result["isError"] == json!(true)+ }++ fn first_text(result: &Value) -> String {+ result["content"][0]["text"].as_str().unwrap_or_default().to_string()+ }++ fn safari() -> AppIdentity {+ app(200, "com.apple.Safari", "Safari")+ }++ // --- HTTP and JSON-RPC ---++ #[test]+ fn requests_without_the_bearer_token_get_401() {+ let rig = rig();+ let body = json!({"jsonrpc": "2.0", "id": 1, "method": "tools/list"}).to_string();+ for authorization in [None, Some("Bearer wrong"), Some(TOKEN), Some("Basic secret-token")] {+ let reply = post(&rig, authorization, &body);+ assert_eq!(reply.status, 401, "{authorization:?}");+ }+ assert_eq!(post(&rig, Some("Bearer secret-token"), &body).status, 200);+ }++ #[test]+ fn only_post_to_mcp_is_served() {+ let rig = rig();+ let auth = Some("Bearer secret-token");+ let get = rig.helper.handle(&HttpRequest { method: "GET", path: "/mcp", authorization: auth, body: "" });+ assert_eq!(get.status, 405);+ let other = rig.helper.handle(&HttpRequest { method: "POST", path: "/other", authorization: auth, body: "{}" });+ assert_eq!(other.status, 404);+ }++ #[test]+ fn initialize_reports_the_server_and_tools_capability() {+ let rig = rig();+ let body = rig.rpc("initialize", json!({"protocolVersion": "2025-03-26", "capabilities": {}, "clientInfo": {"name": "t", "version": "1"}}));+ let result = &body["result"];+ assert_eq!(result["protocolVersion"], "2025-03-26");+ assert!(result["capabilities"]["tools"].is_object());+ assert_eq!(result["serverInfo"]["name"], "gentle-dot-computer");+ }++ #[test]+ fn initialize_answers_an_unknown_protocol_version_with_its_own() {+ let rig = rig();+ let body = rig.rpc("initialize", json!({"protocolVersion": "1999-01-01"}));+ assert_eq!(body["result"]["protocolVersion"], "2025-06-18");+ }++ #[test]+ fn notifications_are_accepted_without_a_body() {+ let rig = rig();+ let body = json!({"jsonrpc": "2.0", "method": "notifications/initialized"}).to_string();+ assert_eq!(post(&rig, Some("Bearer secret-token"), &body), HttpReply { status: 202, body: None });+ }++ #[test]+ fn invalid_json_is_a_parse_error() {+ let reply = post(&rig(), Some("Bearer secret-token"), "{not json");+ assert_eq!(reply.status, 400);+ assert_eq!(reply.body.unwrap()["error"]["code"], -32700);+ }++ #[test]+ fn unknown_methods_are_rejected() {+ let body = rig().rpc("resources/list", json!({}));+ assert_eq!(body["error"]["code"], -32601);+ }++ #[test]+ fn tools_list_describes_every_tool_with_an_input_schema() {+ let body = rig().rpc("tools/list", json!({}));+ let tools = body["result"]["tools"].as_array().unwrap();+ let names: Vec<&str> = tools.iter().map(|t| t["name"].as_str().unwrap()).collect();+ assert_eq!(+ names,+ ["screenshot", "click", "move", "drag", "scroll", "type", "key", "open_app", "list_apps", "wait"]+ );+ for tool in tools {+ let schema = &tool["inputSchema"];+ assert_eq!(schema["type"], "object", "{tool}");+ assert_eq!(schema["properties"]["intent"]["type"], "string", "{tool}");+ assert!(tool["description"].as_str().is_some_and(|d| !d.is_empty()));+ }+ let click = tools.iter().find(|t| t["name"] == "click").unwrap();+ assert_eq!(click["inputSchema"]["required"], json!(["x", "y"]));+ assert_eq!(click["inputSchema"]["properties"]["button"]["enum"], json!(["left", "right"]));+ let wait = tools.iter().find(|t| t["name"] == "wait").unwrap();+ assert_eq!(wait["inputSchema"]["properties"]["ms"]["maximum"], 5000);+ }++ #[test]+ fn an_unknown_tool_is_a_protocol_error() {+ let rig = rig();+ let body = rig.rpc("tools/call", json!({"name": "format_disk", "arguments": {}}));+ assert_eq!(body["error"]["code"], -32602);+ assert_eq!(rig.dialogs.grants.load(Ordering::SeqCst), 0);+ }++ // --- Session grant (S24.3) ---++ #[test]+ fn deny_returns_a_tool_error_and_starts_nothing() {+ let rig = rig_with(false, true);+ let result = rig.call("screenshot", json!({}));+ assert!(is_error(&result));+ assert!(first_text(&result).contains("not allowed"), "{result}");+ assert_eq!(rig.dialogs.grants.load(Ordering::SeqCst), 1);+ assert_eq!(rig.desktop.captures.load(Ordering::SeqCst), 0);+ assert!(rig.states.lock().unwrap().is_empty());+ assert!(!rig.control.is_active(NOW));+ }++ #[test]+ fn allow_starts_a_session_once_and_reports_it() {+ let rig = rig();+ rig.ready();+ rig.ready();+ assert_eq!(rig.dialogs.grants.load(Ordering::SeqCst), 1);+ assert_eq!(+ *rig.states.lock().unwrap(),+ vec![StateEvent { active: true, ends_at: Some(NOW + SESSION_MS), reason: Some(Reason::Granted) }]+ );+ }++ #[test]+ fn after_the_timeout_the_next_call_asks_again() {+ let rig = rig();+ rig.ready();+ rig.clock.advance(SESSION_MS);+ rig.ready();+ assert_eq!(rig.dialogs.grants.load(Ordering::SeqCst), 2);+ }++ #[test]+ fn after_a_stop_the_next_call_asks_again() {+ let rig = rig();+ rig.ready();+ rig.control.end(Reason::Panic);+ rig.ready();+ assert_eq!(rig.dialogs.grants.load(Ordering::SeqCst), 2);+ }++ #[test]+ fn an_action_queued_before_a_stop_is_dropped() {+ let rig = rig();+ rig.ready();+ let queued_at = rig.control.epoch();+ rig.control.end(Reason::Stopped);+ let result = rig.helper.call_tool_from(queued_at, "click", &json!({"x": 10, "y": 10})).unwrap();+ assert!(is_error(&result));+ assert!(first_text(&result).contains("stopped"), "{result}");+ assert!(rig.desktop.events().is_empty());+ assert_eq!(rig.dialogs.grants.load(Ordering::SeqCst), 1);+ }++ #[test]+ fn a_stop_during_wait_ends_it_early() {+ let rig = rig();+ rig.ready();+ let control = rig.control.clone();+ *rig.clock.on_sleep.lock().unwrap() = Some(Box::new(move || control.end(Reason::Panic)));+ let result = rig.call("wait", json!({"ms": 3000}));+ assert!(is_error(&result));+ assert!(first_text(&result).contains("stopped"), "{result}");+ assert!(rig.clock.now_ms() - NOW < 3000);+ }++ // --- Screenshots and coordinates (S24.2) ---++ #[test]+ fn a_screenshot_is_an_image_block_plus_its_geometry() {+ let rig = rig();+ let result = rig.call("screenshot", json!({}));+ let content = result["content"].as_array().unwrap();+ let image = content.iter().find(|c| c["type"] == "image").unwrap();+ assert_eq!(image["mimeType"], "image/jpeg");+ assert_eq!(image["data"], "anBlZw==");+ let meta = content.iter().rfind(|c| c["type"] == "text").unwrap();+ let meta: Value = serde_json::from_str(meta["text"].as_str().unwrap()).unwrap();+ assert_eq!(meta, json!({"imageWidth": 1280, "imageHeight": 800, "scale": 0.5, "frontApp": "Safari"}));+ }++ #[test]+ fn coordinates_need_a_screenshot_first() {+ let rig = rig();+ let result = rig.call("click", json!({"x": 10, "y": 10}));+ assert!(is_error(&result));+ assert!(first_text(&result).contains("screenshot"), "{result}");+ assert!(rig.desktop.events().is_empty());+ }++ #[test]+ fn click_maps_pixels_to_points_and_returns_a_fresh_screenshot() {+ let rig = rig();+ rig.ready();+ let result = rig.call("click", json!({"x": 100, "y": 50, "button": "right", "count": 2}));+ assert!(!is_error(&result), "{result}");+ let at = Point { x: 200.0, y: 100.0 };+ assert_eq!(rig.desktop.events(), vec![InputEvent::Click { at, button: MouseButton::Right, count: 2 }]);+ assert_eq!(rig.desktop.captures.load(Ordering::SeqCst), 2);+ assert!(result["content"].as_array().unwrap().iter().any(|c| c["type"] == "image"));+ }++ #[test]+ fn move_drag_and_scroll_map_their_points() {+ let rig = rig();+ rig.ready();+ rig.call("move", json!({"x": 1, "y": 2}));+ rig.call("drag", json!({"fromX": 10, "fromY": 20, "toX": 30, "toY": 40}));+ rig.call("scroll", json!({"x": 5, "y": 6, "dx": 0, "dy": 120}));+ let p = |x, y| Point { x, y };+ assert_eq!(+ rig.desktop.events(),+ vec![+ InputEvent::Move(p(2.0, 4.0)),+ InputEvent::Drag { from: p(20.0, 40.0), to: p(60.0, 80.0) },+ InputEvent::Scroll { at: p(10.0, 12.0), dx: 0, dy: 120 },+ ]+ );+ }++ #[test]+ fn invalid_arguments_are_tool_errors() {+ let rig = rig();+ rig.ready();+ for (name, args) in [+ ("click", json!({"x": 1})),+ ("click", json!({"x": 1, "y": 1, "count": 4})),+ ("click", json!({"x": 1, "y": 1, "button": "middle"})),+ ("click", json!({"x": 5000, "y": 1})),+ ("wait", json!({"ms": 5001})),+ ("key", json!({"combo": "cmd+nope"})),+ ("type", json!({})),+ ] {+ let result = rig.call(name, args.clone());+ assert!(is_error(&result), "{name} {args}");+ }+ assert!(rig.desktop.events().is_empty());+ }++ // --- Permissions ---++ #[test]+ fn missing_permissions_are_explained_before_asking_for_a_session() {+ let rig = rig();+ rig.desktop.permissions.lock().unwrap().screen_recording = false;+ let result = rig.call("screenshot", json!({}));+ assert!(first_text(&result).contains("Screen Recording"), "{result}");+ rig.desktop.permissions.lock().unwrap().screen_recording = true;+ rig.desktop.permissions.lock().unwrap().accessibility = false;+ let result = rig.call("key", json!({"combo": "tab"}));+ assert!(first_text(&result).contains("Accessibility"), "{result}");+ assert_eq!(rig.dialogs.grants.load(Ordering::SeqCst), 0);+ }++ #[test]+ fn screenshots_before_macos_14_are_a_clear_error() {+ let rig = rig();+ rig.desktop.permissions.lock().unwrap().screenshots_supported = false;+ let result = rig.call("screenshot", json!({}));+ assert!(is_error(&result));+ assert!(first_text(&result).contains("macOS 14"), "{result}");+ }++ // --- Blocklist and rate limit (S24.4) ---++ #[test]+ fn pointer_actions_on_a_blocked_window_are_refused() {+ let rig = rig();+ rig.ready();+ *rig.desktop.owner_at.lock().unwrap() =+ Box::new(|_| Some(app(300, "com.apple.systempreferences", "System Settings")));+ let result = rig.call("click", json!({"x": 10, "y": 10}));+ assert!(is_error(&result));+ assert!(first_text(&result).contains("System Settings"), "{result}");+ assert!(rig.desktop.events().is_empty());+ }++ #[test]+ fn a_drag_onto_a_blocked_window_is_refused() {+ let rig = rig();+ rig.ready();+ *rig.desktop.owner_at.lock().unwrap() = Box::new(|at: Point| {+ if at.x > 100.0 {+ Some(app(301, "com.1password.1password", "1Password"))+ } else {+ Some(safari())+ }+ });+ let result = rig.call("drag", json!({"fromX": 10, "fromY": 10, "toX": 200, "toY": 10}));+ assert!(is_error(&result));+ assert!(rig.desktop.events().is_empty());+ }++ #[test]+ fn gentle_dot_own_windows_are_refused() {+ let rig = rig();+ rig.ready();+ *rig.desktop.owner_at.lock().unwrap() =+ Box::new(|_| Some(AppIdentity { pid: OWN_PID, bundle_id: None, name: "gentle-dot".into() }));+ assert!(is_error(&rig.call("click", json!({"x": 10, "y": 10}))));+ }++ #[test]+ fn keyboard_actions_check_the_frontmost_app() {+ let rig = rig();+ rig.ready();+ *rig.desktop.frontmost.lock().unwrap() = Some(app(302, "com.apple.keychainaccess", "Keychain Access"));+ assert!(is_error(&rig.call("type", json!({"text": "hello"}))));+ assert!(is_error(&rig.call("key", json!({"combo": "cmd+a"}))));+ assert!(rig.desktop.events().is_empty());+ }++ #[test]+ fn opening_a_blocked_app_is_refused() {+ let rig = rig();+ rig.ready();+ assert!(is_error(&rig.call("open_app", json!({"name": "Keychain Access"}))));+ assert!(!is_error(&rig.call("open_app", json!({"name": "com.apple.mail"}))));+ assert_eq!(*rig.desktop.opened.lock().unwrap(), vec!["com.apple.mail".to_string()]);+ }++ #[test]+ fn more_than_ten_calls_per_second_are_refused() {+ let rig = rig();+ for _ in 0..10 {+ assert!(!is_error(&rig.call("list_apps", json!({}))));+ }+ let result = rig.call("list_apps", json!({}));+ assert!(is_error(&result));+ assert!(first_text(&result).contains("10"), "{result}");+ rig.clock.advance(1000);+ assert!(!is_error(&rig.call("list_apps", json!({}))));+ }++ // --- Risky actions (S24.4) ---++ fn send_button() -> Vec<AxElement> {+ vec![AxElement { role: "AXButton".into(), title: Some("Send".into()), ..AxElement::default() }]+ }++ #[test]+ fn a_risky_click_needs_a_confirmation_showing_action_and_target() {+ let rig = rig();+ rig.ready();+ *rig.desktop.elements.lock().unwrap() = send_button();+ let result = rig.call("click", json!({"x": 10, "y": 10}));+ assert!(!is_error(&result), "{result}");+ let asked = rig.dialogs.confirmations.lock().unwrap().clone();+ assert_eq!(asked.len(), 1);+ assert!(asked[0].contains("Send") && asked[0].contains("Safari"), "{asked:?}");+ assert_eq!(rig.desktop.events().len(), 1);+ }++ #[test]+ fn a_declined_confirmation_is_a_tool_error_and_nothing_happens() {+ let rig = rig_with(true, false);+ rig.ready();+ *rig.desktop.elements.lock().unwrap() = send_button();+ let result = rig.call("click", json!({"x": 10, "y": 10}));+ assert!(is_error(&result));+ assert!(first_text(&result).contains("declined"), "{result}");+ assert!(rig.desktop.events().is_empty());+ }++ #[test]+ fn harmless_actions_are_not_confirmed() {+ let rig = rig();+ rig.ready();+ rig.call("click", json!({"x": 10, "y": 10}));+ rig.call("key", json!({"combo": "cmd+t"}));+ assert!(rig.dialogs.confirmations.lock().unwrap().is_empty());+ assert_eq!(rig.desktop.events().len(), 2);+ }++ #[test]+ fn return_in_a_send_like_field_is_confirmed() {+ let rig = rig_with(true, false);+ rig.ready();+ let composer = AxElement { role: "AXTextArea".into(), description: Some("Message".into()), ..AxElement::default() };+ *rig.desktop.focus.lock().unwrap() = Some(Focus { element: composer, ..Focus::default() });+ assert!(is_error(&rig.call("key", json!({"combo": "cmd+return"}))));+ assert!(rig.desktop.events().is_empty());+ }++ #[test]+ fn a_risky_intent_adds_a_confirmation() {+ let rig = rig();+ rig.ready();+ rig.call("scroll", json!({"x": 1, "y": 1, "dx": 0, "dy": 10, "intent": "buy the ticket"}));+ assert_eq!(rig.dialogs.confirmations.lock().unwrap().len(), 1);+ }++ #[test]+ fn the_previous_app_is_brought_back_after_a_dialog() {+ let rig = rig();+ rig.ready();+ *rig.desktop.elements.lock().unwrap() = send_button();+ let frontmost = rig.desktop.clone();+ *rig.dialogs.during.lock().unwrap() = Some(Box::new(move || {+ *frontmost.frontmost.lock().unwrap() = Some(app(OWN_PID, "dev.gentleman.gentle-dot", "Gentle Dot"));+ }));+ rig.call("click", json!({"x": 10, "y": 10}));+ assert_eq!(*rig.desktop.activated.lock().unwrap(), vec![200]);+ }++ // --- Typing ---++ #[test]+ fn typing_sends_newlines_as_return_and_checks_them() {+ let rig = rig();+ rig.ready();+ let result = rig.call("type", json!({"text": "hi\nthere"}));+ assert!(!is_error(&result), "{result}");+ assert_eq!(+ rig.desktop.events(),+ vec![+ InputEvent::Text("hi".into()),+ InputEvent::Key(parse_combo("return").unwrap()),+ InputEvent::Text("there".into()),+ ]+ );+ // In a send-like field the newline needs a confirmation.+ let rig = rig_with(true, false);+ rig.ready();+ let composer = AxElement { role: "AXTextField".into(), title: Some("Reply".into()), ..AxElement::default() };+ *rig.desktop.focus.lock().unwrap() = Some(Focus { element: composer, ..Focus::default() });+ assert!(is_error(&rig.call("type", json!({"text": "ok\n"}))));+ assert!(rig.desktop.events().is_empty());+ }++ #[test]+ fn long_text_is_sent_in_chunks_of_twenty_utf16_units() {+ let rig = rig();+ rig.ready();+ let text = "ñ".repeat(25) + "😀";+ rig.call("type", json!({"text": text}));+ let chunks: Vec<String> = rig+ .desktop+ .events()+ .into_iter()+ .map(|e| match e {+ InputEvent::Text(t) => t,+ other => panic!("{other:?}"),+ })+ .collect();+ assert_eq!(chunks.concat(), text);+ assert!(chunks.iter().all(|c| c.encode_utf16().count() <= 20), "{chunks:?}");+ }++ #[test]+ fn list_apps_returns_names_and_bundle_ids() {+ let rig = rig();+ let result = rig.call("list_apps", json!({}));+ let apps: Value = serde_json::from_str(&first_text(&result)).unwrap();+ assert_eq!(apps[0], json!({"name": "Safari", "bundleId": "com.apple.Safari", "active": true}));+ }+}diff --git a/apps/desktop/src-tauri/src/computer/mod.rs b/apps/desktop/src-tauri/src/computer/mod.rsnew file mode 100644index 0000000..ee36b2a--- /dev/null+++ b/apps/desktop/src-tauri/src/computer/mod.rs@@ -0,0 +1,133 @@+//! Computer control (S24): a native helper inside the app that serves MCP on loopback so the+//! agent can see and drive the Mac. Every rule (session grant, blocklist, rate limit, risky+//! action confirmation, panic stop) lives here, outside the agent's reach.+//!+//! The rules are pure modules tested on every OS. `Desktop` is the seam to the real Mac+//! (`macos`), and `fake` records events for the tests. Off macOS the helper does not start.++pub mod app;+pub mod blocklist;+pub mod control;+pub mod coords;+pub mod encode;+#[cfg(test)]+mod fake;+pub mod keys;+#[cfg(target_os = "macos")]+mod macos;+pub mod mcp;+pub mod rate;+pub mod risk;+pub mod server;+pub mod session;++use blocklist::AppIdentity;+use coords::{Point, ScreenshotGeometry};+use keys::KeyCombo;+use risk::{AxElement, Focus};+use serde::{Deserialize, Serialize};++#[cfg(target_os = "macos")]+pub use macos::{request_permission, MacDesktop};+++#[derive(Debug, Clone, Copy, PartialEq, Eq)]+pub enum MouseButton {+ Left,+ Right,+}++/// One input the helper posts. Points are global, in points.+#[derive(Debug, Clone, PartialEq)]+pub enum InputEvent {+ Move(Point),+ Click { at: Point, button: MouseButton, count: u8 },+ Drag { from: Point, to: Point },+ /// `dy > 0` scrolls down and `dx > 0` scrolls right, in pixels.+ Scroll { at: Point, dx: i32, dy: i32 },+ Key(KeyCombo),+ /// A chunk of text without newlines (newlines are sent as Return).+ Text(String),+}++/// An encoded screenshot of the display under the pointer, Gentle Dot's windows excluded.+#[derive(Debug, Clone)]+pub struct Capture {+ pub data: Vec<u8>,+ pub mime_type: &'static str,+ pub geometry: ScreenshotGeometry,+}++#[derive(Debug, Clone, PartialEq, Eq, Serialize)]+#[serde(rename_all = "camelCase")]+pub struct AppInfo {+ pub name: String,+ pub bundle_id: Option<String>,+ pub active: bool,+}++/// The `computer_permissions` reply (L60), plus `screenshotsSupported` (false before macOS 14,+/// where ScreenCaptureKit cannot take screenshots).+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]+#[serde(rename_all = "camelCase")]+pub struct Permissions {+ pub accessibility: bool,+ pub screen_recording: bool,+ pub screenshots_supported: bool,+}++/// Which macOS permission the user is asked for (`computer_request_permission {kind}`, L60).+#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize)]+#[serde(rename_all = "camelCase")]+pub enum PermissionKind {+ Accessibility,+ ScreenRecording,+}++/// The Mac, as the helper sees it.+pub trait Desktop: Send + Sync {+ fn permissions(&self) -> Permissions;+ /// A screenshot of the display under the pointer, downscaled to `coords::MAX_LONG_EDGE`.+ fn capture(&self) -> Result<Capture, String>;+ fn post(&self, event: &InputEvent) -> Result<(), String>;+ fn list_apps(&self) -> Result<Vec<AppInfo>, String>;+ /// Opens (or activates) an app by name or bundle id.+ fn open_app(&self, name: &str) -> Result<(), String>;+ /// The Accessibility element at a point, then its ancestors; empty when unknown.+ fn element_at(&self, at: Point) -> Vec<AxElement>;+ fn focused_element(&self) -> Option<Focus>;+ fn frontmost_app(&self) -> Option<AppIdentity>;+ /// The app owning the topmost window under a point.+ fn window_owner_at(&self, at: Point) -> Option<AppIdentity>;+ /// Brings an app back to the front (after a dialog took the focus).+ fn activate(&self, pid: i32);+}++/// Native dialogs owned by the app, never the webview (S24.3, S24.4).+pub trait Dialogs: Send + Sync {+ /// "Allow Gentle Dot to see and control this Mac for 30 minutes?" (Allow / Deny).+ fn ask_grant(&self) -> bool;+ /// A per-action confirmation showing the action and its target.+ fn confirm(&self, message: &str) -> bool;+}++pub trait Clock: Send + Sync {+ /// Unix epoch milliseconds.+ fn now_ms(&self) -> u64;+ fn sleep_ms(&self, ms: u64);+}++pub struct SystemClock;++impl Clock for SystemClock {+ fn now_ms(&self) -> u64 {+ std::time::SystemTime::now()+ .duration_since(std::time::UNIX_EPOCH)+ .map(|d| d.as_millis() as u64)+ .unwrap_or_default()+ }++ fn sleep_ms(&self, ms: u64) {+ std::thread::sleep(std::time::Duration::from_millis(ms));+ }+}diff --git a/apps/desktop/src-tauri/src/computer/rate.rs b/apps/desktop/src-tauri/src/computer/rate.rsnew file mode 100644index 0000000..d43dd9a--- /dev/null+++ b/apps/desktop/src-tauri/src/computer/rate.rs@@ -0,0 +1,65 @@+//! The action rate limit (S24.4): at most `max` tool calls in any sliding window.++use std::collections::VecDeque;++pub const MAX_ACTIONS_PER_SECOND: usize = 10;++pub struct RateLimiter {+ max: usize,+ window_ms: u64,+ accepted: VecDeque<u64>,+}++impl RateLimiter {+ pub fn new(max: usize, window_ms: u64) -> Self {+ RateLimiter { max, window_ms, accepted: VecDeque::new() }+ }++ /// Counts a call at `now_ms` and returns true, or returns false (not counted) when+ /// `max` calls were already accepted within the last `window_ms`.+ pub fn try_acquire(&mut self, now_ms: u64) -> bool {+ while self.accepted.front().is_some_and(|&t| t + self.window_ms <= now_ms) {+ self.accepted.pop_front();+ }+ if self.accepted.len() >= self.max {+ return false;+ }+ self.accepted.push_back(now_ms);+ true+ }+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn ten_calls_within_a_second_are_accepted_and_the_eleventh_is_not() {+ let mut limiter = RateLimiter::new(MAX_ACTIONS_PER_SECOND, 1000);+ for i in 0..10 {+ assert!(limiter.try_acquire(1_000 + i * 50), "call {i}");+ }+ assert!(!limiter.try_acquire(1_999));+ }++ #[test]+ fn calls_are_accepted_again_once_the_window_slides() {+ let mut limiter = RateLimiter::new(2, 1000);+ assert!(limiter.try_acquire(0));+ assert!(limiter.try_acquire(500));+ assert!(!limiter.try_acquire(999));+ assert!(limiter.try_acquire(1000));+ assert!(!limiter.try_acquire(1499));+ assert!(limiter.try_acquire(1500));+ }++ #[test]+ fn rejected_calls_do_not_count() {+ let mut limiter = RateLimiter::new(1, 1000);+ assert!(limiter.try_acquire(0));+ for t in [100, 200, 900] {+ assert!(!limiter.try_acquire(t));+ }+ assert!(limiter.try_acquire(1000));+ }+}diff --git a/apps/desktop/src-tauri/src/computer/risk.rs b/apps/desktop/src-tauri/src/computer/risk.rsnew file mode 100644index 0000000..049d8a6--- /dev/null+++ b/apps/desktop/src-tauri/src/computer/risk.rs@@ -0,0 +1,237 @@+//! Which actions need a native per-action confirmation (S24.4). A declared intent only adds+//! checks: an action is risky when the target is risky or the intent is.++/// Words that make an action risky, in English and Spanish. `place order` is a phrase.+const RISKY: &[&str] = &[+ "send", "pay", "buy", "purchase", "delete", "remove", "submit", "confirm", "transfer", "place order",+ "checkout", "enviar", "pagar", "comprar", "eliminar", "borrar", "confirmar", "transferir",+];++/// Extra words that make a text field send-like (a chat or comment composer), where+/// Return usually sends.+const SEND_LIKE_FIELD: &[&str] = &[+ "message", "messages", "reply", "comment", "chat", "post", "mensaje", "mensajes", "responder", "respuesta",+ "comentario", "comentar", "publicar",+];++/// Roles a click can press: buttons, menu items, and links (web pages often render buttons as links).+const PRESSABLE_ROLES: &[&str] = &["AXButton", "AXMenuButton", "AXMenuItem", "AXMenuBarItem", "AXLink"];+const TEXT_INPUT_ROLES: &[&str] = &["AXTextField", "AXTextArea", "AXComboBox"];++/// What the Accessibility API reports about an element.+#[derive(Debug, Clone, Default, PartialEq, Eq)]+pub struct AxElement {+ pub role: String,+ pub title: Option<String>,+ pub description: Option<String>,+ pub value: Option<String>,+}++/// The focused element and its window's default button, if any.+#[derive(Debug, Clone, Default, PartialEq, Eq)]+pub struct Focus {+ pub element: AxElement,+ pub is_default_button: bool,+ pub default_button: Option<AxElement>,+}++/// The facts the classifier needs about an action.+#[derive(Debug, Clone, Copy)]+pub enum Act<'a> {+ /// A click: the element under the point first, then its ancestors.+ Click(&'a [AxElement]),+ /// Return (or Enter), from `key` or a newline in `type`.+ Return(Option<&'a Focus>),+ /// Every other action: only the intent can make it risky.+ Other,+}++fn words(text: &str) -> Vec<String> {+ text.split(|c: char| !c.is_alphanumeric()).filter(|w| !w.is_empty()).map(str::to_lowercase).collect()+}++/// The first entry of `list` found in `text` as whole words, in order, case-insensitively.+fn find_in(text: &str, list: &[&'static str]) -> Option<&'static str> {+ let words = words(text);+ list.iter().copied().find(|entry| {+ let entry: Vec<&str> = entry.split(' ').collect();+ words.windows(entry.len()).any(|window| window.iter().zip(&entry).all(|(a, b)| a == b))+ })+}++/// The first risky word or phrase in `text`, matched on whole words, case-insensitively.+pub fn risky_word(text: &str) -> Option<&'static str> {+ find_in(text, RISKY)+}++impl AxElement {+ /// The element's title, description, and value, joined for matching and messages.+ pub fn label(&self) -> String {+ [&self.title, &self.description, &self.value]+ .into_iter()+ .flatten()+ .map(|s| s.trim())+ .filter(|s| !s.is_empty())+ .collect::<Vec<_>>()+ .join(" ")+ }++ fn is_pressable(&self) -> bool {+ PRESSABLE_ROLES.contains(&self.role.as_str())+ }+}++fn kind(role: &str) -> &str {+ match role {+ "AXMenuItem" | "AXMenuBarItem" => "menu item",+ "AXLink" => "link",+ "AXTextField" | "AXTextArea" | "AXComboBox" => "field",+ _ => "button",+ }+}++/// A pressable element in `chain` (the element under the point or an ancestor) whose own+/// text, or the text of the elements inside it, is risky.+fn risky_click(chain: &[AxElement]) -> Option<String> {+ let pressable = chain.iter().position(AxElement::is_pressable)?;+ let label = chain[..=pressable].iter().map(AxElement::label).collect::<Vec<_>>().join(" ");+ risky_word(&label).map(|_| format!("the {} “{}”", kind(&chain[pressable].role), label.trim()))+}++fn risky_return(focus: &Focus) -> Option<String> {+ let element = &focus.element;+ let label = element.label();+ if focus.is_default_button {+ return Some(format!("Return presses the default button “{label}”"));+ }+ if element.is_pressable() && risky_word(&label).is_some() {+ return Some(format!("Return presses the {} “{label}”", kind(&element.role)));+ }+ let field = TEXT_INPUT_ROLES.contains(&element.role.as_str());+ if field && (risky_word(&label).is_some() || find_in(&label, SEND_LIKE_FIELD).is_some()) {+ return Some(format!("Return may send the field “{label}”"));+ }+ let default = focus.default_button.as_ref().map(AxElement::label)?;+ risky_word(&default).map(|_| format!("Return may press the default button “{default}”"))+}++/// Why `act` needs a confirmation, or `None` when it does not.+pub fn assess(act: Act<'_>, intent: Option<&str>) -> Option<String> {+ let target = match act {+ Act::Click(chain) => risky_click(chain),+ Act::Return(focus) => focus.and_then(risky_return),+ Act::Other => None,+ };+ let intent = intent.filter(|i| risky_word(i).is_some()).map(|i| format!("the declared intent “{}”", i.trim()));+ match (target, intent) {+ (Some(target), Some(intent)) => Some(format!("{target}; {intent}")),+ (target, intent) => target.or(intent),+ }+}++#[cfg(test)]+mod tests {+ use super::*;++ fn element(role: &str, title: &str) -> AxElement {+ AxElement { role: role.into(), title: Some(title.into()), ..AxElement::default() }+ }++ fn focus(element: AxElement) -> Focus {+ Focus { element, ..Focus::default() }+ }++ #[test]+ fn english_and_spanish_words_match_case_insensitively() {+ assert_eq!(risky_word("SEND now"), Some("send"));+ assert_eq!(risky_word("Pagar ahora"), Some("pagar"));+ assert_eq!(risky_word("Eliminar conversación"), Some("eliminar"));+ assert_eq!(risky_word("Proceed to Checkout"), Some("checkout"));+ }++ #[test]+ fn words_match_whole_words_and_the_phrase_in_order() {+ assert_eq!(risky_word("Place Order"), Some("place order"));+ assert_eq!(risky_word("order a place"), None);+ assert_eq!(risky_word("Sender"), None);+ assert_eq!(risky_word("Paypal"), None);+ assert_eq!(risky_word("send-button"), Some("send"));+ assert_eq!(risky_word(""), None);+ }++ #[test]+ fn clicking_a_risky_button_or_menu_item_is_risky() {+ assert!(assess(Act::Click(&[element("AXButton", "Send")]), None).is_some());+ assert!(assess(Act::Click(&[element("AXMenuItem", "Delete Message")]), None).is_some());+ let described = AxElement { role: "AXButton".into(), description: Some("Comprar".into()), ..AxElement::default() };+ assert!(assess(Act::Click(&[described]), None).is_some());+ }++ #[test]+ fn text_inside_a_risky_button_counts() {+ let label = AxElement { role: "AXStaticText".into(), value: Some("Buy now".into()), ..AxElement::default() };+ let chain = [label, AxElement { role: "AXButton".into(), ..AxElement::default() }];+ assert!(assess(Act::Click(&chain), None).is_some());+ }++ #[test]+ fn harmless_clicks_are_not_risky() {+ assert_eq!(assess(Act::Click(&[element("AXButton", "Open")]), None), None);+ // A risky word outside a button (plain text, a text field) is not a press.+ assert_eq!(assess(Act::Click(&[element("AXStaticText", "Send")]), None), None);+ assert_eq!(assess(Act::Click(&[element("AXTextField", "Delete")]), None), None);+ assert_eq!(assess(Act::Click(&[]), None), None);+ }++ #[test]+ fn return_on_a_default_button_is_risky() {+ let default = Focus { element: element("AXButton", "OK"), is_default_button: true, default_button: None };+ assert!(assess(Act::Return(Some(&default)), None).is_some());+ }++ #[test]+ fn return_on_a_focused_risky_button_is_risky() {+ assert!(assess(Act::Return(Some(&focus(element("AXButton", "Transfer")))), None).is_some());+ }++ #[test]+ fn return_in_a_send_like_field_is_risky() {+ let composer = AxElement { role: "AXTextArea".into(), description: Some("Message #general".into()), ..AxElement::default() };+ assert!(assess(Act::Return(Some(&focus(composer))), None).is_some());+ let spanish = AxElement { role: "AXTextField".into(), title: Some("Escribe un mensaje".into()), ..AxElement::default() };+ assert!(assess(Act::Return(Some(&focus(spanish))), None).is_some());+ }++ #[test]+ fn return_with_a_risky_default_button_in_the_window_is_risky() {+ let field = Focus {+ element: element("AXTextField", "Amount"),+ is_default_button: false,+ default_button: Some(element("AXButton", "Pay")),+ };+ assert!(assess(Act::Return(Some(&field)), None).is_some());+ }++ #[test]+ fn return_in_an_ordinary_field_is_not_risky() {+ let search = Focus {+ element: element("AXTextField", "Search"),+ is_default_button: false,+ default_button: Some(element("AXButton", "Done")),+ };+ assert_eq!(assess(Act::Return(Some(&search)), None), None);+ assert_eq!(assess(Act::Return(None), None), None);+ }++ #[test]+ fn a_risky_intent_makes_any_action_risky() {+ assert!(assess(Act::Other, Some("borrar el archivo")).is_some());+ assert!(assess(Act::Click(&[element("AXButton", "Open")]), Some("purchase the plan")).is_some());+ }++ #[test]+ fn a_harmless_intent_never_removes_a_check() {+ assert_eq!(assess(Act::Other, Some("open the menu")), None);+ assert!(assess(Act::Click(&[element("AXButton", "Send")]), Some("just looking")).is_some());+ }+}diff --git a/apps/desktop/src-tauri/src/computer/server.rs b/apps/desktop/src-tauri/src/computer/server.rsnew file mode 100644index 0000000..048ab02--- /dev/null+++ b/apps/desktop/src-tauri/src/computer/server.rs@@ -0,0 +1,125 @@+//! Serves `Helper` on `127.0.0.1:<random port>/mcp` with a fresh random token per launch+//! (S24.1). The token lives only in memory; the webview gets it from `computer_endpoint`.++use super::mcp::{Helper, HttpRequest};+use serde::Serialize;+use std::io::Read;+use std::sync::Arc;+use std::thread;++/// The `computer_endpoint` reply (L60).+#[derive(Debug, Clone, PartialEq, Eq, Serialize)]+pub struct Endpoint {+ pub url: String,+ pub token: String,+}++/// 32 random bytes, hex encoded.+pub fn new_token() -> String {+ let mut bytes = [0u8; 32];+ getrandom::fill(&mut bytes).expect("the OS random source is unavailable");+ bytes.iter().map(|b| format!("{b:02x}")).collect()+}++/// Binds a random loopback port and serves each request on its own thread (actions are+/// serialized inside `Helper`).+pub fn start(helper: Arc<Helper>, token: String) -> std::io::Result<Endpoint> {+ let server = tiny_http::Server::http("127.0.0.1:0").map_err(std::io::Error::other)?;+ let port = server.server_addr().to_ip().map(|a| a.port()).ok_or_else(|| std::io::Error::other("no port"))?;+ thread::Builder::new().name("computer-mcp".into()).spawn(move || {+ for request in server.incoming_requests() {+ let helper = helper.clone();+ thread::spawn(move || serve(&helper, request));+ }+ })?;+ Ok(Endpoint { url: format!("http://127.0.0.1:{port}/mcp"), token })+}++/// Requests larger than this are refused: tool calls are small.+const MAX_BODY: u64 = 1 << 20;++fn serve(helper: &Helper, mut request: tiny_http::Request) {+ let authorization = request+ .headers()+ .iter()+ .find(|h| h.field.equiv("Authorization"))+ .map(|h| h.value.as_str().to_string());+ let mut body = String::new();+ let read = request.as_reader().take(MAX_BODY).read_to_string(&mut body);+ let method = request.method().as_str().to_string();+ let path = request.url().split('?').next().unwrap_or_default().to_string();+ let reply = match read {+ Ok(_) => helper.handle(&HttpRequest {+ method: &method,+ path: &path,+ authorization: authorization.as_deref(),+ body: &body,+ }),+ Err(_) => super::mcp::HttpReply { status: 400, body: None },+ };+ let is_json = reply.body.is_some();+ let text = reply.body.map(|b| b.to_string()).unwrap_or_default();+ let mut response = tiny_http::Response::from_string(text).with_status_code(reply.status);+ if is_json {+ response.add_header(tiny_http::Header::from_bytes("Content-Type", "application/json").expect("valid header"));+ }+ if reply.status == 401 {+ response.add_header(tiny_http::Header::from_bytes("WWW-Authenticate", "Bearer").expect("valid header"));+ }+ let _ = request.respond(response);+}++#[cfg(test)]+mod tests {+ use super::super::control::Control;+ use super::super::fake::{FakeClock, FakeDesktop, FakeDialogs};+ use super::*;+ use std::io::Write;+ use std::net::TcpStream;++ #[test]+ fn tokens_are_long_random_and_fresh() {+ let (a, b) = (new_token(), new_token());+ assert_eq!(a.len(), 64);+ assert!(a.chars().all(|c| c.is_ascii_hexdigit()));+ assert_ne!(a, b);+ }++ fn exchange(url: &str, request: &str) -> String {+ let address = url.trim_start_matches("http://").trim_end_matches("/mcp");+ let mut stream = TcpStream::connect(address).unwrap();+ stream.write_all(request.as_bytes()).unwrap();+ let mut reply = String::new();+ stream.read_to_string(&mut reply).unwrap();+ reply+ }++ #[test]+ fn the_loopback_server_checks_the_token_and_answers_json() {+ let token = new_token();+ let helper = Helper::new(+ Arc::new(FakeDesktop::default()),+ Arc::new(FakeDialogs::new(false, false)),+ Arc::new(FakeClock::new(0)),+ Arc::new(Control::new(|_| {})),+ token.clone(),+ 1,+ );+ let endpoint = start(Arc::new(helper), token.clone()).unwrap();+ assert!(endpoint.url.starts_with("http://127.0.0.1:") && endpoint.url.ends_with("/mcp"));++ let body = r#"{"jsonrpc":"2.0","id":1,"method":"tools/list"}"#;+ let request = |auth: &str| {+ format!(+ "POST /mcp HTTP/1.1\r\nHost: 127.0.0.1\r\n{auth}Content-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}",+ body.len()+ )+ };+ let denied = exchange(&endpoint.url, &request(""));+ assert!(denied.starts_with("HTTP/1.1 401"), "{denied}");+ let allowed = exchange(&endpoint.url, &request(&format!("Authorization: Bearer {token}\r\n")));+ assert!(allowed.starts_with("HTTP/1.1 200"), "{allowed}");+ assert!(allowed.to_ascii_lowercase().contains("content-type: application/json"), "{allowed}");+ assert!(allowed.contains("\"screenshot\""), "{allowed}");+ }+}diff --git a/apps/desktop/src-tauri/src/computer/session.rs b/apps/desktop/src-tauri/src/computer/session.rsnew file mode 100644index 0000000..be3952b--- /dev/null+++ b/apps/desktop/src-tauri/src/computer/session.rs@@ -0,0 +1,138 @@+//! The computer-control session (S24.3): idle until the user allows it, then granted until+//! `ends_at`, ended by the Stop button, the tray item, the panic shortcut, or the timeout.+//! Every end bumps `epoch`, so actions queued under an earlier epoch are dropped.++use serde::Serialize;++pub const SESSION_MS: u64 = 30 * 60 * 1000;++#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]+#[serde(rename_all = "lowercase")]+pub enum Reason {+ Granted,+ Stopped,+ Panic,+ Timeout,+}++/// The `computer://state` event and the `computer_status` reply (L60). `ends_at` is in+/// Unix epoch milliseconds.+#[derive(Debug, Clone, PartialEq, Eq, Serialize)]+#[serde(rename_all = "camelCase")]+pub struct StateEvent {+ pub active: bool,+ #[serde(skip_serializing_if = "Option::is_none")]+ pub ends_at: Option<u64>,+ #[serde(skip_serializing_if = "Option::is_none")]+ pub reason: Option<Reason>,+}++#[derive(Debug, Default)]+pub struct Session {+ ends_at: Option<u64>,+ epoch: u64,+}++impl Session {+ pub fn is_active(&self, now_ms: u64) -> bool {+ self.ends_at.is_some_and(|end| now_ms < end)+ }++ pub fn epoch(&self) -> u64 {+ self.epoch+ }++ pub fn status(&self, now_ms: u64) -> StateEvent {+ let active = self.is_active(now_ms);+ StateEvent { active, ends_at: self.ends_at.filter(|_| active), reason: None }+ }++ /// Starts (or restarts) a session of `SESSION_MS`.+ pub fn grant(&mut self, now_ms: u64) -> StateEvent {+ let ends_at = now_ms + SESSION_MS;+ self.ends_at = Some(ends_at);+ StateEvent { active: true, ends_at: Some(ends_at), reason: Some(Reason::Granted) }+ }++ /// Ends the session for `reason`. `None` when there was none to end.+ pub fn end(&mut self, reason: Reason) -> Option<StateEvent> {+ self.ends_at.take()?;+ self.epoch += 1;+ Some(StateEvent { active: false, ends_at: None, reason: Some(reason) })+ }++ /// Ends the session with `Timeout` once `now_ms` reaches its end.+ pub fn expire(&mut self, now_ms: u64) -> Option<StateEvent> {+ if self.ends_at.is_some_and(|end| now_ms >= end) {+ self.end(Reason::Timeout)+ } else {+ None+ }+ }+}++#[cfg(test)]+mod tests {+ use super::*;++ const NOW: u64 = 1_700_000_000_000;++ #[test]+ fn a_new_session_is_idle() {+ let session = Session::default();+ assert!(!session.is_active(NOW));+ assert_eq!(session.status(NOW), StateEvent { active: false, ends_at: None, reason: None });+ }++ #[test]+ fn granting_starts_thirty_minutes() {+ let mut session = Session::default();+ let event = session.grant(NOW);+ assert_eq!(event, StateEvent { active: true, ends_at: Some(NOW + SESSION_MS), reason: Some(Reason::Granted) });+ assert!(session.is_active(NOW + SESSION_MS - 1));+ assert_eq!(session.status(NOW + 5).ends_at, Some(NOW + SESSION_MS));+ }++ #[test]+ fn stop_and_panic_end_the_session_and_bump_the_epoch() {+ for reason in [Reason::Stopped, Reason::Panic] {+ let mut session = Session::default();+ session.grant(NOW);+ let epoch = session.epoch();+ assert_eq!(session.end(reason), Some(StateEvent { active: false, ends_at: None, reason: Some(reason) }));+ assert!(!session.is_active(NOW));+ assert_eq!(session.epoch(), epoch + 1);+ }+ }++ #[test]+ fn ending_an_idle_session_reports_nothing() {+ let mut session = Session::default();+ assert_eq!(session.end(Reason::Stopped), None);+ assert_eq!(session.epoch(), 0);+ }++ #[test]+ fn the_session_times_out_at_its_end() {+ let mut session = Session::default();+ session.grant(NOW);+ assert_eq!(session.expire(NOW + SESSION_MS - 1), None);+ assert!(!session.is_active(NOW + SESSION_MS));+ assert_eq!(+ session.expire(NOW + SESSION_MS),+ Some(StateEvent { active: false, ends_at: None, reason: Some(Reason::Timeout) })+ );+ assert_eq!(session.epoch(), 1);+ assert_eq!(session.expire(NOW + SESSION_MS + 1), None);+ }++ #[test]+ fn the_event_serializes_like_the_l60_contract() {+ let mut session = Session::default();+ let granted = serde_json::to_value(session.grant(NOW)).unwrap();+ assert_eq!(granted, serde_json::json!({"active": true, "endsAt": NOW + SESSION_MS, "reason": "granted"}));+ let panic = serde_json::to_value(session.end(Reason::Panic).unwrap()).unwrap();+ assert_eq!(panic, serde_json::json!({"active": false, "reason": "panic"}));+ assert_eq!(serde_json::to_value(session.status(NOW)).unwrap(), serde_json::json!({"active": false}));+ }+}diff --git a/apps/desktop/src-tauri/src/lib.rs b/apps/desktop/src-tauri/src/lib.rsindex 677e03c..37e517c 100644--- a/apps/desktop/src-tauri/src/lib.rs+++ b/apps/desktop/src-tauri/src/lib.rs@@ -1,3 +1,4 @@+pub mod computer;pub mod config;pub mod daemon;pub mod geometry;diff --git a/apps/desktop/src-tauri/src/shell.rs b/apps/desktop/src-tauri/src/shell.rsindex d9f7477..be80bb1 100644--- a/apps/desktop/src-tauri/src/shell.rs+++ b/apps/desktop/src-tauri/src/shell.rs@@ -1,6 +1,8 @@//! Tauri wiring: windows, commands, tray menu, global shortcut, Dot snapping,//! and the daemon lifecycle (design §9).+use crate::computer::app as computer;+use crate::computer::session::Reason;use crate::config::{self, ConnectionInfo, DesktopConfig, DEFAULT_SHORTCUT};use crate::daemon::{Daemon, RestartOutcome};use crate::geometry::{self, Rect};@@ -258,12 +260,17 @@ fn conversations_on(value: Option<&str>) -> bool {/// The tray menu, top to bottom (`-` is a separator). "New conversation" only exists with the/// conversations list on; in the single continuous chat there is nothing new to start.-fn tray_menu_ids(conversations: bool) -> Vec<&'static str> {+/// "Stop computer control" exists where computer control does (macOS, S24.3).+fn tray_menu_ids(conversations: bool, computer_control: bool) -> Vec<&'static str> {let mut ids = vec!["open"];if conversations {ids.push("new");}- ids.extend(["browser", "-", "restart", "autostart", "-", "quit"]);+ ids.push("browser");+ if computer_control {+ ids.push("computer-stop");+ }+ ids.extend(["-", "restart", "autostart", "-", "quit"]);ids}@@ -274,11 +281,18 @@ fn build_tray(app: &AppHandle, config: &DesktopConfig) -> tauri::Result<TrayIcon.or_else(|_| MenuItem::with_id(app, "open", "Open", true, None::<&str>))?;let menu = Menu::new(app)?;let conversations = conversations_on(std::env::var("GENTLE_DOT_CONVERSATIONS").ok().as_deref());- for id in tray_menu_ids(conversations) {+ for id in tray_menu_ids(conversations, cfg!(target_os = "macos")) {match id {"open" => menu.append(&open)?,"new" => menu.append(&MenuItem::with_id(app, "new", "New conversation", true, None::<&str>)?)?,"browser" => menu.append(&MenuItem::with_id(app, "browser", "Open in browser", true, None::<&str>)?)?,+ "computer-stop" => menu.append(&MenuItem::with_id(+ app,+ "computer-stop",+ "Stop computer control",+ true,+ None::<&str>,+ )?)?,"restart" => {menu.append(&MenuItem::with_id(app, "restart", "Restart assistant", true, None::<&str>)?)?;}@@ -305,6 +319,10 @@ fn build_tray(app: &AppHandle, config: &DesktopConfig) -> tauri::Result<TrayIcon"open" => show_panel(app),"new" => show_panel(app).and_then(|()| app.emit("dot://new-conversation", ()).map_err(err)),"browser" => open_in_browser(app),+ "computer-stop" => {+ computer::stop(app, Reason::Stopped);+ Ok(())+ }"restart" => {restart_daemon(app.clone());Ok(())@@ -360,6 +378,19 @@ fn restart_daemon(app: AppHandle) {});}+/// ⌥⇧Esc ends computer control from anywhere, even while another app is in front.+#[cfg(target_os = "macos")]+fn register_panic_shortcut(app: &AppHandle) {+ let handler = |app: &AppHandle, _: &_, event: tauri_plugin_global_shortcut::ShortcutEvent| {+ if event.state() == ShortcutState::Pressed {+ computer::stop(app, Reason::Panic);+ }+ };+ if let Err(error) = app.global_shortcut().on_shortcut(computer::PANIC_SHORTCUT, handler) {+ eprintln!("gentle-dot: the computer-control panic shortcut is unavailable: {error}");+ }+}+fn register_shortcut(app: &AppHandle, shortcut: &str) {let handler = |app: &AppHandle, _: &_, event: tauri_plugin_global_shortcut::ShortcutEvent| {if event.state() == ShortcutState::Pressed {@@ -403,7 +434,17 @@ pub fn run() {.plugin(tauri_plugin_autostart::init(MacosLauncher::LaunchAgent, None)).plugin(tauri_plugin_opener::init()).plugin(tauri_plugin_dialog::init())- .invoke_handler(tauri::generate_handler![connection_info, toggle_panel, hide_panel, set_dot_state])+ .invoke_handler(tauri::generate_handler![+ connection_info,+ toggle_panel,+ hide_panel,+ set_dot_state,+ computer::computer_endpoint,+ computer::computer_permissions,+ computer::computer_request_permission,+ computer::computer_stop,+ computer::computer_status+ ]).setup(move |app| {#[cfg(target_os = "macos")]app.set_activation_policy(tauri::ActivationPolicy::Accessory);@@ -417,6 +458,9 @@ pub fn run() {}});app.manage(Shell { config: config.clone(), daemon, snap: spawn_snapper(handle.clone()), places_windows });+ app.manage(computer::start(&handle));+ #[cfg(target_os = "macos")]+ register_panic_shortcut(&handle);build_windows(&handle, &config, places_windows)?;build_tray(&handle, &config)?;@@ -457,7 +501,7 @@ mod tests {#[test]fn tray_hides_new_conversation_in_the_single_chat() {assert_eq!(- tray_menu_ids(false),+ tray_menu_ids(false, false),vec!["open", "browser", "-", "restart", "autostart", "-", "quit"]);}@@ -465,11 +509,19 @@ mod tests {#[test]fn tray_shows_new_conversation_with_the_conversations_list() {assert_eq!(- tray_menu_ids(true),+ tray_menu_ids(true, false),vec!["open", "new", "browser", "-", "restart", "autostart", "-", "quit"]);}+ #[test]+ fn tray_offers_stop_computer_control_where_it_exists() {+ assert_eq!(+ tray_menu_ids(false, true),+ vec!["open", "browser", "computer-stop", "-", "restart", "autostart", "-", "quit"]+ );+ }+#[test]fn conversations_flag_reads_like_the_daemon() {assert!(conversations_on(Some("1")));diff --git a/apps/desktop/src-tauri/tauri.conf.json b/apps/desktop/src-tauri/tauri.conf.jsonindex 36589f3..f2cc106 100644--- a/apps/desktop/src-tauri/tauri.conf.json+++ b/apps/desktop/src-tauri/tauri.conf.json@@ -28,7 +28,7 @@"icons/icon.png"],"macOS": {- "minimumSystemVersion": "11.0"+ "minimumSystemVersion": "12.3"}}}