Skip to content

A personal assistant that lives on your desktop, remembers what matters, and can use your Mac for you.

TypeScript

feat(desktop): add the native macOS computer-control helper

Alan Buscaglia176f547c4530
diff --git a/apps/desktop/src-tauri/Cargo.lock b/apps/desktop/src-tauri/Cargo.lock
index 822455d..ccf1e16 100644
--- a/apps/desktop/src-tauri/Cargo.lock
+++ b/apps/desktop/src-tauri/Cargo.lock
@@ -47,6 +47,12 @@ version = "1.0.104"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470"
+[[package]]
+name = "ascii"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d92bec98840b8f03a5ff5413de5293bfcd8bf96467cf5452609f939ec6f5de16"
+
[[package]]
name = "async-broadcast"
version = "0.7.2"
@@ -498,6 +504,12 @@ dependencies = [
"windows-link",
]
+[[package]]
+name = "chunked_transfer"
+version = "1.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6e4de3bc4ea267985becf712dc6d9eed8b04c953b3fcfb339ebc87acd9804901"
+
[[package]]
name = "combine"
version = "4.6.8"
@@ -1263,7 +1275,18 @@ dependencies = [
name = "gentle-dot"
version = "0.1.0"
dependencies = [
+ "base64 0.22.1",
+ "block2",
+ "getrandom 0.3.4",
+ "image",
"libc",
+ "objc2",
+ "objc2-app-kit",
+ "objc2-application-services",
+ "objc2-core-foundation",
+ "objc2-core-graphics",
+ "objc2-foundation",
+ "objc2-screen-capture-kit",
"serde",
"serde_json",
"tauri",
@@ -1273,6 +1296,7 @@ dependencies = [
"tauri-plugin-global-shortcut",
"tauri-plugin-opener",
"tauri-plugin-single-instance",
+ "tiny_http",
]
[[package]]
@@ -1559,6 +1583,12 @@ version = "1.10.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87"
+[[package]]
+name = "httpdate"
+version = "1.0.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9"
+
[[package]]
name = "hyper"
version = "1.12.0"
@@ -1757,6 +1787,8 @@ dependencies = [
"moxcms",
"num-traits",
"png 0.18.1",
+ "zune-core",
+ "zune-jpeg",
]
[[package]]
@@ -2279,12 +2311,23 @@ checksum = "d49e936b501e5c5bf01fda3a9452ff86dc3ea98ad5f283e1455153142d97518c"
dependencies = [
"bitflags 2.13.2",
"block2",
+ "libc",
"objc2",
"objc2-core-foundation",
"objc2-foundation",
"objc2-quartz-core",
]
+[[package]]
+name = "objc2-application-services"
+version = "0.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "69282c2b5bc58fba07cb9de2113619532eb551e98efe3d8d695509ef45fbd53b"
+dependencies = [
+ "bitflags 2.13.2",
+ "objc2-core-foundation",
+]
+
[[package]]
name = "objc2-cloud-kit"
version = "0.3.2"
@@ -2413,6 +2456,20 @@ dependencies = [
"objc2-foundation",
]
+[[package]]
+name = "objc2-screen-capture-kit"
+version = "0.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "74b7c5390f477482f001bc354d6571a70db7e4f8d5288e860c45521fbce11394"
+dependencies = [
+ "block2",
+ "libc",
+ "objc2",
+ "objc2-core-foundation",
+ "objc2-core-graphics",
+ "objc2-foundation",
+]
+
[[package]]
name = "objc2-security"
version = "0.3.2"
@@ -3994,6 +4051,18 @@ dependencies = [
"time-core",
]
+[[package]]
+name = "tiny_http"
+version = "0.12.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "389915df6413a2e74fb181895f933386023c71110878cd0825588928e64cdc82"
+dependencies = [
+ "ascii",
+ "chunked_transfer",
+ "httpdate",
+ "log",
+]
+
[[package]]
name = "tinystr"
version = "0.8.4"
@@ -5246,6 +5315,21 @@ version = "1.0.23"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b"
+[[package]]
+name = "zune-core"
+version = "0.5.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d56377fd46368984a170bc5aac5567e52ca5da874caa60bea39fcbca78fb658b"
+
+[[package]]
+name = "zune-jpeg"
+version = "0.5.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "27bc9d5b815bc103f142aa054f561d9187d191692ec7c2d1e2b4737f8dbd7296"
+dependencies = [
+ "zune-core",
+]
+
[[package]]
name = "zvariant"
version = "5.15.0"
diff --git a/apps/desktop/src-tauri/Cargo.toml b/apps/desktop/src-tauri/Cargo.toml
index 62e2d86..3222dc7 100644
--- a/apps/desktop/src-tauri/Cargo.toml
+++ b/apps/desktop/src-tauri/Cargo.toml
@@ -22,3 +22,18 @@ tauri-plugin-single-instance = "2"
serde = { version = "1", features = ["derive"] }
serde_json = "1"
libc = "0.2"
+getrandom = "0.3"
+base64 = "0.22"
+tiny_http = "0.12"
+image = { version = "0.25", default-features = false, features = ["jpeg"] }
+
+# The computer-control helper (S24) is macOS only.
+[target.'cfg(target_os = "macos")'.dependencies]
+objc2 = "0.6"
+block2 = "0.6"
+objc2-foundation = { version = "0.3", default-features = false, features = ["std", "NSArray", "NSDictionary", "NSError", "NSString", "NSValue"] }
+objc2-core-foundation = { version = "0.3", default-features = false, features = ["std", "CFArray", "CFCGTypes", "CFData", "CFDictionary", "CFNumber", "CFString"] }
+objc2-core-graphics = { version = "0.3", default-features = false, features = ["std", "CGDataProvider", "CGDirectDisplay", "CGError", "CGEvent", "CGEventTypes", "CGImage", "CGRemoteOperation", "CGWindow"] }
+objc2-app-kit = { version = "0.3", default-features = false, features = ["std", "libc", "NSRunningApplication", "NSWorkspace"] }
+objc2-application-services = { version = "0.3", default-features = false, features = ["std", "AXError", "AXUIElement", "HIServices"] }
+objc2-screen-capture-kit = { version = "0.3", default-features = false, features = ["std", "block2", "libc", "objc2-core-foundation", "objc2-core-graphics", "SCScreenshotManager", "SCShareableContent", "SCStream"] }
diff --git a/apps/desktop/src-tauri/build.rs b/apps/desktop/src-tauri/build.rs
index 93015e8..aba1be9 100644
--- a/apps/desktop/src-tauri/build.rs
+++ b/apps/desktop/src-tauri/build.rs
@@ -2,7 +2,17 @@ use std::env;
use std::path::{Path, PathBuf};
/// Commands the UI may invoke; each one gets an `allow-<command>` permission.
-const COMMANDS: &[&str] = &["connection_info", "toggle_panel", "hide_panel", "set_dot_state"];
+const COMMANDS: &[&str] = &[
+ "connection_info",
+ "toggle_panel",
+ "hide_panel",
+ "set_dot_state",
+ "computer_endpoint",
+ "computer_permissions",
+ "computer_request_permission",
+ "computer_stop",
+ "computer_status",
+];
fn find_in_path(program: &str, path: &str) -> Option<PathBuf> {
env::split_paths(path)
diff --git a/apps/desktop/src-tauri/capabilities/default.json b/apps/desktop/src-tauri/capabilities/default.json
index d3d5053..4f141f3 100644
--- a/apps/desktop/src-tauri/capabilities/default.json
+++ b/apps/desktop/src-tauri/capabilities/default.json
@@ -1,7 +1,7 @@
{
"$schema": "../gen/schemas/desktop-schema.json",
"identifier": "default",
- "description": "What the Dot and panel UI may call: event listening, Dot dragging, opening web pages (sign-in), and the four shell commands.",
+ "description": "What the Dot and panel UI may call: event listening, Dot dragging, opening web pages (sign-in), the four shell commands, and the computer-control commands.",
"windows": ["dot", "panel"],
"permissions": [
"core:event:allow-listen",
@@ -11,6 +11,11 @@
"allow-connection-info",
"allow-toggle-panel",
"allow-hide-panel",
- "allow-set-dot-state"
+ "allow-set-dot-state",
+ "allow-computer-endpoint",
+ "allow-computer-permissions",
+ "allow-computer-request-permission",
+ "allow-computer-stop",
+ "allow-computer-status"
]
}
diff --git a/apps/desktop/src-tauri/permissions/autogenerated/computer_endpoint.toml b/apps/desktop/src-tauri/permissions/autogenerated/computer_endpoint.toml
new file mode 100644
index 0000000..e88dcd1
--- /dev/null
+++ b/apps/desktop/src-tauri/permissions/autogenerated/computer_endpoint.toml
@@ -0,0 +1,11 @@
+# Automatically generated - DO NOT EDIT!
+
+[[permission]]
+identifier = "allow-computer-endpoint"
+description = "Enables the computer_endpoint command without any pre-configured scope."
+commands.allow = ["computer_endpoint"]
+
+[[permission]]
+identifier = "deny-computer-endpoint"
+description = "Denies the computer_endpoint command without any pre-configured scope."
+commands.deny = ["computer_endpoint"]
diff --git a/apps/desktop/src-tauri/permissions/autogenerated/computer_permissions.toml b/apps/desktop/src-tauri/permissions/autogenerated/computer_permissions.toml
new file mode 100644
index 0000000..86d6ba1
--- /dev/null
+++ b/apps/desktop/src-tauri/permissions/autogenerated/computer_permissions.toml
@@ -0,0 +1,11 @@
+# Automatically generated - DO NOT EDIT!
+
+[[permission]]
+identifier = "allow-computer-permissions"
+description = "Enables the computer_permissions command without any pre-configured scope."
+commands.allow = ["computer_permissions"]
+
+[[permission]]
+identifier = "deny-computer-permissions"
+description = "Denies the computer_permissions command without any pre-configured scope."
+commands.deny = ["computer_permissions"]
diff --git a/apps/desktop/src-tauri/permissions/autogenerated/computer_request_permission.toml b/apps/desktop/src-tauri/permissions/autogenerated/computer_request_permission.toml
new file mode 100644
index 0000000..6811c27
--- /dev/null
+++ b/apps/desktop/src-tauri/permissions/autogenerated/computer_request_permission.toml
@@ -0,0 +1,11 @@
+# Automatically generated - DO NOT EDIT!
+
+[[permission]]
+identifier = "allow-computer-request-permission"
+description = "Enables the computer_request_permission command without any pre-configured scope."
+commands.allow = ["computer_request_permission"]
+
+[[permission]]
+identifier = "deny-computer-request-permission"
+description = "Denies the computer_request_permission command without any pre-configured scope."
+commands.deny = ["computer_request_permission"]
diff --git a/apps/desktop/src-tauri/permissions/autogenerated/computer_status.toml b/apps/desktop/src-tauri/permissions/autogenerated/computer_status.toml
new file mode 100644
index 0000000..83f2379
--- /dev/null
+++ b/apps/desktop/src-tauri/permissions/autogenerated/computer_status.toml
@@ -0,0 +1,11 @@
+# Automatically generated - DO NOT EDIT!
+
+[[permission]]
+identifier = "allow-computer-status"
+description = "Enables the computer_status command without any pre-configured scope."
+commands.allow = ["computer_status"]
+
+[[permission]]
+identifier = "deny-computer-status"
+description = "Denies the computer_status command without any pre-configured scope."
+commands.deny = ["computer_status"]
diff --git a/apps/desktop/src-tauri/permissions/autogenerated/computer_stop.toml b/apps/desktop/src-tauri/permissions/autogenerated/computer_stop.toml
new file mode 100644
index 0000000..2bcbf41
--- /dev/null
+++ b/apps/desktop/src-tauri/permissions/autogenerated/computer_stop.toml
@@ -0,0 +1,11 @@
+# Automatically generated - DO NOT EDIT!
+
+[[permission]]
+identifier = "allow-computer-stop"
+description = "Enables the computer_stop command without any pre-configured scope."
+commands.allow = ["computer_stop"]
+
+[[permission]]
+identifier = "deny-computer-stop"
+description = "Denies the computer_stop command without any pre-configured scope."
+commands.deny = ["computer_stop"]
diff --git a/apps/desktop/src-tauri/src/computer/app.rs b/apps/desktop/src-tauri/src/computer/app.rs
new file mode 100644
index 0000000..c7855cc
--- /dev/null
+++ b/apps/desktop/src-tauri/src/computer/app.rs
@@ -0,0 +1,182 @@
+//! The app side of computer control (L60): the `computer_*` commands, the `computer://state`
+//! event, the panic shortcut, the timeout watcher, and the native dialogs. Off macOS the
+//! commands answer "unavailable" and nothing starts.
+
+use super::control::Control;
+use super::server::Endpoint;
+use super::session::{Reason, StateEvent};
+use super::{Clock, PermissionKind, Permissions, SystemClock};
+use std::sync::Arc;
+use tauri::{AppHandle, Emitter, Manager, State};
+
+pub const STATE_EVENT: &str = "computer://state";
+/// ⌥⇧Esc ends the session from anywhere (S24.3).
+pub const PANIC_SHORTCUT: &str = "Alt+Shift+Escape";
+pub const GRANT_TEXT: &str = "Allow Gentle Dot to see and control this Mac for 30 minutes?";
+pub const GRANT_BUTTONS: (&str, &str) = ("Allow", "Deny");
+pub const CONFIRM_BUTTONS: (&str, &str) = ("Allow", "Cancel");
+
+pub struct Computer {
+ control: Arc<Control>,
+ endpoint: Option<Endpoint>,
+ #[cfg(target_os = "macos")]
+ desktop: Arc<dyn super::Desktop>,
+}
+
+/// Creates the session and, on macOS, starts the MCP helper and the timeout watcher.
+pub fn start(app: &AppHandle) -> Computer {
+ let events = app.clone();
+ let control = Arc::new(Control::new(move |event: &StateEvent| {
+ if let Err(error) = events.emit(STATE_EVENT, event) {
+ eprintln!("gentle-dot: cannot report the computer-control state: {error}");
+ }
+ }));
+ #[cfg(target_os = "macos")]
+ {
+ let desktop: Arc<dyn super::Desktop> = Arc::new(super::MacDesktop::default());
+ let endpoint = start_helper(app, control.clone(), desktop.clone())
+ .map_err(|error| eprintln!("gentle-dot: computer control is unavailable: {error}"))
+ .ok();
+ watch_timeout(control.clone());
+ Computer { control, endpoint, desktop }
+ }
+ #[cfg(not(target_os = "macos"))]
+ Computer { control, endpoint: None }
+}
+
+#[cfg(target_os = "macos")]
+fn start_helper(
+ app: &AppHandle,
+ control: Arc<Control>,
+ desktop: Arc<dyn super::Desktop>,
+) -> std::io::Result<Endpoint> {
+ let token = super::server::new_token();
+ let helper = super::mcp::Helper::new(
+ desktop,
+ Arc::new(NativeDialogs { app: app.clone() }),
+ Arc::new(SystemClock),
+ control,
+ token.clone(),
+ std::process::id() as i32,
+ );
+ super::server::start(Arc::new(helper), token)
+}
+
+/// Reports the timeout when it happens, not only at the next tool call.
+#[cfg(target_os = "macos")]
+fn watch_timeout(control: Arc<Control>) {
+ std::thread::spawn(move || loop {
+ std::thread::sleep(std::time::Duration::from_secs(1));
+ control.expire(SystemClock.now_ms());
+ });
+}
+
+/// Ends the session (Stop button, tray item, panic shortcut). Queued actions are dropped.
+pub fn stop(app: &AppHandle, reason: Reason) {
+ if let Some(computer) = app.try_state::<Computer>() {
+ computer.control.end(reason);
+ }
+}
+
+#[tauri::command]
+pub fn computer_endpoint(computer: State<'_, Computer>) -> Option<Endpoint> {
+ computer.endpoint.clone()
+}
+
+#[tauri::command]
+pub fn computer_permissions(computer: State<'_, Computer>) -> Permissions {
+ #[cfg(target_os = "macos")]
+ return computer.desktop.permissions();
+ #[cfg(not(target_os = "macos"))]
+ {
+ let _ = computer;
+ Permissions { accessibility: false, screen_recording: false, screenshots_supported: false }
+ }
+}
+
+#[tauri::command]
+pub fn computer_request_permission(kind: PermissionKind) -> Result<(), String> {
+ #[cfg(target_os = "macos")]
+ return super::request_permission(kind);
+ #[cfg(not(target_os = "macos"))]
+ {
+ let _ = kind;
+ Err("Computer control is only available on macOS.".into())
+ }
+}
+
+#[tauri::command]
+pub fn computer_stop(app: AppHandle) {
+ stop(&app, Reason::Stopped);
+}
+
+#[tauri::command]
+pub fn computer_status(computer: State<'_, Computer>) -> StateEvent {
+ computer.control.status(SystemClock.now_ms())
+}
+
+/// Dialogs shown by the app itself (NSAlert through the dialog plugin), never the webview.
+#[cfg(target_os = "macos")]
+struct NativeDialogs {
+ app: AppHandle,
+}
+
+#[cfg(target_os = "macos")]
+impl NativeDialogs {
+ fn ask(&self, message: &str, (allow, refuse): (&str, &str)) -> bool {
+ use tauri_plugin_dialog::{DialogExt, MessageDialogButtons, MessageDialogKind};
+ self.app
+ .dialog()
+ .message(message)
+ .title("Gentle Dot")
+ .kind(MessageDialogKind::Warning)
+ .buttons(MessageDialogButtons::OkCancelCustom(allow.into(), refuse.into()))
+ .blocking_show()
+ }
+}
+
+#[cfg(target_os = "macos")]
+impl super::Dialogs for NativeDialogs {
+ fn ask_grant(&self) -> bool {
+ self.ask(GRANT_TEXT, GRANT_BUTTONS)
+ }
+
+ fn confirm(&self, message: &str) -> bool {
+ self.ask(message, CONFIRM_BUTTONS)
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn the_grant_dialog_asks_exactly_what_the_spec_says() {
+ assert_eq!(GRANT_TEXT, "Allow Gentle Dot to see and control this Mac for 30 minutes?");
+ assert_eq!(GRANT_BUTTONS, ("Allow", "Deny"));
+ }
+
+ #[test]
+ fn the_panic_shortcut_is_option_shift_escape() {
+ let shortcut: tauri_plugin_global_shortcut::Shortcut = PANIC_SHORTCUT.parse().unwrap();
+ let expected = tauri_plugin_global_shortcut::Shortcut::new(
+ Some(tauri_plugin_global_shortcut::Modifiers::ALT | tauri_plugin_global_shortcut::Modifiers::SHIFT),
+ tauri_plugin_global_shortcut::Code::Escape,
+ );
+ assert_eq!(shortcut, expected);
+ }
+
+ #[test]
+ fn permission_kinds_read_the_l60_names() {
+ let kind: PermissionKind = serde_json::from_str("\"screenRecording\"").unwrap();
+ assert_eq!(kind, PermissionKind::ScreenRecording);
+ let kind: PermissionKind = serde_json::from_str("\"accessibility\"").unwrap();
+ assert_eq!(kind, PermissionKind::Accessibility);
+ }
+
+ #[test]
+ fn permissions_serialize_with_the_l60_field_names() {
+ let value = serde_json::to_value(Permissions { accessibility: true, screen_recording: false, screenshots_supported: true }).unwrap();
+ assert_eq!(value, serde_json::json!({"accessibility": true, "screenRecording": false, "screenshotsSupported": true}));
+ }
+}
diff --git a/apps/desktop/src-tauri/src/computer/blocklist.rs b/apps/desktop/src-tauri/src/computer/blocklist.rs
new file mode 100644
index 0000000..2c64e94
--- /dev/null
+++ b/apps/desktop/src-tauri/src/computer/blocklist.rs
@@ -0,0 +1,153 @@
+//! Apps no action may target (S24.4): Gentle Dot itself, System Settings, the login and
+//! authorization prompts, Keychain Access, and the known password managers.
+
+/// The app an action targets: the frontmost app for keyboard actions, the owner of the
+/// window under the point for pointer actions.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct AppIdentity {
+ pub pid: i32,
+ pub bundle_id: Option<String>,
+ pub name: String,
+}
+
+struct Entry {
+ label: &'static str,
+ /// A bundle id also blocks its helpers (`com.apple.Passwords` blocks `com.apple.Passwords.MenuBarExtra`).
+ bundles: &'static [&'static str],
+ names: &'static [&'static str],
+}
+
+pub const GENTLE_DOT_BUNDLE: &str = "dev.gentleman.gentle-dot";
+
+const BLOCKED: &[Entry] = &[
+ Entry { label: "Gentle Dot", bundles: &[GENTLE_DOT_BUNDLE], names: &["Gentle Dot"] },
+ Entry {
+ label: "System Settings",
+ bundles: &["com.apple.systempreferences"],
+ names: &["System Settings", "System Preferences"],
+ },
+ Entry { label: "SecurityAgent", bundles: &["com.apple.SecurityAgent"], names: &["SecurityAgent"] },
+ Entry { label: "loginwindow", bundles: &["com.apple.loginwindow"], names: &["loginwindow"] },
+ Entry { label: "Keychain Access", bundles: &["com.apple.keychainaccess"], names: &["Keychain Access"] },
+ Entry {
+ label: "1Password",
+ bundles: &["com.1password", "com.agilebits"],
+ names: &["1Password", "1Password 7", "1Password 8"],
+ },
+ Entry { label: "Bitwarden", bundles: &["com.bitwarden"], names: &["Bitwarden"] },
+ Entry { label: "Dashlane", bundles: &["com.dashlane"], names: &["Dashlane"] },
+ Entry { label: "LastPass", bundles: &["com.lastpass"], names: &["LastPass"] },
+ Entry { label: "Passwords", bundles: &["com.apple.Passwords"], names: &["Passwords"] },
+];
+
+fn bundle_matches(bundle: &str, blocked: &str) -> bool {
+ let (bundle, blocked) = (bundle.to_lowercase(), blocked.to_lowercase());
+ bundle == blocked || bundle.strip_prefix(&blocked).is_some_and(|rest| rest.starts_with('.'))
+}
+
+/// Why `app` may not be targeted, or `None` when it may. `own_pid` catches Gentle Dot
+/// when it runs unbundled (a development build has no bundle id).
+pub fn blocked_reason(app: &AppIdentity, own_pid: i32) -> Option<&'static str> {
+ if app.pid == own_pid {
+ return Some(BLOCKED[0].label);
+ }
+ match &app.bundle_id {
+ Some(bundle) => BLOCKED.iter().find(|e| e.bundles.iter().any(|b| bundle_matches(bundle, b))).map(|e| e.label),
+ None => blocked_name(&app.name),
+ }
+}
+
+/// The same check for `open_app`, which only has a name or a bundle id (`Keychain Access.app` too).
+pub fn blocked_name(name_or_bundle: &str) -> Option<&'static str> {
+ let name = name_or_bundle.trim().trim_end_matches('/');
+ let name = name.rsplit('/').next().unwrap_or(name);
+ let name = name.strip_suffix(".app").unwrap_or(name);
+ BLOCKED
+ .iter()
+ .find(|e| {
+ e.names.iter().any(|n| n.eq_ignore_ascii_case(name)) || e.bundles.iter().any(|b| bundle_matches(name, b))
+ })
+ .map(|e| e.label)
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ const OWN_PID: i32 = 4242;
+
+ fn app(bundle: Option<&str>, name: &str) -> AppIdentity {
+ AppIdentity { pid: 100, bundle_id: bundle.map(str::to_string), name: name.into() }
+ }
+
+ #[test]
+ fn every_listed_bundle_is_blocked() {
+ let cases = [
+ ("com.apple.systempreferences", "System Settings"),
+ ("com.apple.SecurityAgent", "SecurityAgent"),
+ ("com.apple.loginwindow", "loginwindow"),
+ ("com.apple.keychainaccess", "Keychain Access"),
+ ("com.1password.1password", "1Password"),
+ ("com.agilebits.onepassword7", "1Password"),
+ ("com.bitwarden.desktop", "Bitwarden"),
+ ("com.dashlane.dashlanephonefinal", "Dashlane"),
+ ("com.lastpass.LastPass", "LastPass"),
+ ("com.apple.Passwords", "Passwords"),
+ ("dev.gentleman.gentle-dot", "Gentle Dot"),
+ ];
+ for (bundle, label) in cases {
+ assert_eq!(blocked_reason(&app(Some(bundle), "x"), OWN_PID), Some(label), "{bundle}");
+ }
+ }
+
+ #[test]
+ fn helpers_of_a_blocked_bundle_are_blocked() {
+ assert_eq!(blocked_reason(&app(Some("com.apple.Passwords.MenuBarExtra"), "x"), OWN_PID), Some("Passwords"));
+ }
+
+ #[test]
+ fn bundle_ids_compare_case_insensitively() {
+ assert_eq!(blocked_reason(&app(Some("COM.APPLE.KEYCHAINACCESS"), "x"), OWN_PID), Some("Keychain Access"));
+ }
+
+ #[test]
+ fn similar_prefixes_are_not_blocked() {
+ assert_eq!(blocked_reason(&app(Some("com.apple.PasswordsX"), "x"), OWN_PID), None);
+ assert_eq!(blocked_reason(&app(Some("com.lastpassword.app"), "x"), OWN_PID), None);
+ }
+
+ #[test]
+ fn ordinary_apps_are_allowed() {
+ assert_eq!(blocked_reason(&app(Some("com.apple.Safari"), "Safari"), OWN_PID), None);
+ assert_eq!(blocked_reason(&app(None, "Some Tool"), OWN_PID), None);
+ }
+
+ #[test]
+ fn gentle_dot_is_blocked_by_its_own_pid() {
+ let unbundled = AppIdentity { pid: OWN_PID, bundle_id: None, name: "gentle-dot".into() };
+ assert_eq!(blocked_reason(&unbundled, OWN_PID), Some("Gentle Dot"));
+ }
+
+ #[test]
+ fn an_app_without_a_bundle_id_is_checked_by_name() {
+ assert_eq!(blocked_reason(&app(None, "SecurityAgent"), OWN_PID), Some("SecurityAgent"));
+ }
+
+ #[test]
+ fn open_app_names_and_bundle_ids_are_checked() {
+ assert_eq!(blocked_name("System Settings"), Some("System Settings"));
+ assert_eq!(blocked_name("system preferences"), Some("System Settings"));
+ assert_eq!(blocked_name("Keychain Access.app"), Some("Keychain Access"));
+ assert_eq!(blocked_name("com.bitwarden.desktop"), Some("Bitwarden"));
+ assert_eq!(blocked_name(" 1password "), Some("1Password"));
+ assert_eq!(blocked_name("Safari"), None);
+ assert_eq!(blocked_name("com.apple.Safari"), None);
+ }
+
+ #[test]
+ fn open_app_paths_are_checked_by_their_app_name() {
+ assert_eq!(blocked_name("/System/Applications/System Settings.app"), Some("System Settings"));
+ assert_eq!(blocked_name("/Applications/1Password.app/"), Some("1Password"));
+ assert_eq!(blocked_name("/Applications/Safari.app"), None);
+ }
+}
diff --git a/apps/desktop/src-tauri/src/computer/control.rs b/apps/desktop/src-tauri/src/computer/control.rs
new file mode 100644
index 0000000..d0ede33
--- /dev/null
+++ b/apps/desktop/src-tauri/src/computer/control.rs
@@ -0,0 +1,55 @@
+//! The session shared by the MCP server and the app (Stop button, tray item, panic shortcut,
+//! timeout watcher). Every transition is reported to `notify` (the `computer://state` event).
+
+use super::session::{Reason, Session, StateEvent};
+use std::sync::Mutex;
+
+type Notify = Box<dyn Fn(&StateEvent) + Send + Sync>;
+
+pub struct Control {
+ session: Mutex<Session>,
+ notify: Notify,
+}
+
+impl Control {
+ pub fn new(notify: impl Fn(&StateEvent) + Send + Sync + 'static) -> Self {
+ Control { session: Mutex::new(Session::default()), notify: Box::new(notify) }
+ }
+
+ fn with<T>(&self, f: impl FnOnce(&mut Session) -> T) -> T {
+ f(&mut self.session.lock().unwrap_or_else(|poisoned| poisoned.into_inner()))
+ }
+
+ fn report(&self, event: Option<StateEvent>) {
+ if let Some(event) = event {
+ (self.notify)(&event);
+ }
+ }
+
+ pub fn status(&self, now_ms: u64) -> StateEvent {
+ self.with(|s| s.status(now_ms))
+ }
+
+ pub fn is_active(&self, now_ms: u64) -> bool {
+ self.with(|s| s.is_active(now_ms))
+ }
+
+ pub fn epoch(&self) -> u64 {
+ self.with(|s| s.epoch())
+ }
+
+ pub fn grant(&self, now_ms: u64) {
+ let event = self.with(|s| s.grant(now_ms));
+ self.report(Some(event));
+ }
+
+ pub fn end(&self, reason: Reason) {
+ let event = self.with(|s| s.end(reason));
+ self.report(event);
+ }
+
+ pub fn expire(&self, now_ms: u64) {
+ let event = self.with(|s| s.expire(now_ms));
+ self.report(event);
+ }
+}
diff --git a/apps/desktop/src-tauri/src/computer/coords.rs b/apps/desktop/src-tauri/src/computer/coords.rs
new file mode 100644
index 0000000..dd51581
--- /dev/null
+++ b/apps/desktop/src-tauri/src/computer/coords.rs
@@ -0,0 +1,137 @@
+//! Screenshot geometry (S24.2): the size a capture is downscaled to, and how a pixel of the
+//! latest screenshot maps back to a global point (the unit macOS input events use).
+
+/// Long edge of every screenshot, in pixels.
+pub const MAX_LONG_EDGE: u32 = 1280;
+
+#[derive(Debug, Clone, Copy, PartialEq)]
+pub struct Point {
+ pub x: f64,
+ pub y: f64,
+}
+
+/// The size that fits `width × height` within a long edge of `max`, keeping the aspect
+/// ratio. Never upscales and never returns a zero side.
+pub fn fit_long_edge(width: u32, height: u32, max: u32) -> (u32, u32) {
+ let long = width.max(height).max(1);
+ if long <= max {
+ return (width.max(1), height.max(1));
+ }
+ let fit = |side: u32| ((u64::from(side) * u64::from(max) + u64::from(long) / 2) / u64::from(long)).max(1) as u32;
+ (fit(width), fit(height))
+}
+
+/// Where a screenshot came from: its size in pixels, the display's top-left corner in
+/// global points, and `scale`, the screenshot pixels per point.
+#[derive(Debug, Clone, Copy, PartialEq)]
+pub struct ScreenshotGeometry {
+ pub image_width: u32,
+ pub image_height: u32,
+ pub scale: f64,
+ pub origin: Point,
+}
+
+#[derive(Debug, Clone, PartialEq)]
+pub enum CoordError {
+ NotFinite,
+ OutOfBounds { width: u32, height: u32 },
+}
+
+impl std::fmt::Display for CoordError {
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ match self {
+ CoordError::NotFinite => write!(f, "coordinates must be numbers"),
+ CoordError::OutOfBounds { width, height } => {
+ write!(f, "coordinates are outside the latest screenshot ({width} × {height} pixels)")
+ }
+ }
+ }
+}
+
+impl ScreenshotGeometry {
+ /// The geometry of a display `width_pt` points wide captured at `image_width × image_height`.
+ pub fn for_display(origin: Point, width_pt: f64, image_width: u32, image_height: u32) -> Self {
+ ScreenshotGeometry { image_width, image_height, scale: f64::from(image_width) / width_pt, origin }
+ }
+
+ /// Maps a screenshot pixel to a global point, rounded to a whole point.
+ pub fn to_point(&self, x: f64, y: f64) -> Result<Point, CoordError> {
+ if !x.is_finite() || !y.is_finite() {
+ return Err(CoordError::NotFinite);
+ }
+ let (width, height) = (self.image_width, self.image_height);
+ if x < 0.0 || y < 0.0 || x >= f64::from(width) || y >= f64::from(height) {
+ return Err(CoordError::OutOfBounds { width, height });
+ }
+ Ok(Point { x: (self.origin.x + x / self.scale).round(), y: (self.origin.y + y / self.scale).round() })
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ fn retina_laptop() -> ScreenshotGeometry {
+ // A 1512 × 982 pt display captured at 1280 × 831 px.
+ ScreenshotGeometry::for_display(Point { x: 0.0, y: 0.0 }, 1512.0, 1280, 831)
+ }
+
+ #[test]
+ fn landscape_and_portrait_fit_the_long_edge() {
+ assert_eq!(fit_long_edge(3024, 1964, 1280), (1280, 831));
+ assert_eq!(fit_long_edge(1080, 1920, 1280), (720, 1280));
+ }
+
+ #[test]
+ fn small_captures_are_never_upscaled() {
+ assert_eq!(fit_long_edge(1024, 768, 1280), (1024, 768));
+ assert_eq!(fit_long_edge(1280, 1280, 1280), (1280, 1280));
+ }
+
+ #[test]
+ fn extreme_ratios_keep_at_least_one_pixel() {
+ assert_eq!(fit_long_edge(100_000, 10, 1280), (1280, 1));
+ assert_eq!(fit_long_edge(0, 0, 1280), (1, 1));
+ }
+
+ #[test]
+ fn scale_is_screenshot_pixels_per_point() {
+ let geometry = retina_laptop();
+ assert!((geometry.scale - 1280.0 / 1512.0).abs() < 1e-9);
+ assert_eq!((geometry.image_width, geometry.image_height), (1280, 831));
+ }
+
+ #[test]
+ fn pixels_map_to_points_with_the_scale() {
+ let geometry = ScreenshotGeometry::for_display(Point { x: 0.0, y: 0.0 }, 2560.0, 1280, 720);
+ assert_eq!(geometry.to_point(640.0, 360.0), Ok(Point { x: 1280.0, y: 720.0 }));
+ assert_eq!(geometry.to_point(0.0, 0.0), Ok(Point { x: 0.0, y: 0.0 }));
+ }
+
+ #[test]
+ fn the_display_origin_is_added() {
+ // A second display left of and above the main one.
+ let geometry = ScreenshotGeometry::for_display(Point { x: -1920.0, y: -300.0 }, 1920.0, 1280, 720);
+ assert_eq!(geometry.to_point(640.0, 360.0), Ok(Point { x: -960.0, y: 240.0 }));
+ }
+
+ #[test]
+ fn points_are_rounded_to_whole_points() {
+ // 100 px / (1280/1512) = 118.125 pt; 101 px = 119.30625 pt; 1279 px = 1510.81875 pt.
+ let geometry = retina_laptop();
+ assert_eq!(geometry.to_point(100.0, 101.0), Ok(Point { x: 118.0, y: 119.0 }));
+ assert_eq!(geometry.to_point(1279.0, 0.5), Ok(Point { x: 1511.0, y: 1.0 }));
+ }
+
+ #[test]
+ fn pixels_outside_the_screenshot_are_rejected() {
+ let geometry = retina_laptop();
+ let outside = Err(CoordError::OutOfBounds { width: 1280, height: 831 });
+ assert_eq!(geometry.to_point(1280.0, 10.0), outside);
+ assert_eq!(geometry.to_point(10.0, 831.0), outside);
+ assert_eq!(geometry.to_point(-1.0, 10.0), outside);
+ assert_eq!(geometry.to_point(10.0, -0.5), outside);
+ assert_eq!(geometry.to_point(f64::NAN, 10.0), Err(CoordError::NotFinite));
+ assert_eq!(geometry.to_point(10.0, f64::INFINITY), Err(CoordError::NotFinite));
+ }
+}
diff --git a/apps/desktop/src-tauri/src/computer/encode.rs b/apps/desktop/src-tauri/src/computer/encode.rs
new file mode 100644
index 0000000..3e22061
--- /dev/null
+++ b/apps/desktop/src-tauri/src/computer/encode.rs
@@ -0,0 +1,66 @@
+//! Screenshot encoding: ScreenCaptureKit hands back BGRA rows; the model gets a JPEG.
+
+pub const JPEG_QUALITY: u8 = 80;
+
+/// Encodes a BGRA bitmap whose rows are `bytes_per_row` long (rows may be padded) as JPEG.
+pub fn bgra_to_jpeg(
+ bgra: &[u8],
+ width: u32,
+ height: u32,
+ bytes_per_row: usize,
+ quality: u8,
+) -> Result<Vec<u8>, String> {
+ let (w, h) = (width as usize, height as usize);
+ if w == 0 || h == 0 || bytes_per_row < w * 4 || bgra.len() < bytes_per_row * (h - 1) + w * 4 {
+ let size = bgra.len();
+ return Err(format!("unexpected bitmap: {width} × {height}, {bytes_per_row} bytes per row, {size} bytes"));
+ }
+ let mut rgb = Vec::with_capacity(w * h * 3);
+ for row in bgra.chunks(bytes_per_row).take(h) {
+ for i in (0..w * 4).step_by(4) {
+ rgb.extend_from_slice(&[row[i + 2], row[i + 1], row[i]]);
+ }
+ }
+ let mut jpeg = Vec::new();
+ image::codecs::jpeg::JpegEncoder::new_with_quality(&mut jpeg, quality)
+ .encode(&rgb, width, height, image::ExtendedColorType::Rgb8)
+ .map_err(|e| format!("cannot encode the screenshot: {e}"))?;
+ Ok(jpeg)
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ fn decode(jpeg: &[u8]) -> image::RgbImage {
+ image::load_from_memory_with_format(jpeg, image::ImageFormat::Jpeg).unwrap().to_rgb8()
+ }
+
+ #[test]
+ fn bgra_becomes_a_jpeg_of_the_same_size_and_colors() {
+ // 8 × 8 pure red in BGRA.
+ let bgra: Vec<u8> = [0u8, 0, 255, 255].repeat(64);
+ let jpeg = bgra_to_jpeg(&bgra, 8, 8, 32, 90).unwrap();
+ assert_eq!(&jpeg[..2], &[0xFF, 0xD8]);
+ let image = decode(&jpeg);
+ assert_eq!(image.dimensions(), (8, 8));
+ let [r, g, b] = image.get_pixel(4, 4).0;
+ assert!(r > 240 && g < 20 && b < 20, "{r} {g} {b}");
+ }
+
+ #[test]
+ fn padded_rows_are_skipped() {
+ // 8 × 8 blue, each row padded with 16 bytes of white.
+ let row: Vec<u8> = [[255u8, 0, 0, 255].repeat(8), vec![255; 16]].concat();
+ let jpeg = bgra_to_jpeg(&row.repeat(8), 8, 8, 48, 90).unwrap();
+ let [r, g, b] = decode(&jpeg).get_pixel(7, 7).0;
+ assert!(b > 240 && r < 20 && g < 20, "{r} {g} {b}");
+ }
+
+ #[test]
+ fn short_or_empty_bitmaps_are_rejected() {
+ assert!(bgra_to_jpeg(&[0; 10], 2, 2, 8, 80).is_err());
+ assert!(bgra_to_jpeg(&[], 0, 0, 0, 80).is_err());
+ assert!(bgra_to_jpeg(&[0; 64], 4, 4, 8, 80).is_err());
+ }
+}
diff --git a/apps/desktop/src-tauri/src/computer/fake.rs b/apps/desktop/src-tauri/src/computer/fake.rs
new file mode 100644
index 0000000..5e887ff
--- /dev/null
+++ b/apps/desktop/src-tauri/src/computer/fake.rs
@@ -0,0 +1,174 @@
+//! Test doubles: a Mac that records events, scripted dialogs, and a clock that sleeps instantly.
+
+use super::blocklist::AppIdentity;
+use super::coords::{Point, ScreenshotGeometry};
+use super::risk::{AxElement, Focus};
+use super::{AppInfo, Capture, Clock, Desktop, Dialogs, InputEvent, Permissions};
+use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering};
+use std::sync::Mutex;
+
+pub fn app(pid: i32, bundle: &str, name: &str) -> AppIdentity {
+ AppIdentity { pid, bundle_id: Some(bundle.into()), name: name.into() }
+}
+
+type OwnerAt = Box<dyn Fn(Point) -> Option<AppIdentity> + Send + Sync>;
+
+pub struct FakeDesktop {
+ pub events: Mutex<Vec<InputEvent>>,
+ pub permissions: Mutex<Permissions>,
+ pub frontmost: Mutex<Option<AppIdentity>>,
+ pub owner_at: Mutex<OwnerAt>,
+ pub elements: Mutex<Vec<AxElement>>,
+ pub focus: Mutex<Option<Focus>>,
+ pub opened: Mutex<Vec<String>>,
+ pub activated: Mutex<Vec<i32>>,
+ pub captures: AtomicUsize,
+ /// A 2560 × 1600 pt display at the origin, captured at 1280 × 800 (scale 0.5).
+ pub geometry: ScreenshotGeometry,
+}
+
+impl Default for FakeDesktop {
+ fn default() -> Self {
+ let safari = app(200, "com.apple.Safari", "Safari");
+ let owner = safari.clone();
+ FakeDesktop {
+ events: Mutex::new(Vec::new()),
+ permissions: Mutex::new(Permissions { accessibility: true, screen_recording: true, screenshots_supported: true }),
+ frontmost: Mutex::new(Some(safari)),
+ owner_at: Mutex::new(Box::new(move |_| Some(owner.clone()))),
+ elements: Mutex::new(Vec::new()),
+ focus: Mutex::new(None),
+ opened: Mutex::new(Vec::new()),
+ activated: Mutex::new(Vec::new()),
+ captures: AtomicUsize::new(0),
+ geometry: ScreenshotGeometry::for_display(Point { x: 0.0, y: 0.0 }, 2560.0, 1280, 800),
+ }
+ }
+}
+
+impl FakeDesktop {
+ pub fn events(&self) -> Vec<InputEvent> {
+ self.events.lock().unwrap().clone()
+ }
+}
+
+impl Desktop for FakeDesktop {
+ fn permissions(&self) -> Permissions {
+ *self.permissions.lock().unwrap()
+ }
+
+ fn capture(&self) -> Result<Capture, String> {
+ self.captures.fetch_add(1, Ordering::SeqCst);
+ Ok(Capture { data: b"jpeg".to_vec(), mime_type: "image/jpeg", geometry: self.geometry })
+ }
+
+ fn post(&self, event: &InputEvent) -> Result<(), String> {
+ self.events.lock().unwrap().push(event.clone());
+ Ok(())
+ }
+
+ fn list_apps(&self) -> Result<Vec<AppInfo>, String> {
+ Ok(vec![
+ AppInfo { name: "Safari".into(), bundle_id: Some("com.apple.Safari".into()), active: true },
+ AppInfo { name: "Mail".into(), bundle_id: Some("com.apple.mail".into()), active: false },
+ ])
+ }
+
+ fn open_app(&self, name: &str) -> Result<(), String> {
+ self.opened.lock().unwrap().push(name.into());
+ Ok(())
+ }
+
+ fn element_at(&self, _at: Point) -> Vec<AxElement> {
+ self.elements.lock().unwrap().clone()
+ }
+
+ fn focused_element(&self) -> Option<Focus> {
+ self.focus.lock().unwrap().clone()
+ }
+
+ fn frontmost_app(&self) -> Option<AppIdentity> {
+ self.frontmost.lock().unwrap().clone()
+ }
+
+ fn window_owner_at(&self, at: Point) -> Option<AppIdentity> {
+ (self.owner_at.lock().unwrap())(at)
+ }
+
+ fn activate(&self, pid: i32) {
+ self.activated.lock().unwrap().push(pid);
+ }
+}
+
+type Hook = Box<dyn Fn() + Send + Sync>;
+
+/// Answers dialogs with fixed choices and records what was asked.
+pub struct FakeDialogs {
+ pub allow: Mutex<bool>,
+ pub confirm: Mutex<bool>,
+ pub grants: AtomicUsize,
+ pub confirmations: Mutex<Vec<String>>,
+ /// Runs while a dialog is open (for example, to move the focus to Gentle Dot).
+ pub during: Mutex<Option<Hook>>,
+}
+
+impl FakeDialogs {
+ pub fn new(allow: bool, confirm: bool) -> Self {
+ FakeDialogs {
+ allow: Mutex::new(allow),
+ confirm: Mutex::new(confirm),
+ grants: AtomicUsize::new(0),
+ confirmations: Mutex::new(Vec::new()),
+ during: Mutex::new(None),
+ }
+ }
+
+ fn run_hook(&self) {
+ if let Some(hook) = self.during.lock().unwrap().as_ref() {
+ hook();
+ }
+ }
+}
+
+impl Dialogs for FakeDialogs {
+ fn ask_grant(&self) -> bool {
+ self.grants.fetch_add(1, Ordering::SeqCst);
+ self.run_hook();
+ *self.allow.lock().unwrap()
+ }
+
+ fn confirm(&self, message: &str) -> bool {
+ self.confirmations.lock().unwrap().push(message.into());
+ self.run_hook();
+ *self.confirm.lock().unwrap()
+ }
+}
+
+/// Starts at a fixed time; `sleep_ms` advances it and runs `on_sleep`.
+pub struct FakeClock {
+ pub now: AtomicU64,
+ pub on_sleep: Mutex<Option<Hook>>,
+}
+
+impl FakeClock {
+ pub fn new(now: u64) -> Self {
+ FakeClock { now: AtomicU64::new(now), on_sleep: Mutex::new(None) }
+ }
+
+ pub fn advance(&self, ms: u64) {
+ self.now.fetch_add(ms, Ordering::SeqCst);
+ }
+}
+
+impl Clock for FakeClock {
+ fn now_ms(&self) -> u64 {
+ self.now.load(Ordering::SeqCst)
+ }
+
+ fn sleep_ms(&self, ms: u64) {
+ self.advance(ms);
+ if let Some(hook) = self.on_sleep.lock().unwrap().as_ref() {
+ hook();
+ }
+ }
+}
diff --git a/apps/desktop/src-tauri/src/computer/keys.rs b/apps/desktop/src-tauri/src/computer/keys.rs
new file mode 100644
index 0000000..8792e98
--- /dev/null
+++ b/apps/desktop/src-tauri/src/computer/keys.rs
@@ -0,0 +1,137 @@
+//! Key combos for the `key` tool (S24.2), such as `cmd+shift+t` or `return`, parsed into
+//! modifiers and one macOS virtual key code (ANSI layout).
+
+#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
+pub struct Modifiers {
+ pub cmd: bool,
+ pub shift: bool,
+ pub alt: bool,
+ pub ctrl: bool,
+ pub fn_key: bool,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub struct KeyCombo {
+ pub modifiers: Modifiers,
+ /// Canonical key name, for messages and the risk check.
+ pub key: &'static str,
+ /// macOS virtual key code (`kVK_*`).
+ pub code: u16,
+}
+
+impl KeyCombo {
+ /// Return (or Enter) with any modifiers: the key that submits forms and sends messages.
+ pub fn is_return(&self) -> bool {
+ self.key == "return"
+ }
+}
+
+/// Named keys and their virtual key codes. Single characters use the ANSI layout.
+const KEYS: &[(&str, u16)] = &[
+ ("a", 0), ("s", 1), ("d", 2), ("f", 3), ("h", 4), ("g", 5), ("z", 6), ("x", 7), ("c", 8), ("v", 9),
+ ("b", 11), ("q", 12), ("w", 13), ("e", 14), ("r", 15), ("y", 16), ("t", 17), ("1", 18), ("2", 19),
+ ("3", 20), ("4", 21), ("6", 22), ("5", 23), ("=", 24), ("9", 25), ("7", 26), ("-", 27), ("8", 28),
+ ("0", 29), ("]", 30), ("o", 31), ("u", 32), ("[", 33), ("i", 34), ("p", 35), ("return", 36), ("l", 37),
+ ("j", 38), ("'", 39), ("k", 40), (";", 41), ("\\", 42), (",", 43), ("/", 44), ("n", 45), ("m", 46),
+ (".", 47), ("tab", 48), ("space", 49), ("`", 50), ("delete", 51), ("escape", 53), ("f5", 96), ("f6", 97),
+ ("f7", 98), ("f3", 99), ("f8", 100), ("f9", 101), ("f11", 103), ("f10", 109), ("f12", 111), ("home", 115),
+ ("pageup", 116), ("forwarddelete", 117), ("f4", 118), ("end", 119), ("f2", 120), ("pagedown", 121),
+ ("f1", 122), ("left", 123), ("right", 124), ("down", 125), ("up", 126),
+];
+
+/// Other spellings of the named keys.
+const ALIASES: &[(&str, &str)] = &[
+ ("enter", "return"), ("esc", "escape"), ("backspace", "delete"), ("del", "forwarddelete"),
+ ("page_up", "pageup"), ("page_down", "pagedown"), ("arrowleft", "left"), ("arrowright", "right"),
+ ("arrowup", "up"), ("arrowdown", "down"), ("minus", "-"), ("equal", "="),
+ ("comma", ","), ("period", "."), ("slash", "/"), ("backslash", "\\"),
+];
+
+/// Parses `mod+mod+key`: case-insensitive, parts joined by `+`, any number of modifiers
+/// (`cmd`, `shift`, `alt`/`option`, `ctrl`/`control`, `fn`) and exactly one key.
+pub fn parse_combo(combo: &str) -> Result<KeyCombo, String> {
+ let mut modifiers = Modifiers::default();
+ let mut key = None;
+ for part in combo.split('+') {
+ let part = part.trim().to_lowercase();
+ let flag = match part.as_str() {
+ "cmd" | "command" => &mut modifiers.cmd,
+ "shift" => &mut modifiers.shift,
+ "alt" | "option" | "opt" => &mut modifiers.alt,
+ "ctrl" | "control" => &mut modifiers.ctrl,
+ "fn" => &mut modifiers.fn_key,
+ _ => {
+ if key.is_some() {
+ return Err(format!("`{combo}` names more than one key"));
+ }
+ let name = ALIASES.iter().find(|(alias, _)| *alias == part).map_or(part.as_str(), |(_, name)| name);
+ let found = KEYS.iter().find(|(known, _)| *known == name);
+ key = Some(*found.ok_or_else(|| format!("unknown key `{part}` in `{combo}`"))?);
+ continue;
+ }
+ };
+ *flag = true;
+ }
+ let (key, code) = key.ok_or_else(|| format!("`{combo}` names no key"))?;
+ Ok(KeyCombo { modifiers, key, code })
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ fn mods(cmd: bool, shift: bool, alt: bool, ctrl: bool) -> Modifiers {
+ Modifiers { cmd, shift, alt, ctrl, fn_key: false }
+ }
+
+ #[test]
+ fn a_plain_named_key() {
+ let combo = parse_combo("return").unwrap();
+ assert_eq!((combo.key, combo.code, combo.modifiers), ("return", 36, Modifiers::default()));
+ }
+
+ #[test]
+ fn modifiers_and_a_letter() {
+ let combo = parse_combo("cmd+shift+t").unwrap();
+ assert_eq!((combo.key, combo.code, combo.modifiers), ("t", 17, mods(true, true, false, false)));
+ }
+
+ #[test]
+ fn modifier_spellings_and_case_are_accepted() {
+ let combo = parse_combo(" Command + Option + Control + A ").unwrap();
+ assert_eq!((combo.key, combo.code, combo.modifiers), ("a", 0, mods(true, false, true, true)));
+ assert_eq!(parse_combo("opt+ctrl+left").unwrap().modifiers, mods(false, false, true, true));
+ assert!(parse_combo("fn+f1").unwrap().modifiers.fn_key);
+ }
+
+ #[test]
+ fn aliases_name_the_same_key() {
+ assert_eq!(parse_combo("enter").unwrap().code, 36);
+ assert_eq!(parse_combo("esc").unwrap().code, 53);
+ assert_eq!(parse_combo("backspace").unwrap().code, 51);
+ assert_eq!(parse_combo("cmd+minus").unwrap().code, 27);
+ }
+
+ #[test]
+ fn digits_punctuation_and_function_keys() {
+ assert_eq!(parse_combo("cmd+1").unwrap().code, 18);
+ assert_eq!(parse_combo("cmd+,").unwrap().code, 43);
+ assert_eq!(parse_combo("f12").unwrap().code, 111);
+ assert_eq!(parse_combo("space").unwrap().code, 49);
+ }
+
+ #[test]
+ fn invalid_combos_are_rejected() {
+ for combo in ["", "cmd+", "cmd", "cmd+shift", "cmd+a+b", "hyper+a", "f13", "cmd++a"] {
+ assert!(parse_combo(combo).is_err(), "{combo:?} should be rejected");
+ }
+ }
+
+ #[test]
+ fn return_is_recognized_with_any_modifiers() {
+ assert!(parse_combo("return").unwrap().is_return());
+ assert!(parse_combo("cmd+return").unwrap().is_return());
+ assert!(parse_combo("ctrl+enter").unwrap().is_return());
+ assert!(!parse_combo("tab").unwrap().is_return());
+ }
+}
diff --git a/apps/desktop/src-tauri/src/computer/macos.rs b/apps/desktop/src-tauri/src/computer/macos.rs
new file mode 100644
index 0000000..698871f
--- /dev/null
+++ b/apps/desktop/src-tauri/src/computer/macos.rs
@@ -0,0 +1,460 @@
+//! The real Mac behind `Desktop`: ScreenCaptureKit screenshots (macOS 14+), Quartz events,
+//! the Accessibility API, and NSWorkspace. Nothing here decides policy; `mcp` does.
+
+use super::blocklist::AppIdentity;
+use super::coords::{fit_long_edge, Point, ScreenshotGeometry, MAX_LONG_EDGE};
+use super::encode::{bgra_to_jpeg, JPEG_QUALITY};
+use super::keys::KeyCombo;
+use super::risk::{AxElement, Focus};
+use super::{AppInfo, Capture, Desktop, InputEvent, MouseButton, PermissionKind, Permissions};
+use block2::RcBlock;
+use objc2::rc::Retained;
+use objc2::{available, AnyThread};
+use objc2_app_kit::{NSApplicationActivationOptions, NSApplicationActivationPolicy, NSRunningApplication, NSWorkspace};
+use objc2_application_services::{AXError, AXIsProcessTrusted, AXIsProcessTrustedWithOptions, AXUIElement};
+use objc2_core_foundation::{CFBoolean, CFDictionary, CFRetained, CFString, CFType, CGPoint, CGRect, CGSize};
+use objc2_core_graphics::{
+ kCGNullWindowID, kCGWindowAlpha, kCGWindowBounds, kCGWindowOwnerName, kCGWindowOwnerPID, CGDataProvider, CGDirectDisplayID, CGError, CGEvent, CGEventField, CGEventFlags,
+ CGEventTapLocation, CGEventType, CGGetDisplaysWithPoint, CGImage, CGMouseButton,
+ CGPreflightScreenCaptureAccess, CGRequestScreenCaptureAccess, CGScrollEventUnit, CGWindowListCopyWindowInfo,
+ CGWindowListOption,
+};
+use objc2_foundation::{NSArray, NSDictionary, NSError, NSNumber, NSString};
+use objc2_screen_capture_kit::{
+ SCContentFilter, SCScreenshotManager, SCShareableContent, SCStreamConfiguration, SCWindow,
+};
+use std::ptr::NonNull;
+use std::sync::mpsc;
+use std::thread;
+use std::time::Duration;
+
+const CAPTURE_TIMEOUT: Duration = Duration::from_secs(5);
+/// Pause between the events of one action, so apps see distinct presses.
+const EVENT_GAP: Duration = Duration::from_millis(15);
+const DRAG_STEPS: u32 = 12;
+/// How many ancestors of the element under a point are read (text inside a button).
+const AX_ANCESTORS: usize = 4;
+/// `kCVPixelFormatType_32BGRA`.
+const PIXEL_FORMAT_BGRA: u32 = u32::from_be_bytes(*b"BGRA");
+
+/// Values crossing from a ScreenCaptureKit completion handler to the waiting thread.
+/// The objects are immutable snapshots, used by one thread at a time.
+struct Handoff<T>(T);
+unsafe impl<T> Send for Handoff<T> {}
+
+pub struct MacDesktop {
+ own_pid: i32,
+}
+
+impl Default for MacDesktop {
+ fn default() -> Self {
+ MacDesktop { own_pid: std::process::id() as i32 }
+ }
+}
+
+fn identity(app: &NSRunningApplication) -> AppIdentity {
+ AppIdentity {
+ pid: app.processIdentifier(),
+ bundle_id: app.bundleIdentifier().map(|b| b.to_string()),
+ name: app.localizedName().map(|n| n.to_string()).unwrap_or_default(),
+ }
+}
+
+fn post_event(event: Option<CFRetained<CGEvent>>) -> Result<(), String> {
+ let event = event.ok_or("macOS could not create the input event")?;
+ CGEvent::post(CGEventTapLocation::HIDEventTap, Some(&event));
+ thread::sleep(EVENT_GAP);
+ Ok(())
+}
+
+fn mouse(kind: CGEventType, at: Point, button: CGMouseButton) -> Option<CFRetained<CGEvent>> {
+ CGEvent::new_mouse_event(None, kind, CGPoint { x: at.x, y: at.y }, button)
+}
+
+/// The held modifiers of a combo: their key codes (`kVK_Command`, ...) and flags.
+fn modifiers(combo: &KeyCombo) -> Vec<(u16, CGEventFlags)> {
+ let m = combo.modifiers;
+ [
+ (m.cmd, 55, CGEventFlags::MaskCommand),
+ (m.shift, 56, CGEventFlags::MaskShift),
+ (m.alt, 58, CGEventFlags::MaskAlternate),
+ (m.ctrl, 59, CGEventFlags::MaskControl),
+ (m.fn_key, 63, CGEventFlags::MaskSecondaryFn),
+ ]
+ .into_iter()
+ .filter(|(on, ..)| *on)
+ .map(|(_, code, flag)| (code, flag))
+ .collect()
+}
+
+fn key_event(code: u16, down: bool, flags: CGEventFlags) -> Result<(), String> {
+ let event = CGEvent::new_keyboard_event(None, code, down);
+ if let Some(event) = &event {
+ CGEvent::set_flags(Some(event), flags);
+ }
+ post_event(event)
+}
+
+fn click(at: Point, button: MouseButton, count: u8) -> Result<(), String> {
+ let (down, up, cg_button) = match button {
+ MouseButton::Left => (CGEventType::LeftMouseDown, CGEventType::LeftMouseUp, CGMouseButton::Left),
+ MouseButton::Right => (CGEventType::RightMouseDown, CGEventType::RightMouseUp, CGMouseButton::Right),
+ };
+ post_event(mouse(CGEventType::MouseMoved, at, CGMouseButton::Left))?;
+ for state in 1..=i64::from(count) {
+ for kind in [down, up] {
+ let event = mouse(kind, at, cg_button);
+ if let Some(event) = &event {
+ CGEvent::set_integer_value_field(Some(event), CGEventField::MouseEventClickState, state);
+ }
+ post_event(event)?;
+ }
+ }
+ Ok(())
+}
+
+fn drag(from: Point, to: Point) -> Result<(), String> {
+ post_event(mouse(CGEventType::MouseMoved, from, CGMouseButton::Left))?;
+ post_event(mouse(CGEventType::LeftMouseDown, from, CGMouseButton::Left))?;
+ for step in 1..=DRAG_STEPS {
+ let t = f64::from(step) / f64::from(DRAG_STEPS);
+ let at = Point { x: from.x + (to.x - from.x) * t, y: from.y + (to.y - from.y) * t };
+ post_event(mouse(CGEventType::LeftMouseDragged, at, CGMouseButton::Left))?;
+ }
+ post_event(mouse(CGEventType::LeftMouseUp, to, CGMouseButton::Left))
+}
+
+fn scroll(at: Point, dx: i32, dy: i32) -> Result<(), String> {
+ post_event(mouse(CGEventType::MouseMoved, at, CGMouseButton::Left))?;
+ // Positive wheel values scroll up and left; the tool's dy > 0 scrolls down.
+ post_event(CGEvent::new_scroll_wheel_event2(None, CGScrollEventUnit::Pixel, 2, -dy, -dx, 0))
+}
+
+/// Presses the modifiers, then the key, then releases them in reverse, like a person does.
+fn key(combo: &KeyCombo) -> Result<(), String> {
+ let held = modifiers(combo);
+ let mut flags = CGEventFlags::empty();
+ for (code, flag) in &held {
+ flags |= *flag;
+ key_event(*code, true, flags)?;
+ }
+ key_event(combo.code, true, flags)?;
+ key_event(combo.code, false, flags)?;
+ for (code, flag) in held.iter().rev() {
+ flags &= !*flag;
+ key_event(*code, false, flags)?;
+ }
+ Ok(())
+}
+
+fn text(chunk: &str) -> Result<(), String> {
+ let units: Vec<u16> = chunk.encode_utf16().collect();
+ for down in [true, false] {
+ let event = CGEvent::new_keyboard_event(None, 0, down);
+ if let Some(event) = &event {
+ CGEvent::set_flags(Some(event), CGEventFlags::empty());
+ // SAFETY: `units` outlives the call and holds `units.len()` UTF-16 units.
+ unsafe { CGEvent::keyboard_set_unicode_string(Some(event), units.len() as _, units.as_ptr()) };
+ }
+ post_event(event)?;
+ }
+ Ok(())
+}
+
+fn pointer_location() -> Point {
+ let event = CGEvent::new(None);
+ let at = CGEvent::location(event.as_deref());
+ Point { x: at.x, y: at.y }
+}
+
+fn display_at(at: Point) -> Option<CGDirectDisplayID> {
+ let mut display: CGDirectDisplayID = 0;
+ let mut count = 0u32;
+ // SAFETY: both out-pointers are valid for one display.
+ let error = unsafe { CGGetDisplaysWithPoint(CGPoint { x: at.x, y: at.y }, 1, &mut display, &mut count) };
+ (error == CGError::Success && count == 1).then_some(display)
+}
+
+fn shareable_content() -> Result<Retained<SCShareableContent>, String> {
+ let (tx, rx) = mpsc::channel();
+ let handler = RcBlock::new(move |content: *mut SCShareableContent, error: *mut NSError| {
+ // SAFETY: ScreenCaptureKit passes valid (or null) objects to the handler.
+ let content = unsafe { Retained::retain(content) };
+ let error = unsafe { error.as_ref() }.map(|e| e.localizedDescription().to_string());
+ let _ = tx.send(Handoff(content.ok_or_else(|| error.unwrap_or_else(|| "no shareable content".into()))));
+ });
+ // SAFETY: the handler matches the documented signature.
+ unsafe { SCShareableContent::getShareableContentWithCompletionHandler(&handler) };
+ let Handoff(content) = rx.recv_timeout(CAPTURE_TIMEOUT).map_err(|_| "the screen capture timed out")?;
+ content.map_err(|e| format!("cannot list the screen content: {e}"))
+}
+
+fn capture_image(filter: &SCContentFilter, config: &SCStreamConfiguration) -> Result<CFRetained<CGImage>, String> {
+ let (tx, rx) = mpsc::channel();
+ let handler = RcBlock::new(move |image: *mut CGImage, error: *mut NSError| {
+ // SAFETY: the image is valid for the handler's duration; retaining keeps it alive.
+ let image = NonNull::new(image).map(|image| unsafe { CFRetained::retain(image) });
+ let error = unsafe { error.as_ref() }.map(|e| e.localizedDescription().to_string());
+ let _ = tx.send(Handoff(image.ok_or_else(|| error.unwrap_or_else(|| "no image".into()))));
+ });
+ // SAFETY: the handler matches the documented signature.
+ unsafe {
+ SCScreenshotManager::captureImageWithFilter_configuration_completionHandler(filter, config, Some(&handler))
+ };
+ let Handoff(image) = rx.recv_timeout(CAPTURE_TIMEOUT).map_err(|_| "the screenshot timed out")?;
+ image.map_err(|e| format!("cannot take the screenshot: {e}"))
+}
+
+fn jpeg_of(image: &CGImage) -> Result<(Vec<u8>, u32, u32), String> {
+ let (width, height) = (CGImage::width(Some(image)), CGImage::height(Some(image)));
+ if CGImage::bits_per_pixel(Some(image)) != 32 {
+ return Err("unexpected screenshot pixel format".into());
+ }
+ let data = CGImage::data_provider(Some(image)).and_then(|p| CGDataProvider::data(Some(&p)));
+ let bytes = data.ok_or("cannot read the screenshot pixels")?.to_vec();
+ let jpeg = bgra_to_jpeg(&bytes, width as u32, height as u32, CGImage::bytes_per_row(Some(image)), JPEG_QUALITY)?;
+ Ok((jpeg, width as u32, height as u32))
+}
+
+fn ax_string(element: &AXUIElement, attribute: &str) -> Option<String> {
+ let value = ax_value(element, attribute)?;
+ value.downcast_ref::<CFString>().map(|s| s.to_string()).filter(|s| !s.trim().is_empty())
+}
+
+fn ax_value(element: &AXUIElement, attribute: &str) -> Option<CFRetained<CFType>> {
+ let attribute = CFString::from_str(attribute);
+ let mut value: *const CFType = std::ptr::null();
+ // SAFETY: `value` is a valid out-pointer; on success it holds a +1 reference.
+ let error = unsafe { element.copy_attribute_value(&attribute, NonNull::from(&mut value)) };
+ if error != AXError::Success {
+ return None;
+ }
+ NonNull::new(value.cast_mut()).map(|value| unsafe { CFRetained::from_raw(value) })
+}
+
+fn ax_element(element: &AXUIElement, attribute: &str) -> Option<CFRetained<AXUIElement>> {
+ ax_value(element, attribute)?.downcast::<AXUIElement>().ok()
+}
+
+fn describe(element: &AXUIElement) -> AxElement {
+ AxElement {
+ role: ax_string(element, "AXRole").unwrap_or_default(),
+ title: ax_string(element, "AXTitle"),
+ description: ax_string(element, "AXDescription"),
+ value: ax_string(element, "AXValue"),
+ }
+}
+
+fn system_wide() -> CFRetained<AXUIElement> {
+ // SAFETY: plain constructor.
+ let element = unsafe { AXUIElement::new_system_wide() };
+ // A hung app must not stall the helper.
+ unsafe { element.set_messaging_timeout(0.5) };
+ element
+}
+
+type WindowInfo = NSDictionary<NSString, objc2::runtime::AnyObject>;
+
+/// A window-list key (a `CFString` constant) as the bridged `NSString`.
+fn info_key(key: &'static CFString) -> &'static NSString {
+ // SAFETY: CFString is toll-free bridged to NSString.
+ unsafe { &*(key as *const CFString).cast::<NSString>() }
+}
+
+/// `{X, Y, Width, Height}` of a window-list entry, in global points.
+fn window_bounds(entry: &WindowInfo) -> Option<CGRect> {
+ // SAFETY: extern constants provided by CoreGraphics.
+ let bounds = entry.objectForKey(info_key(unsafe { kCGWindowBounds }))?;
+ let bounds = bounds.downcast::<NSDictionary>().ok()?;
+ let number = |key: &str| -> Option<f64> {
+ let value = bounds.objectForKey(&NSString::from_str(key))?;
+ Some(value.downcast::<NSNumber>().ok()?.doubleValue())
+ };
+ let origin = CGPoint::new(number("X")?, number("Y")?);
+ Some(CGRect::new(origin, CGSize::new(number("Width")?, number("Height")?)))
+}
+
+impl Desktop for MacDesktop {
+ fn permissions(&self) -> Permissions {
+ Permissions {
+ // SAFETY: plain queries.
+ accessibility: unsafe { AXIsProcessTrusted() },
+ screen_recording: CGPreflightScreenCaptureAccess(),
+ screenshots_supported: available!(macos = 14.0),
+ }
+ }
+
+ fn capture(&self) -> Result<Capture, String> {
+ if !available!(macos = 14.0) {
+ return Err("screenshots need macOS 14 or later".into());
+ }
+ let display_id = display_at(pointer_location()).ok_or("no display under the pointer")?;
+ let content = shareable_content()?;
+ // SAFETY: reading properties of the snapshot.
+ let display = unsafe { content.displays() }
+ .iter()
+ .find(|d| unsafe { d.displayID() } == display_id)
+ .ok_or("the display under the pointer cannot be captured")?;
+ let own: Vec<Retained<SCWindow>> = unsafe { content.windows() }
+ .iter()
+ .filter(|w| unsafe { w.owningApplication() }.is_some_and(|app| unsafe { app.processID() } == self.own_pid))
+ .collect();
+ let excluded = NSArray::from_retained_slice(&own);
+ let frame = unsafe { display.frame() };
+ let (width, height) = fit_long_edge(frame.size.width as u32, frame.size.height as u32, MAX_LONG_EDGE);
+ let filter = unsafe {
+ SCContentFilter::initWithDisplay_excludingWindows(SCContentFilter::alloc(), &display, &excluded)
+ };
+ let config = unsafe { SCStreamConfiguration::new() };
+ unsafe {
+ config.setWidth(width as usize);
+ config.setHeight(height as usize);
+ config.setPixelFormat(PIXEL_FORMAT_BGRA);
+ config.setShowsCursor(true);
+ }
+ let image = capture_image(&filter, &config)?;
+ let (data, image_width, image_height) = jpeg_of(&image)?;
+ let origin = Point { x: frame.origin.x, y: frame.origin.y };
+ let geometry = ScreenshotGeometry::for_display(origin, frame.size.width, image_width, image_height);
+ Ok(Capture { data, mime_type: "image/jpeg", geometry })
+ }
+
+ fn post(&self, event: &InputEvent) -> Result<(), String> {
+ match event {
+ InputEvent::Move(at) => post_event(mouse(CGEventType::MouseMoved, *at, CGMouseButton::Left)),
+ InputEvent::Click { at, button, count } => click(*at, *button, *count),
+ InputEvent::Drag { from, to } => drag(*from, *to),
+ InputEvent::Scroll { at, dx, dy } => scroll(*at, *dx, *dy),
+ InputEvent::Key(combo) => key(combo),
+ InputEvent::Text(chunk) => text(chunk),
+ }
+ }
+
+ fn list_apps(&self) -> Result<Vec<AppInfo>, String> {
+ let apps = NSWorkspace::sharedWorkspace().runningApplications();
+ Ok(apps
+ .iter()
+ .filter(|app| app.activationPolicy() == NSApplicationActivationPolicy::Regular)
+ .map(|app| {
+ let id = identity(&app);
+ AppInfo { name: id.name, bundle_id: id.bundle_id, active: app.isActive() }
+ })
+ .collect())
+ }
+
+ fn open_app(&self, name: &str) -> Result<(), String> {
+ // A bundle id has dots and no spaces; anything else is an app name.
+ let by_bundle = name.contains('.') && !name.contains(' ') && !name.ends_with(".app");
+ let output = std::process::Command::new("/usr/bin/open")
+ .arg(if by_bundle { "-b" } else { "-a" })
+ .arg(name)
+ .output()
+ .map_err(|e| format!("cannot run `open`: {e}"))?;
+ if output.status.success() {
+ Ok(())
+ } else {
+ Err(format!("cannot open `{name}`: {}", String::from_utf8_lossy(&output.stderr).trim()))
+ }
+ }
+
+ fn element_at(&self, at: Point) -> Vec<AxElement> {
+ let mut found: *const AXUIElement = std::ptr::null();
+ // SAFETY: `found` is a valid out-pointer; on success it holds a +1 reference.
+ let out = NonNull::from(&mut found);
+ let error = unsafe { system_wide().copy_element_at_position(at.x as f32, at.y as f32, out) };
+ let Some(found) = NonNull::new(found.cast_mut()).filter(|_| error == AXError::Success) else {
+ return Vec::new();
+ };
+ let mut element = unsafe { CFRetained::from_raw(found) };
+ let mut chain = vec![describe(&element)];
+ while chain.len() <= AX_ANCESTORS {
+ let Some(parent) = ax_element(&element, "AXParent") else { break };
+ chain.push(describe(&parent));
+ element = parent;
+ }
+ chain
+ }
+
+ fn focused_element(&self) -> Option<Focus> {
+ let focused = ax_element(&system_wide(), "AXFocusedUIElement")?;
+ let default = ax_element(&focused, "AXWindow").and_then(|window| ax_element(&window, "AXDefaultButton"));
+ let is_default_button = default.as_ref().is_some_and(|d| **d == *focused);
+ Some(Focus { element: describe(&focused), is_default_button, default_button: default.as_deref().map(describe) })
+ }
+
+ fn frontmost_app(&self) -> Option<AppIdentity> {
+ NSWorkspace::sharedWorkspace().frontmostApplication().map(|app| identity(&app))
+ }
+
+ fn window_owner_at(&self, at: Point) -> Option<AppIdentity> {
+ let options = CGWindowListOption::OptionOnScreenOnly | CGWindowListOption::ExcludeDesktopElements;
+ let list = CGWindowListCopyWindowInfo(options, kCGNullWindowID)?;
+ // SAFETY: CFArray of CFDictionary is toll-free bridged to NSArray of NSDictionary.
+ let list: &NSArray<WindowInfo> = unsafe { &*(CFRetained::as_ptr(&list).as_ptr() as *const _) };
+ // SAFETY: extern constants provided by CoreGraphics.
+ let (alpha_key, pid_key, name_key) =
+ unsafe { (info_key(kCGWindowAlpha), info_key(kCGWindowOwnerPID), info_key(kCGWindowOwnerName)) };
+ let point = CGPoint { x: at.x, y: at.y };
+ // Front to back: the first visible window containing the point owns it.
+ let entry = list.iter().find(|entry| {
+ let alpha = entry
+ .objectForKey(alpha_key)
+ .and_then(|a| a.downcast::<NSNumber>().ok())
+ .map_or(1.0, |a| a.doubleValue());
+ let bounds = window_bounds(entry);
+ alpha > 0.0
+ && bounds.is_some_and(|b| {
+ point.x >= b.origin.x
+ && point.y >= b.origin.y
+ && point.x < b.origin.x + b.size.width
+ && point.y < b.origin.y + b.size.height
+ })
+ })?;
+ let pid = entry.objectForKey(pid_key)?.downcast::<NSNumber>().ok()?.intValue();
+ match NSRunningApplication::runningApplicationWithProcessIdentifier(pid) {
+ Some(app) => Some(identity(&app)),
+ None => {
+ let name = entry
+ .objectForKey(name_key)
+ .and_then(|n| n.downcast::<NSString>().ok())
+ .map(|n| n.to_string())
+ .unwrap_or_default();
+ Some(AppIdentity { pid, bundle_id: None, name })
+ }
+ }
+ }
+
+ fn activate(&self, pid: i32) {
+ if let Some(app) = NSRunningApplication::runningApplicationWithProcessIdentifier(pid) {
+ app.activateWithOptions(NSApplicationActivationOptions::empty());
+ }
+ }
+}
+
+/// Shows the system prompt for `kind` (macOS shows it once per app) and opens its pane in System Settings.
+pub fn request_permission(kind: PermissionKind) -> Result<(), String> {
+ match kind {
+ PermissionKind::Accessibility => {
+ // SAFETY: the extern static is a valid CFString provided by HIServices.
+ let key: &CFString = unsafe { objc2_application_services::kAXTrustedCheckOptionPrompt };
+ let options = CFDictionary::<CFString, CFBoolean>::from_slices(&[key], &[CFBoolean::new(true)]);
+ // SAFETY: `options` is a valid dictionary with the documented key.
+ unsafe { AXIsProcessTrustedWithOptions(Some(options.as_opaque())) };
+ }
+ PermissionKind::ScreenRecording => {
+ CGRequestScreenCaptureAccess();
+ }
+ }
+ open_privacy_pane(kind)
+}
+
+/// Opens the Privacy & Security pane for `kind` in System Settings.
+pub fn open_privacy_pane(kind: PermissionKind) -> Result<(), String> {
+ let pane = match kind {
+ PermissionKind::Accessibility => "Privacy_Accessibility",
+ PermissionKind::ScreenRecording => "Privacy_ScreenCapture",
+ };
+ let url = format!("x-apple.systempreferences:com.apple.preference.security?{pane}");
+ let status = std::process::Command::new("/usr/bin/open").arg(url).status().map_err(|e| e.to_string())?;
+ status.success().then_some(()).ok_or_else(|| "cannot open System Settings".into())
+}
diff --git a/apps/desktop/src-tauri/src/computer/mcp.rs b/apps/desktop/src-tauri/src/computer/mcp.rs
new file mode 100644
index 0000000..58b0f84
--- /dev/null
+++ b/apps/desktop/src-tauri/src/computer/mcp.rs
@@ -0,0 +1,1085 @@
+//! The MCP endpoint (S24.1): JSON-RPC 2.0 over streamable HTTP with `initialize`,
+//! `tools/list`, and `tools/call`, behind a per-launch bearer token. Each call passes the
+//! guards in order: rate limit, permissions, session grant, blocklist, risky-action
+//! confirmation; then it acts and returns a fresh screenshot (S24.2–S24.4).
+
+use super::blocklist::{blocked_name, blocked_reason, AppIdentity};
+use super::control::Control;
+use super::coords::{Point, ScreenshotGeometry};
+use super::keys::{parse_combo, KeyCombo};
+use super::rate::{RateLimiter, MAX_ACTIONS_PER_SECOND};
+use super::risk::{assess, Act};
+use super::{Clock, Desktop, Dialogs, InputEvent, MouseButton};
+use base64::Engine as _;
+use serde_json::{json, Map, Value};
+use std::sync::{Arc, Mutex, MutexGuard};
+
+/// Supported MCP revisions, newest first; the newest answers an unknown request.
+const PROTOCOL_VERSIONS: &[&str] = &["2025-06-18", "2025-03-26", "2024-11-05"];
+/// Time for the screen to settle before the screenshot that follows an action.
+const SETTLE_MS: u64 = 300;
+const OPEN_APP_SETTLE_MS: u64 = 1500;
+const MAX_WAIT_MS: u64 = 5000;
+const WAIT_SLICE_MS: u64 = 100;
+/// macOS posts at most 20 UTF-16 units of text per keyboard event.
+const TEXT_CHUNK_UNITS: usize = 20;
+
+const STOPPED: &str = "Computer control was stopped; this action was dropped.";
+const NOT_ALLOWED: &str = "Computer control was not allowed by the user. Nothing was done.";
+const NEEDS_PERMISSION: &str = "Gentle Dot needs the";
+const GRANT_IT: &str = "Ask the user to grant it in Connectors → Computer.";
+const NEEDS_SCREENSHOT: &str = "Take a screenshot first: coordinates are pixels of the latest screenshot.";
+
+const INSTRUCTIONS: &str = "Controls this Mac. Take a screenshot first; x and y are pixels of the latest \
+screenshot. Every action returns a fresh screenshot. The user must allow a session, risky actions ask \
+the user to confirm, and some apps (System Settings, password managers) are off limits.";
+
+pub struct HttpRequest<'a> {
+ pub method: &'a str,
+ pub path: &'a str,
+ pub authorization: Option<&'a str>,
+ pub body: &'a str,
+}
+
+#[derive(Debug, Clone, PartialEq)]
+pub struct HttpReply {
+ pub status: u16,
+ pub body: Option<Value>,
+}
+
+enum Tool {
+ Screenshot,
+ Click { x: f64, y: f64, button: MouseButton, count: u8 },
+ Move { x: f64, y: f64 },
+ Drag { from: (f64, f64), to: (f64, f64) },
+ Scroll { x: f64, y: f64, dx: i32, dy: i32 },
+ Type { text: String },
+ Key { combo: KeyCombo },
+ OpenApp { name: String },
+ ListApps,
+ Wait { ms: u64 },
+}
+
+struct Call {
+ tool: Tool,
+ intent: Option<String>,
+}
+
+type Content = Vec<Value>;
+
+fn text(text: impl Into<String>) -> Value {
+ json!({"type": "text", "text": text.into()})
+}
+
+fn tool_error(message: &str) -> Value {
+ json!({"content": [text(message)], "isError": true})
+}
+
+fn lock<T>(mutex: &Mutex<T>) -> MutexGuard<'_, T> {
+ mutex.lock().unwrap_or_else(|poisoned| poisoned.into_inner())
+}
+
+fn constant_time_eq(a: &[u8], b: &[u8]) -> bool {
+ a.len() == b.len() && a.iter().zip(b).fold(0u8, |acc, (x, y)| acc | (x ^ y)) == 0
+}
+
+fn rpc_reply(status: u16, id: Value, outcome: Result<Value, (i64, String)>) -> HttpReply {
+ let body = match outcome {
+ Ok(result) => json!({"jsonrpc": "2.0", "id": id, "result": result}),
+ Err((code, message)) => json!({"jsonrpc": "2.0", "id": id, "error": {"code": code, "message": message}}),
+ };
+ HttpReply { status, body: Some(body) }
+}
+
+fn schema(properties: Value, required: &[&str]) -> Value {
+ let mut properties = properties.as_object().cloned().unwrap_or_default();
+ properties.insert(
+ "intent".into(),
+ json!({"type": "string", "description": "What this step is for, in a few words. Shown to the user when the action needs a confirmation."}),
+ );
+ json!({"type": "object", "properties": properties, "required": required, "additionalProperties": false})
+}
+
+fn coordinate(description: &str) -> Value {
+ json!({"type": "number", "minimum": 0, "description": description})
+}
+
+/// The tool catalog (S24.2).
+fn tools() -> Value {
+ let x = coordinate("Pixels from the left edge of the latest screenshot.");
+ let y = coordinate("Pixels from the top edge of the latest screenshot.");
+ json!([
+ {"name": "screenshot", "description": "Take a screenshot of the display under the pointer. Returns the image and its size.",
+ "inputSchema": schema(json!({}), &[])},
+ {"name": "click", "description": "Click at a point of the latest screenshot.",
+ "inputSchema": schema(json!({"x": x, "y": y,
+ "button": {"type": "string", "enum": ["left", "right"], "default": "left"},
+ "count": {"type": "integer", "enum": [1, 2, 3], "default": 1, "description": "2 for a double click, 3 for a triple click."}}), &["x", "y"])},
+ {"name": "move", "description": "Move the pointer to a point of the latest screenshot.",
+ "inputSchema": schema(json!({"x": x, "y": y}), &["x", "y"])},
+ {"name": "drag", "description": "Press at one point, move to another, and release.",
+ "inputSchema": schema(json!({"fromX": x, "fromY": y, "toX": x, "toY": y}), &["fromX", "fromY", "toX", "toY"])},
+ {"name": "scroll", "description": "Scroll at a point. dy > 0 scrolls down and dx > 0 scrolls right, in pixels.",
+ "inputSchema": schema(json!({"x": x, "y": y, "dx": {"type": "integer"}, "dy": {"type": "integer"}}), &["x", "y", "dx", "dy"])},
+ {"name": "type", "description": "Type text into the frontmost app. A newline presses Return.",
+ "inputSchema": schema(json!({"text": {"type": "string"}}), &["text"])},
+ {"name": "key", "description": "Press a key combo in the frontmost app, for example `cmd+shift+t`, `return`, or `esc`.",
+ "inputSchema": schema(json!({"combo": {"type": "string"}}), &["combo"])},
+ {"name": "open_app", "description": "Open or bring forward an app by name or bundle id.",
+ "inputSchema": schema(json!({"name": {"type": "string"}}), &["name"])},
+ {"name": "list_apps", "description": "List the running apps.",
+ "inputSchema": schema(json!({}), &[])},
+ {"name": "wait", "description": "Wait, then take a screenshot.",
+ "inputSchema": schema(json!({"ms": {"type": "integer", "minimum": 0, "maximum": MAX_WAIT_MS}}), &["ms"])}
+ ])
+}
+
+fn number(args: &Map<String, Value>, key: &str) -> Result<f64, String> {
+ args.get(key).and_then(Value::as_f64).ok_or_else(|| format!("`{key}` must be a number"))
+}
+
+fn integer(args: &Map<String, Value>, key: &str) -> Result<i64, String> {
+ let value = number(args, key)?;
+ if value.fract() != 0.0 || value.abs() > 1e9 {
+ return Err(format!("`{key}` must be a whole number"));
+ }
+ Ok(value as i64)
+}
+
+fn string(args: &Map<String, Value>, key: &str) -> Result<String, String> {
+ args.get(key).and_then(Value::as_str).map(str::to_string).ok_or_else(|| format!("`{key}` must be a string"))
+}
+
+/// The call for `name`, `None` for an unknown tool, or why the arguments are invalid.
+fn parse_call(name: &str, args: &Value) -> Option<Result<Call, String>> {
+ let empty = Map::new();
+ let args = args.as_object().unwrap_or(&empty);
+ let point = |kx: &str, ky: &str| -> Result<(f64, f64), String> { Ok((number(args, kx)?, number(args, ky)?)) };
+ let tool = || -> Result<Tool, String> {
+ Ok(match name {
+ "screenshot" => Tool::Screenshot,
+ "click" => {
+ let (x, y) = point("x", "y")?;
+ let button = match args.get("button").map(|b| b.as_str()) {
+ None | Some(Some("left")) => MouseButton::Left,
+ Some(Some("right")) => MouseButton::Right,
+ _ => return Err("`button` must be \"left\" or \"right\"".into()),
+ };
+ let count = if args.contains_key("count") { integer(args, "count")? } else { 1 };
+ if !(1..=3).contains(&count) {
+ return Err("`count` must be 1, 2, or 3".into());
+ }
+ Tool::Click { x, y, button, count: count as u8 }
+ }
+ "move" => {
+ let (x, y) = point("x", "y")?;
+ Tool::Move { x, y }
+ }
+ "drag" => Tool::Drag { from: point("fromX", "fromY")?, to: point("toX", "toY")? },
+ "scroll" => {
+ let (x, y) = point("x", "y")?;
+ let delta = |key| integer(args, key).and_then(|v| i32::try_from(v).map_err(|e| e.to_string()));
+ Tool::Scroll { x, y, dx: delta("dx")?, dy: delta("dy")? }
+ }
+ "type" => Tool::Type { text: string(args, "text")? },
+ "key" => Tool::Key { combo: parse_combo(&string(args, "combo")?)? },
+ "open_app" => {
+ let name = string(args, "name")?.trim().to_string();
+ if name.is_empty() {
+ return Err("`name` must not be empty".into());
+ }
+ Tool::OpenApp { name }
+ }
+ "list_apps" => Tool::ListApps,
+ "wait" => {
+ let ms = integer(args, "ms")?;
+ if !(0..=MAX_WAIT_MS as i64).contains(&ms) {
+ return Err(format!("`ms` must be between 0 and {MAX_WAIT_MS}"));
+ }
+ Tool::Wait { ms: ms as u64 }
+ }
+ other => return Err(format!("unknown tool `{other}`")),
+ })
+ };
+ const NAMES: &[&str] =
+ &["screenshot", "click", "move", "drag", "scroll", "type", "key", "open_app", "list_apps", "wait"];
+ if !NAMES.contains(&name) {
+ return None;
+ }
+ let intent = args.get("intent").and_then(Value::as_str).map(str::to_string);
+ Some(tool().map(|tool| Call { tool, intent }))
+}
+
+/// Splits text into chunks of at most `TEXT_CHUNK_UNITS` UTF-16 units, never inside a character.
+fn chunks(text: &str) -> Vec<String> {
+ let mut out = Vec::new();
+ let mut current = String::new();
+ for c in text.chars() {
+ if current.encode_utf16().count() + c.len_utf16() > TEXT_CHUNK_UNITS {
+ out.push(std::mem::take(&mut current));
+ }
+ current.push(c);
+ }
+ if !current.is_empty() {
+ out.push(current);
+ }
+ out
+}
+
+pub struct Helper {
+ desktop: Arc<dyn Desktop>,
+ dialogs: Arc<dyn Dialogs>,
+ clock: Arc<dyn Clock>,
+ control: Arc<Control>,
+ token: String,
+ own_pid: i32,
+ limiter: Mutex<RateLimiter>,
+ /// Serializes actions and holds the geometry of the latest screenshot.
+ actions: Mutex<Option<ScreenshotGeometry>>,
+}
+
+impl Helper {
+ pub fn new(
+ desktop: Arc<dyn Desktop>,
+ dialogs: Arc<dyn Dialogs>,
+ clock: Arc<dyn Clock>,
+ control: Arc<Control>,
+ token: String,
+ own_pid: i32,
+ ) -> Self {
+ Helper {
+ desktop,
+ dialogs,
+ clock,
+ control,
+ token,
+ own_pid,
+ limiter: Mutex::new(RateLimiter::new(MAX_ACTIONS_PER_SECOND, 1000)),
+ actions: Mutex::new(None),
+ }
+ }
+
+ fn authorized(&self, header: Option<&str>) -> bool {
+ let Some((scheme, token)) = header.and_then(|h| h.trim().split_once(' ')) else {
+ return false;
+ };
+ scheme.eq_ignore_ascii_case("bearer") && constant_time_eq(token.trim().as_bytes(), self.token.as_bytes())
+ }
+
+ pub fn handle(&self, request: &HttpRequest<'_>) -> HttpReply {
+ if !self.authorized(request.authorization) {
+ return rpc_reply(401, Value::Null, Err((-32001, "Unauthorized".into())));
+ }
+ if request.path != "/mcp" {
+ return HttpReply { status: 404, body: None };
+ }
+ if request.method != "POST" {
+ return HttpReply { status: 405, body: None };
+ }
+ let Ok(message) = serde_json::from_str::<Value>(request.body) else {
+ return rpc_reply(400, Value::Null, Err((-32700, "Parse error".into())));
+ };
+ if !message.is_object() {
+ return rpc_reply(400, Value::Null, Err((-32600, "Invalid request".into())));
+ }
+ // Notifications and responses get no reply.
+ let (Some(method), Some(id)) = (message["method"].as_str(), message.get("id").cloned()) else {
+ return HttpReply { status: 202, body: None };
+ };
+ let params = message.get("params").cloned().unwrap_or(Value::Null);
+ let outcome = match method {
+ "initialize" => Ok(self.initialize(&params)),
+ "ping" => Ok(json!({})),
+ "tools/list" => Ok(json!({"tools": tools()})),
+ "tools/call" => self.tools_call(&params),
+ _ => Err((-32601, format!("Method not found: {method}"))),
+ };
+ rpc_reply(200, id, outcome)
+ }
+
+ fn initialize(&self, params: &Value) -> Value {
+ let requested = params["protocolVersion"].as_str();
+ let version = requested.filter(|v| PROTOCOL_VERSIONS.contains(v)).unwrap_or(PROTOCOL_VERSIONS[0]);
+ json!({
+ "protocolVersion": version,
+ "capabilities": {"tools": {"listChanged": false}},
+ "serverInfo": {"name": "gentle-dot-computer", "version": env!("CARGO_PKG_VERSION")},
+ "instructions": INSTRUCTIONS,
+ })
+ }
+
+ fn tools_call(&self, params: &Value) -> Result<Value, (i64, String)> {
+ let name = params["name"].as_str().ok_or((-32602, "Invalid params: `name` is required".to_string()))?;
+ let args = params.get("arguments").cloned().unwrap_or_else(|| json!({}));
+ self.control.expire(self.clock.now_ms());
+ let epoch = self.control.epoch();
+ self.call_tool_from(epoch, name, &args).ok_or((-32602, format!("Unknown tool: {name}")))
+ }
+
+ /// A `tools/call` that arrived while the session was at `epoch`.
+ fn call_tool_from(&self, epoch: u64, name: &str, args: &Value) -> Option<Value> {
+ let outcome = parse_call(name, args)?.and_then(|call| self.run(epoch, call));
+ Some(match outcome {
+ Ok(content) => json!({"content": content}),
+ Err(message) => tool_error(&message),
+ })
+ }
+
+ fn run(&self, epoch: u64, call: Call) -> Result<Content, String> {
+ if !lock(&self.limiter).try_acquire(self.clock.now_ms()) {
+ return Err(format!("Too many actions: at most {MAX_ACTIONS_PER_SECOND} per second. Wait a moment."));
+ }
+ self.check_permissions(&call.tool)?;
+ let mut latest = lock(&self.actions);
+ self.control.expire(self.clock.now_ms());
+ if self.control.epoch() != epoch {
+ return Err(STOPPED.into());
+ }
+ if !self.control.is_active(self.clock.now_ms()) {
+ if !self.with_dialog(|| self.dialogs.ask_grant()) {
+ return Err(NOT_ALLOWED.into());
+ }
+ self.control.grant(self.clock.now_ms());
+ }
+ self.act(epoch, &mut latest, call)
+ }
+
+ fn check_permissions(&self, tool: &Tool) -> Result<(), String> {
+ let permissions = self.desktop.permissions();
+ let sees = !matches!(tool, Tool::ListApps);
+ let acts = matches!(
+ tool,
+ Tool::Click { .. }
+ | Tool::Move { .. }
+ | Tool::Drag { .. }
+ | Tool::Scroll { .. }
+ | Tool::Type { .. }
+ | Tool::Key { .. }
+ );
+ if sees && !permissions.screenshots_supported {
+ return Err("Screenshots need macOS 14 or later, so computer control is unavailable on this Mac.".into());
+ }
+ if sees && !permissions.screen_recording {
+ return Err(format!("{NEEDS_PERMISSION} Screen Recording permission. {GRANT_IT}"));
+ }
+ if acts && !permissions.accessibility {
+ return Err(format!("{NEEDS_PERMISSION} Accessibility permission. {GRANT_IT}"));
+ }
+ Ok(())
+ }
+
+ /// Runs a dialog, then brings back the app that was in front (a dialog activates Gentle Dot).
+ fn with_dialog(&self, dialog: impl FnOnce() -> bool) -> bool {
+ let before = self.desktop.frontmost_app();
+ let answer = dialog();
+ if let Some(before) = before {
+ if self.desktop.frontmost_app().map(|app| app.pid) != Some(before.pid) {
+ self.desktop.activate(before.pid);
+ }
+ }
+ answer
+ }
+
+ fn still_running(&self, epoch: u64) -> Result<(), String> {
+ if self.control.epoch() != epoch || !self.control.is_active(self.clock.now_ms()) {
+ return Err(STOPPED.into());
+ }
+ Ok(())
+ }
+
+ fn post(&self, epoch: u64, event: InputEvent) -> Result<(), String> {
+ self.still_running(epoch)?;
+ self.desktop.post(&event)
+ }
+
+ fn refuse_blocked(&self, app: &AppIdentity) -> Result<(), String> {
+ match blocked_reason(app, self.own_pid) {
+ Some(label) => Err(format!("Gentle Dot does not control {label}. The action was refused.")),
+ None => Ok(()),
+ }
+ }
+
+ /// The app owning the window under `at`, refused when blocked. Its name, for messages.
+ fn pointer_target(&self, at: Point) -> Result<String, String> {
+ match self.desktop.window_owner_at(at) {
+ Some(app) => self.refuse_blocked(&app).map(|()| app.name),
+ None => Ok("the desktop".into()),
+ }
+ }
+
+ /// The frontmost app, refused when blocked. Its name, for messages.
+ fn keyboard_target(&self) -> Result<String, String> {
+ let app = self.desktop.frontmost_app().ok_or("Cannot tell which app is in front.")?;
+ self.refuse_blocked(&app).map(|()| app.name)
+ }
+
+ fn confirm_if_risky(&self, risk: Option<String>, action: &str, target: &str) -> Result<(), String> {
+ let Some(reason) = risk else {
+ return Ok(());
+ };
+ let message = format!("{action} in {target}.\n\nThis needs your OK because of {reason}.");
+ if self.with_dialog(|| self.dialogs.confirm(&message)) {
+ Ok(())
+ } else {
+ Err(format!("The user declined: {action} in {target}. Nothing was done."))
+ }
+ }
+
+ fn screenshot(&self, latest: &mut Option<ScreenshotGeometry>) -> Result<Content, String> {
+ let capture = self.desktop.capture()?;
+ let geometry = capture.geometry;
+ *latest = Some(geometry);
+ let front = self.desktop.frontmost_app().map(|app| app.name);
+ let meta = json!({
+ "imageWidth": geometry.image_width,
+ "imageHeight": geometry.image_height,
+ "scale": geometry.scale,
+ "frontApp": front,
+ });
+ let data = base64::engine::general_purpose::STANDARD.encode(&capture.data);
+ Ok(vec![json!({"type": "image", "data": data, "mimeType": capture.mime_type}), text(meta.to_string())])
+ }
+
+ /// What was done, then a fresh screenshot once the screen settled.
+ fn after(&self, latest: &mut Option<ScreenshotGeometry>, done: String, settle_ms: u64) -> Result<Content, String> {
+ self.clock.sleep_ms(settle_ms);
+ let mut content = vec![text(done)];
+ match self.screenshot(latest) {
+ Ok(blocks) => content.extend(blocks),
+ Err(error) => content.push(text(format!("No screenshot: {error}"))),
+ }
+ Ok(content)
+ }
+
+ fn act(&self, epoch: u64, latest: &mut Option<ScreenshotGeometry>, call: Call) -> Result<Content, String> {
+ let intent = call.intent.as_deref();
+ let map = |latest: &Option<ScreenshotGeometry>, (x, y): (f64, f64)| -> Result<Point, String> {
+ latest.ok_or(NEEDS_SCREENSHOT)?.to_point(x, y).map_err(|e| e.to_string())
+ };
+ match call.tool {
+ Tool::Screenshot => self.screenshot(latest),
+ Tool::Click { x, y, button, count } => {
+ let at = map(latest, (x, y))?;
+ let target = self.pointer_target(at)?;
+ let chain = self.desktop.element_at(at);
+ let action = match (button, count) {
+ (MouseButton::Right, _) => format!("Right-click at ({x}, {y})"),
+ (_, 2) => format!("Double-click at ({x}, {y})"),
+ (_, 3) => format!("Triple-click at ({x}, {y})"),
+ _ => format!("Click at ({x}, {y})"),
+ };
+ self.confirm_if_risky(assess(Act::Click(&chain), intent), &action, &target)?;
+ self.post(epoch, InputEvent::Click { at, button, count })?;
+ self.after(latest, format!("{action} in {target}: done."), SETTLE_MS)
+ }
+ Tool::Move { x, y } => {
+ let at = map(latest, (x, y))?;
+ let target = self.pointer_target(at)?;
+ let action = format!("Move the pointer to ({x}, {y})");
+ self.confirm_if_risky(assess(Act::Other, intent), &action, &target)?;
+ self.post(epoch, InputEvent::Move(at))?;
+ self.after(latest, format!("{action}: done."), SETTLE_MS)
+ }
+ Tool::Drag { from, to } => {
+ let (start, end) = (map(latest, from)?, map(latest, to)?);
+ let source = self.pointer_target(start)?;
+ let destination = self.pointer_target(end)?;
+ let action = format!("Drag from ({}, {}) to ({}, {})", from.0, from.1, to.0, to.1);
+ let target = if source == destination { source } else { format!("{source} and {destination}") };
+ self.confirm_if_risky(assess(Act::Other, intent), &action, &target)?;
+ self.post(epoch, InputEvent::Drag { from: start, to: end })?;
+ self.after(latest, format!("{action}: done."), SETTLE_MS)
+ }
+ Tool::Scroll { x, y, dx, dy } => {
+ let at = map(latest, (x, y))?;
+ let target = self.pointer_target(at)?;
+ let action = format!("Scroll by ({dx}, {dy}) at ({x}, {y})");
+ self.confirm_if_risky(assess(Act::Other, intent), &action, &target)?;
+ self.post(epoch, InputEvent::Scroll { at, dx, dy })?;
+ self.after(latest, format!("{action}: done."), SETTLE_MS)
+ }
+ Tool::Type { text: typed } => {
+ let target = self.keyboard_target()?;
+ let typed = typed.replace("\r\n", "\n").replace('\r', "\n");
+ let focus = self.desktop.focused_element();
+ let act = if typed.contains('\n') { Act::Return(focus.as_ref()) } else { Act::Other };
+ let action = format!("Type {} characters", typed.chars().count());
+ self.confirm_if_risky(assess(act, intent), &action, &target)?;
+ let return_key = parse_combo("return")?;
+ for (i, line) in typed.split('\n').enumerate() {
+ if i > 0 {
+ self.post(epoch, InputEvent::Key(return_key))?;
+ }
+ for chunk in chunks(line) {
+ self.post(epoch, InputEvent::Text(chunk))?;
+ }
+ }
+ self.after(latest, format!("{action} in {target}: done."), SETTLE_MS)
+ }
+ Tool::Key { combo } => {
+ let target = self.keyboard_target()?;
+ let focus = self.desktop.focused_element();
+ let act = if combo.is_return() { Act::Return(focus.as_ref()) } else { Act::Other };
+ let action = format!("Press {}", combo.key);
+ self.confirm_if_risky(assess(act, intent), &action, &target)?;
+ self.post(epoch, InputEvent::Key(combo))?;
+ self.after(latest, format!("{action} in {target}: done."), SETTLE_MS)
+ }
+ Tool::OpenApp { name } => {
+ if let Some(label) = blocked_name(&name) {
+ return Err(format!("Gentle Dot does not control {label}. The app was not opened."));
+ }
+ let action = format!("Open {name}");
+ self.confirm_if_risky(assess(Act::Other, intent), &action, "macOS")?;
+ self.still_running(epoch)?;
+ self.desktop.open_app(&name)?;
+ self.after(latest, format!("{action}: done."), OPEN_APP_SETTLE_MS)
+ }
+ Tool::ListApps => {
+ let apps = self.desktop.list_apps()?;
+ Ok(vec![text(serde_json::to_string(&apps).map_err(|e| e.to_string())?)])
+ }
+ Tool::Wait { ms } => {
+ let mut left = ms;
+ while left > 0 {
+ let slice = left.min(WAIT_SLICE_MS);
+ self.clock.sleep_ms(slice);
+ left -= slice;
+ self.still_running(epoch)?;
+ }
+ self.after(latest, format!("Waited {ms} ms."), 0)
+ }
+ }
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::super::blocklist::AppIdentity;
+ use super::super::coords::Point;
+ use super::super::fake::{app, FakeClock, FakeDesktop, FakeDialogs};
+ use super::super::keys::parse_combo;
+ use super::super::risk::{AxElement, Focus};
+ use super::super::session::{Reason, StateEvent, SESSION_MS};
+ use super::super::{InputEvent, MouseButton};
+ use super::*;
+ use serde_json::json;
+ use std::sync::atomic::Ordering;
+ use std::sync::Mutex;
+
+ const TOKEN: &str = "secret-token";
+ const OWN_PID: i32 = 4242;
+ const NOW: u64 = 1_700_000_000_000;
+
+ struct Rig {
+ helper: Helper,
+ desktop: Arc<FakeDesktop>,
+ dialogs: Arc<FakeDialogs>,
+ clock: Arc<FakeClock>,
+ control: Arc<Control>,
+ states: Arc<Mutex<Vec<StateEvent>>>,
+ }
+
+ fn rig_with(allow: bool, confirm: bool) -> Rig {
+ let desktop = Arc::new(FakeDesktop::default());
+ let dialogs = Arc::new(FakeDialogs::new(allow, confirm));
+ let clock = Arc::new(FakeClock::new(NOW));
+ let states = Arc::new(Mutex::new(Vec::new()));
+ let sink = states.clone();
+ let control = Arc::new(Control::new(move |event| sink.lock().unwrap().push(event.clone())));
+ let helper =
+ Helper::new(desktop.clone(), dialogs.clone(), clock.clone(), control.clone(), TOKEN.into(), OWN_PID);
+ Rig { helper, desktop, dialogs, clock, control, states }
+ }
+
+ fn rig() -> Rig {
+ rig_with(true, true)
+ }
+
+ fn post(rig: &Rig, authorization: Option<&str>, body: &str) -> HttpReply {
+ rig.helper.handle(&HttpRequest { method: "POST", path: "/mcp", authorization, body })
+ }
+
+ impl Rig {
+ fn rpc(&self, method: &str, params: Value) -> Value {
+ let body = json!({"jsonrpc": "2.0", "id": 7, "method": method, "params": params}).to_string();
+ let reply = post(self, Some(&format!("Bearer {TOKEN}")), &body);
+ assert_eq!(reply.status, 200, "{reply:?}");
+ let body = reply.body.expect("a JSON-RPC reply");
+ assert_eq!((body["jsonrpc"].as_str(), body["id"].as_i64()), (Some("2.0"), Some(7)));
+ body
+ }
+
+ fn call(&self, name: &str, arguments: Value) -> Value {
+ let body = self.rpc("tools/call", json!({"name": name, "arguments": arguments}));
+ body["result"].clone()
+ }
+
+ /// Grants a session and takes the screenshot that coordinates refer to.
+ fn ready(&self) {
+ assert!(!is_error(&self.call("screenshot", json!({}))));
+ }
+ }
+
+ fn is_error(result: &Value) -> bool {
+ result["isError"] == json!(true)
+ }
+
+ fn first_text(result: &Value) -> String {
+ result["content"][0]["text"].as_str().unwrap_or_default().to_string()
+ }
+
+ fn safari() -> AppIdentity {
+ app(200, "com.apple.Safari", "Safari")
+ }
+
+ // --- HTTP and JSON-RPC ---
+
+ #[test]
+ fn requests_without_the_bearer_token_get_401() {
+ let rig = rig();
+ let body = json!({"jsonrpc": "2.0", "id": 1, "method": "tools/list"}).to_string();
+ for authorization in [None, Some("Bearer wrong"), Some(TOKEN), Some("Basic secret-token")] {
+ let reply = post(&rig, authorization, &body);
+ assert_eq!(reply.status, 401, "{authorization:?}");
+ }
+ assert_eq!(post(&rig, Some("Bearer secret-token"), &body).status, 200);
+ }
+
+ #[test]
+ fn only_post_to_mcp_is_served() {
+ let rig = rig();
+ let auth = Some("Bearer secret-token");
+ let get = rig.helper.handle(&HttpRequest { method: "GET", path: "/mcp", authorization: auth, body: "" });
+ assert_eq!(get.status, 405);
+ let other = rig.helper.handle(&HttpRequest { method: "POST", path: "/other", authorization: auth, body: "{}" });
+ assert_eq!(other.status, 404);
+ }
+
+ #[test]
+ fn initialize_reports_the_server_and_tools_capability() {
+ let rig = rig();
+ let body = rig.rpc("initialize", json!({"protocolVersion": "2025-03-26", "capabilities": {}, "clientInfo": {"name": "t", "version": "1"}}));
+ let result = &body["result"];
+ assert_eq!(result["protocolVersion"], "2025-03-26");
+ assert!(result["capabilities"]["tools"].is_object());
+ assert_eq!(result["serverInfo"]["name"], "gentle-dot-computer");
+ }
+
+ #[test]
+ fn initialize_answers_an_unknown_protocol_version_with_its_own() {
+ let rig = rig();
+ let body = rig.rpc("initialize", json!({"protocolVersion": "1999-01-01"}));
+ assert_eq!(body["result"]["protocolVersion"], "2025-06-18");
+ }
+
+ #[test]
+ fn notifications_are_accepted_without_a_body() {
+ let rig = rig();
+ let body = json!({"jsonrpc": "2.0", "method": "notifications/initialized"}).to_string();
+ assert_eq!(post(&rig, Some("Bearer secret-token"), &body), HttpReply { status: 202, body: None });
+ }
+
+ #[test]
+ fn invalid_json_is_a_parse_error() {
+ let reply = post(&rig(), Some("Bearer secret-token"), "{not json");
+ assert_eq!(reply.status, 400);
+ assert_eq!(reply.body.unwrap()["error"]["code"], -32700);
+ }
+
+ #[test]
+ fn unknown_methods_are_rejected() {
+ let body = rig().rpc("resources/list", json!({}));
+ assert_eq!(body["error"]["code"], -32601);
+ }
+
+ #[test]
+ fn tools_list_describes_every_tool_with_an_input_schema() {
+ let body = rig().rpc("tools/list", json!({}));
+ let tools = body["result"]["tools"].as_array().unwrap();
+ let names: Vec<&str> = tools.iter().map(|t| t["name"].as_str().unwrap()).collect();
+ assert_eq!(
+ names,
+ ["screenshot", "click", "move", "drag", "scroll", "type", "key", "open_app", "list_apps", "wait"]
+ );
+ for tool in tools {
+ let schema = &tool["inputSchema"];
+ assert_eq!(schema["type"], "object", "{tool}");
+ assert_eq!(schema["properties"]["intent"]["type"], "string", "{tool}");
+ assert!(tool["description"].as_str().is_some_and(|d| !d.is_empty()));
+ }
+ let click = tools.iter().find(|t| t["name"] == "click").unwrap();
+ assert_eq!(click["inputSchema"]["required"], json!(["x", "y"]));
+ assert_eq!(click["inputSchema"]["properties"]["button"]["enum"], json!(["left", "right"]));
+ let wait = tools.iter().find(|t| t["name"] == "wait").unwrap();
+ assert_eq!(wait["inputSchema"]["properties"]["ms"]["maximum"], 5000);
+ }
+
+ #[test]
+ fn an_unknown_tool_is_a_protocol_error() {
+ let rig = rig();
+ let body = rig.rpc("tools/call", json!({"name": "format_disk", "arguments": {}}));
+ assert_eq!(body["error"]["code"], -32602);
+ assert_eq!(rig.dialogs.grants.load(Ordering::SeqCst), 0);
+ }
+
+ // --- Session grant (S24.3) ---
+
+ #[test]
+ fn deny_returns_a_tool_error_and_starts_nothing() {
+ let rig = rig_with(false, true);
+ let result = rig.call("screenshot", json!({}));
+ assert!(is_error(&result));
+ assert!(first_text(&result).contains("not allowed"), "{result}");
+ assert_eq!(rig.dialogs.grants.load(Ordering::SeqCst), 1);
+ assert_eq!(rig.desktop.captures.load(Ordering::SeqCst), 0);
+ assert!(rig.states.lock().unwrap().is_empty());
+ assert!(!rig.control.is_active(NOW));
+ }
+
+ #[test]
+ fn allow_starts_a_session_once_and_reports_it() {
+ let rig = rig();
+ rig.ready();
+ rig.ready();
+ assert_eq!(rig.dialogs.grants.load(Ordering::SeqCst), 1);
+ assert_eq!(
+ *rig.states.lock().unwrap(),
+ vec![StateEvent { active: true, ends_at: Some(NOW + SESSION_MS), reason: Some(Reason::Granted) }]
+ );
+ }
+
+ #[test]
+ fn after_the_timeout_the_next_call_asks_again() {
+ let rig = rig();
+ rig.ready();
+ rig.clock.advance(SESSION_MS);
+ rig.ready();
+ assert_eq!(rig.dialogs.grants.load(Ordering::SeqCst), 2);
+ }
+
+ #[test]
+ fn after_a_stop_the_next_call_asks_again() {
+ let rig = rig();
+ rig.ready();
+ rig.control.end(Reason::Panic);
+ rig.ready();
+ assert_eq!(rig.dialogs.grants.load(Ordering::SeqCst), 2);
+ }
+
+ #[test]
+ fn an_action_queued_before_a_stop_is_dropped() {
+ let rig = rig();
+ rig.ready();
+ let queued_at = rig.control.epoch();
+ rig.control.end(Reason::Stopped);
+ let result = rig.helper.call_tool_from(queued_at, "click", &json!({"x": 10, "y": 10})).unwrap();
+ assert!(is_error(&result));
+ assert!(first_text(&result).contains("stopped"), "{result}");
+ assert!(rig.desktop.events().is_empty());
+ assert_eq!(rig.dialogs.grants.load(Ordering::SeqCst), 1);
+ }
+
+ #[test]
+ fn a_stop_during_wait_ends_it_early() {
+ let rig = rig();
+ rig.ready();
+ let control = rig.control.clone();
+ *rig.clock.on_sleep.lock().unwrap() = Some(Box::new(move || control.end(Reason::Panic)));
+ let result = rig.call("wait", json!({"ms": 3000}));
+ assert!(is_error(&result));
+ assert!(first_text(&result).contains("stopped"), "{result}");
+ assert!(rig.clock.now_ms() - NOW < 3000);
+ }
+
+ // --- Screenshots and coordinates (S24.2) ---
+
+ #[test]
+ fn a_screenshot_is_an_image_block_plus_its_geometry() {
+ let rig = rig();
+ let result = rig.call("screenshot", json!({}));
+ let content = result["content"].as_array().unwrap();
+ let image = content.iter().find(|c| c["type"] == "image").unwrap();
+ assert_eq!(image["mimeType"], "image/jpeg");
+ assert_eq!(image["data"], "anBlZw==");
+ let meta = content.iter().rfind(|c| c["type"] == "text").unwrap();
+ let meta: Value = serde_json::from_str(meta["text"].as_str().unwrap()).unwrap();
+ assert_eq!(meta, json!({"imageWidth": 1280, "imageHeight": 800, "scale": 0.5, "frontApp": "Safari"}));
+ }
+
+ #[test]
+ fn coordinates_need_a_screenshot_first() {
+ let rig = rig();
+ let result = rig.call("click", json!({"x": 10, "y": 10}));
+ assert!(is_error(&result));
+ assert!(first_text(&result).contains("screenshot"), "{result}");
+ assert!(rig.desktop.events().is_empty());
+ }
+
+ #[test]
+ fn click_maps_pixels_to_points_and_returns_a_fresh_screenshot() {
+ let rig = rig();
+ rig.ready();
+ let result = rig.call("click", json!({"x": 100, "y": 50, "button": "right", "count": 2}));
+ assert!(!is_error(&result), "{result}");
+ let at = Point { x: 200.0, y: 100.0 };
+ assert_eq!(rig.desktop.events(), vec![InputEvent::Click { at, button: MouseButton::Right, count: 2 }]);
+ assert_eq!(rig.desktop.captures.load(Ordering::SeqCst), 2);
+ assert!(result["content"].as_array().unwrap().iter().any(|c| c["type"] == "image"));
+ }
+
+ #[test]
+ fn move_drag_and_scroll_map_their_points() {
+ let rig = rig();
+ rig.ready();
+ rig.call("move", json!({"x": 1, "y": 2}));
+ rig.call("drag", json!({"fromX": 10, "fromY": 20, "toX": 30, "toY": 40}));
+ rig.call("scroll", json!({"x": 5, "y": 6, "dx": 0, "dy": 120}));
+ let p = |x, y| Point { x, y };
+ assert_eq!(
+ rig.desktop.events(),
+ vec![
+ InputEvent::Move(p(2.0, 4.0)),
+ InputEvent::Drag { from: p(20.0, 40.0), to: p(60.0, 80.0) },
+ InputEvent::Scroll { at: p(10.0, 12.0), dx: 0, dy: 120 },
+ ]
+ );
+ }
+
+ #[test]
+ fn invalid_arguments_are_tool_errors() {
+ let rig = rig();
+ rig.ready();
+ for (name, args) in [
+ ("click", json!({"x": 1})),
+ ("click", json!({"x": 1, "y": 1, "count": 4})),
+ ("click", json!({"x": 1, "y": 1, "button": "middle"})),
+ ("click", json!({"x": 5000, "y": 1})),
+ ("wait", json!({"ms": 5001})),
+ ("key", json!({"combo": "cmd+nope"})),
+ ("type", json!({})),
+ ] {
+ let result = rig.call(name, args.clone());
+ assert!(is_error(&result), "{name} {args}");
+ }
+ assert!(rig.desktop.events().is_empty());
+ }
+
+ // --- Permissions ---
+
+ #[test]
+ fn missing_permissions_are_explained_before_asking_for_a_session() {
+ let rig = rig();
+ rig.desktop.permissions.lock().unwrap().screen_recording = false;
+ let result = rig.call("screenshot", json!({}));
+ assert!(first_text(&result).contains("Screen Recording"), "{result}");
+ rig.desktop.permissions.lock().unwrap().screen_recording = true;
+ rig.desktop.permissions.lock().unwrap().accessibility = false;
+ let result = rig.call("key", json!({"combo": "tab"}));
+ assert!(first_text(&result).contains("Accessibility"), "{result}");
+ assert_eq!(rig.dialogs.grants.load(Ordering::SeqCst), 0);
+ }
+
+ #[test]
+ fn screenshots_before_macos_14_are_a_clear_error() {
+ let rig = rig();
+ rig.desktop.permissions.lock().unwrap().screenshots_supported = false;
+ let result = rig.call("screenshot", json!({}));
+ assert!(is_error(&result));
+ assert!(first_text(&result).contains("macOS 14"), "{result}");
+ }
+
+ // --- Blocklist and rate limit (S24.4) ---
+
+ #[test]
+ fn pointer_actions_on_a_blocked_window_are_refused() {
+ let rig = rig();
+ rig.ready();
+ *rig.desktop.owner_at.lock().unwrap() =
+ Box::new(|_| Some(app(300, "com.apple.systempreferences", "System Settings")));
+ let result = rig.call("click", json!({"x": 10, "y": 10}));
+ assert!(is_error(&result));
+ assert!(first_text(&result).contains("System Settings"), "{result}");
+ assert!(rig.desktop.events().is_empty());
+ }
+
+ #[test]
+ fn a_drag_onto_a_blocked_window_is_refused() {
+ let rig = rig();
+ rig.ready();
+ *rig.desktop.owner_at.lock().unwrap() = Box::new(|at: Point| {
+ if at.x > 100.0 {
+ Some(app(301, "com.1password.1password", "1Password"))
+ } else {
+ Some(safari())
+ }
+ });
+ let result = rig.call("drag", json!({"fromX": 10, "fromY": 10, "toX": 200, "toY": 10}));
+ assert!(is_error(&result));
+ assert!(rig.desktop.events().is_empty());
+ }
+
+ #[test]
+ fn gentle_dot_own_windows_are_refused() {
+ let rig = rig();
+ rig.ready();
+ *rig.desktop.owner_at.lock().unwrap() =
+ Box::new(|_| Some(AppIdentity { pid: OWN_PID, bundle_id: None, name: "gentle-dot".into() }));
+ assert!(is_error(&rig.call("click", json!({"x": 10, "y": 10}))));
+ }
+
+ #[test]
+ fn keyboard_actions_check_the_frontmost_app() {
+ let rig = rig();
+ rig.ready();
+ *rig.desktop.frontmost.lock().unwrap() = Some(app(302, "com.apple.keychainaccess", "Keychain Access"));
+ assert!(is_error(&rig.call("type", json!({"text": "hello"}))));
+ assert!(is_error(&rig.call("key", json!({"combo": "cmd+a"}))));
+ assert!(rig.desktop.events().is_empty());
+ }
+
+ #[test]
+ fn opening_a_blocked_app_is_refused() {
+ let rig = rig();
+ rig.ready();
+ assert!(is_error(&rig.call("open_app", json!({"name": "Keychain Access"}))));
+ assert!(!is_error(&rig.call("open_app", json!({"name": "com.apple.mail"}))));
+ assert_eq!(*rig.desktop.opened.lock().unwrap(), vec!["com.apple.mail".to_string()]);
+ }
+
+ #[test]
+ fn more_than_ten_calls_per_second_are_refused() {
+ let rig = rig();
+ for _ in 0..10 {
+ assert!(!is_error(&rig.call("list_apps", json!({}))));
+ }
+ let result = rig.call("list_apps", json!({}));
+ assert!(is_error(&result));
+ assert!(first_text(&result).contains("10"), "{result}");
+ rig.clock.advance(1000);
+ assert!(!is_error(&rig.call("list_apps", json!({}))));
+ }
+
+ // --- Risky actions (S24.4) ---
+
+ fn send_button() -> Vec<AxElement> {
+ vec![AxElement { role: "AXButton".into(), title: Some("Send".into()), ..AxElement::default() }]
+ }
+
+ #[test]
+ fn a_risky_click_needs_a_confirmation_showing_action_and_target() {
+ let rig = rig();
+ rig.ready();
+ *rig.desktop.elements.lock().unwrap() = send_button();
+ let result = rig.call("click", json!({"x": 10, "y": 10}));
+ assert!(!is_error(&result), "{result}");
+ let asked = rig.dialogs.confirmations.lock().unwrap().clone();
+ assert_eq!(asked.len(), 1);
+ assert!(asked[0].contains("Send") && asked[0].contains("Safari"), "{asked:?}");
+ assert_eq!(rig.desktop.events().len(), 1);
+ }
+
+ #[test]
+ fn a_declined_confirmation_is_a_tool_error_and_nothing_happens() {
+ let rig = rig_with(true, false);
+ rig.ready();
+ *rig.desktop.elements.lock().unwrap() = send_button();
+ let result = rig.call("click", json!({"x": 10, "y": 10}));
+ assert!(is_error(&result));
+ assert!(first_text(&result).contains("declined"), "{result}");
+ assert!(rig.desktop.events().is_empty());
+ }
+
+ #[test]
+ fn harmless_actions_are_not_confirmed() {
+ let rig = rig();
+ rig.ready();
+ rig.call("click", json!({"x": 10, "y": 10}));
+ rig.call("key", json!({"combo": "cmd+t"}));
+ assert!(rig.dialogs.confirmations.lock().unwrap().is_empty());
+ assert_eq!(rig.desktop.events().len(), 2);
+ }
+
+ #[test]
+ fn return_in_a_send_like_field_is_confirmed() {
+ let rig = rig_with(true, false);
+ rig.ready();
+ let composer = AxElement { role: "AXTextArea".into(), description: Some("Message".into()), ..AxElement::default() };
+ *rig.desktop.focus.lock().unwrap() = Some(Focus { element: composer, ..Focus::default() });
+ assert!(is_error(&rig.call("key", json!({"combo": "cmd+return"}))));
+ assert!(rig.desktop.events().is_empty());
+ }
+
+ #[test]
+ fn a_risky_intent_adds_a_confirmation() {
+ let rig = rig();
+ rig.ready();
+ rig.call("scroll", json!({"x": 1, "y": 1, "dx": 0, "dy": 10, "intent": "buy the ticket"}));
+ assert_eq!(rig.dialogs.confirmations.lock().unwrap().len(), 1);
+ }
+
+ #[test]
+ fn the_previous_app_is_brought_back_after_a_dialog() {
+ let rig = rig();
+ rig.ready();
+ *rig.desktop.elements.lock().unwrap() = send_button();
+ let frontmost = rig.desktop.clone();
+ *rig.dialogs.during.lock().unwrap() = Some(Box::new(move || {
+ *frontmost.frontmost.lock().unwrap() = Some(app(OWN_PID, "dev.gentleman.gentle-dot", "Gentle Dot"));
+ }));
+ rig.call("click", json!({"x": 10, "y": 10}));
+ assert_eq!(*rig.desktop.activated.lock().unwrap(), vec![200]);
+ }
+
+ // --- Typing ---
+
+ #[test]
+ fn typing_sends_newlines_as_return_and_checks_them() {
+ let rig = rig();
+ rig.ready();
+ let result = rig.call("type", json!({"text": "hi\nthere"}));
+ assert!(!is_error(&result), "{result}");
+ assert_eq!(
+ rig.desktop.events(),
+ vec![
+ InputEvent::Text("hi".into()),
+ InputEvent::Key(parse_combo("return").unwrap()),
+ InputEvent::Text("there".into()),
+ ]
+ );
+ // In a send-like field the newline needs a confirmation.
+ let rig = rig_with(true, false);
+ rig.ready();
+ let composer = AxElement { role: "AXTextField".into(), title: Some("Reply".into()), ..AxElement::default() };
+ *rig.desktop.focus.lock().unwrap() = Some(Focus { element: composer, ..Focus::default() });
+ assert!(is_error(&rig.call("type", json!({"text": "ok\n"}))));
+ assert!(rig.desktop.events().is_empty());
+ }
+
+ #[test]
+ fn long_text_is_sent_in_chunks_of_twenty_utf16_units() {
+ let rig = rig();
+ rig.ready();
+ let text = "ñ".repeat(25) + "😀";
+ rig.call("type", json!({"text": text}));
+ let chunks: Vec<String> = rig
+ .desktop
+ .events()
+ .into_iter()
+ .map(|e| match e {
+ InputEvent::Text(t) => t,
+ other => panic!("{other:?}"),
+ })
+ .collect();
+ assert_eq!(chunks.concat(), text);
+ assert!(chunks.iter().all(|c| c.encode_utf16().count() <= 20), "{chunks:?}");
+ }
+
+ #[test]
+ fn list_apps_returns_names_and_bundle_ids() {
+ let rig = rig();
+ let result = rig.call("list_apps", json!({}));
+ let apps: Value = serde_json::from_str(&first_text(&result)).unwrap();
+ assert_eq!(apps[0], json!({"name": "Safari", "bundleId": "com.apple.Safari", "active": true}));
+ }
+}
diff --git a/apps/desktop/src-tauri/src/computer/mod.rs b/apps/desktop/src-tauri/src/computer/mod.rs
new file mode 100644
index 0000000..ee36b2a
--- /dev/null
+++ b/apps/desktop/src-tauri/src/computer/mod.rs
@@ -0,0 +1,133 @@
+//! Computer control (S24): a native helper inside the app that serves MCP on loopback so the
+//! agent can see and drive the Mac. Every rule (session grant, blocklist, rate limit, risky
+//! action confirmation, panic stop) lives here, outside the agent's reach.
+//!
+//! The rules are pure modules tested on every OS. `Desktop` is the seam to the real Mac
+//! (`macos`), and `fake` records events for the tests. Off macOS the helper does not start.
+
+pub mod app;
+pub mod blocklist;
+pub mod control;
+pub mod coords;
+pub mod encode;
+#[cfg(test)]
+mod fake;
+pub mod keys;
+#[cfg(target_os = "macos")]
+mod macos;
+pub mod mcp;
+pub mod rate;
+pub mod risk;
+pub mod server;
+pub mod session;
+
+use blocklist::AppIdentity;
+use coords::{Point, ScreenshotGeometry};
+use keys::KeyCombo;
+use risk::{AxElement, Focus};
+use serde::{Deserialize, Serialize};
+
+#[cfg(target_os = "macos")]
+pub use macos::{request_permission, MacDesktop};
+
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum MouseButton {
+ Left,
+ Right,
+}
+
+/// One input the helper posts. Points are global, in points.
+#[derive(Debug, Clone, PartialEq)]
+pub enum InputEvent {
+ Move(Point),
+ Click { at: Point, button: MouseButton, count: u8 },
+ Drag { from: Point, to: Point },
+ /// `dy > 0` scrolls down and `dx > 0` scrolls right, in pixels.
+ Scroll { at: Point, dx: i32, dy: i32 },
+ Key(KeyCombo),
+ /// A chunk of text without newlines (newlines are sent as Return).
+ Text(String),
+}
+
+/// An encoded screenshot of the display under the pointer, Gentle Dot's windows excluded.
+#[derive(Debug, Clone)]
+pub struct Capture {
+ pub data: Vec<u8>,
+ pub mime_type: &'static str,
+ pub geometry: ScreenshotGeometry,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
+#[serde(rename_all = "camelCase")]
+pub struct AppInfo {
+ pub name: String,
+ pub bundle_id: Option<String>,
+ pub active: bool,
+}
+
+/// The `computer_permissions` reply (L60), plus `screenshotsSupported` (false before macOS 14,
+/// where ScreenCaptureKit cannot take screenshots).
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
+#[serde(rename_all = "camelCase")]
+pub struct Permissions {
+ pub accessibility: bool,
+ pub screen_recording: bool,
+ pub screenshots_supported: bool,
+}
+
+/// Which macOS permission the user is asked for (`computer_request_permission {kind}`, L60).
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub enum PermissionKind {
+ Accessibility,
+ ScreenRecording,
+}
+
+/// The Mac, as the helper sees it.
+pub trait Desktop: Send + Sync {
+ fn permissions(&self) -> Permissions;
+ /// A screenshot of the display under the pointer, downscaled to `coords::MAX_LONG_EDGE`.
+ fn capture(&self) -> Result<Capture, String>;
+ fn post(&self, event: &InputEvent) -> Result<(), String>;
+ fn list_apps(&self) -> Result<Vec<AppInfo>, String>;
+ /// Opens (or activates) an app by name or bundle id.
+ fn open_app(&self, name: &str) -> Result<(), String>;
+ /// The Accessibility element at a point, then its ancestors; empty when unknown.
+ fn element_at(&self, at: Point) -> Vec<AxElement>;
+ fn focused_element(&self) -> Option<Focus>;
+ fn frontmost_app(&self) -> Option<AppIdentity>;
+ /// The app owning the topmost window under a point.
+ fn window_owner_at(&self, at: Point) -> Option<AppIdentity>;
+ /// Brings an app back to the front (after a dialog took the focus).
+ fn activate(&self, pid: i32);
+}
+
+/// Native dialogs owned by the app, never the webview (S24.3, S24.4).
+pub trait Dialogs: Send + Sync {
+ /// "Allow Gentle Dot to see and control this Mac for 30 minutes?" (Allow / Deny).
+ fn ask_grant(&self) -> bool;
+ /// A per-action confirmation showing the action and its target.
+ fn confirm(&self, message: &str) -> bool;
+}
+
+pub trait Clock: Send + Sync {
+ /// Unix epoch milliseconds.
+ fn now_ms(&self) -> u64;
+ fn sleep_ms(&self, ms: u64);
+}
+
+pub struct SystemClock;
+
+impl Clock for SystemClock {
+ fn now_ms(&self) -> u64 {
+ std::time::SystemTime::now()
+ .duration_since(std::time::UNIX_EPOCH)
+ .map(|d| d.as_millis() as u64)
+ .unwrap_or_default()
+ }
+
+ fn sleep_ms(&self, ms: u64) {
+ std::thread::sleep(std::time::Duration::from_millis(ms));
+ }
+}
diff --git a/apps/desktop/src-tauri/src/computer/rate.rs b/apps/desktop/src-tauri/src/computer/rate.rs
new file mode 100644
index 0000000..d43dd9a
--- /dev/null
+++ b/apps/desktop/src-tauri/src/computer/rate.rs
@@ -0,0 +1,65 @@
+//! The action rate limit (S24.4): at most `max` tool calls in any sliding window.
+
+use std::collections::VecDeque;
+
+pub const MAX_ACTIONS_PER_SECOND: usize = 10;
+
+pub struct RateLimiter {
+ max: usize,
+ window_ms: u64,
+ accepted: VecDeque<u64>,
+}
+
+impl RateLimiter {
+ pub fn new(max: usize, window_ms: u64) -> Self {
+ RateLimiter { max, window_ms, accepted: VecDeque::new() }
+ }
+
+ /// Counts a call at `now_ms` and returns true, or returns false (not counted) when
+ /// `max` calls were already accepted within the last `window_ms`.
+ pub fn try_acquire(&mut self, now_ms: u64) -> bool {
+ while self.accepted.front().is_some_and(|&t| t + self.window_ms <= now_ms) {
+ self.accepted.pop_front();
+ }
+ if self.accepted.len() >= self.max {
+ return false;
+ }
+ self.accepted.push_back(now_ms);
+ true
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn ten_calls_within_a_second_are_accepted_and_the_eleventh_is_not() {
+ let mut limiter = RateLimiter::new(MAX_ACTIONS_PER_SECOND, 1000);
+ for i in 0..10 {
+ assert!(limiter.try_acquire(1_000 + i * 50), "call {i}");
+ }
+ assert!(!limiter.try_acquire(1_999));
+ }
+
+ #[test]
+ fn calls_are_accepted_again_once_the_window_slides() {
+ let mut limiter = RateLimiter::new(2, 1000);
+ assert!(limiter.try_acquire(0));
+ assert!(limiter.try_acquire(500));
+ assert!(!limiter.try_acquire(999));
+ assert!(limiter.try_acquire(1000));
+ assert!(!limiter.try_acquire(1499));
+ assert!(limiter.try_acquire(1500));
+ }
+
+ #[test]
+ fn rejected_calls_do_not_count() {
+ let mut limiter = RateLimiter::new(1, 1000);
+ assert!(limiter.try_acquire(0));
+ for t in [100, 200, 900] {
+ assert!(!limiter.try_acquire(t));
+ }
+ assert!(limiter.try_acquire(1000));
+ }
+}
diff --git a/apps/desktop/src-tauri/src/computer/risk.rs b/apps/desktop/src-tauri/src/computer/risk.rs
new file mode 100644
index 0000000..049d8a6
--- /dev/null
+++ b/apps/desktop/src-tauri/src/computer/risk.rs
@@ -0,0 +1,237 @@
+//! Which actions need a native per-action confirmation (S24.4). A declared intent only adds
+//! checks: an action is risky when the target is risky or the intent is.
+
+/// Words that make an action risky, in English and Spanish. `place order` is a phrase.
+const RISKY: &[&str] = &[
+ "send", "pay", "buy", "purchase", "delete", "remove", "submit", "confirm", "transfer", "place order",
+ "checkout", "enviar", "pagar", "comprar", "eliminar", "borrar", "confirmar", "transferir",
+];
+
+/// Extra words that make a text field send-like (a chat or comment composer), where
+/// Return usually sends.
+const SEND_LIKE_FIELD: &[&str] = &[
+ "message", "messages", "reply", "comment", "chat", "post", "mensaje", "mensajes", "responder", "respuesta",
+ "comentario", "comentar", "publicar",
+];
+
+/// Roles a click can press: buttons, menu items, and links (web pages often render buttons as links).
+const PRESSABLE_ROLES: &[&str] = &["AXButton", "AXMenuButton", "AXMenuItem", "AXMenuBarItem", "AXLink"];
+const TEXT_INPUT_ROLES: &[&str] = &["AXTextField", "AXTextArea", "AXComboBox"];
+
+/// What the Accessibility API reports about an element.
+#[derive(Debug, Clone, Default, PartialEq, Eq)]
+pub struct AxElement {
+ pub role: String,
+ pub title: Option<String>,
+ pub description: Option<String>,
+ pub value: Option<String>,
+}
+
+/// The focused element and its window's default button, if any.
+#[derive(Debug, Clone, Default, PartialEq, Eq)]
+pub struct Focus {
+ pub element: AxElement,
+ pub is_default_button: bool,
+ pub default_button: Option<AxElement>,
+}
+
+/// The facts the classifier needs about an action.
+#[derive(Debug, Clone, Copy)]
+pub enum Act<'a> {
+ /// A click: the element under the point first, then its ancestors.
+ Click(&'a [AxElement]),
+ /// Return (or Enter), from `key` or a newline in `type`.
+ Return(Option<&'a Focus>),
+ /// Every other action: only the intent can make it risky.
+ Other,
+}
+
+fn words(text: &str) -> Vec<String> {
+ text.split(|c: char| !c.is_alphanumeric()).filter(|w| !w.is_empty()).map(str::to_lowercase).collect()
+}
+
+/// The first entry of `list` found in `text` as whole words, in order, case-insensitively.
+fn find_in(text: &str, list: &[&'static str]) -> Option<&'static str> {
+ let words = words(text);
+ list.iter().copied().find(|entry| {
+ let entry: Vec<&str> = entry.split(' ').collect();
+ words.windows(entry.len()).any(|window| window.iter().zip(&entry).all(|(a, b)| a == b))
+ })
+}
+
+/// The first risky word or phrase in `text`, matched on whole words, case-insensitively.
+pub fn risky_word(text: &str) -> Option<&'static str> {
+ find_in(text, RISKY)
+}
+
+impl AxElement {
+ /// The element's title, description, and value, joined for matching and messages.
+ pub fn label(&self) -> String {
+ [&self.title, &self.description, &self.value]
+ .into_iter()
+ .flatten()
+ .map(|s| s.trim())
+ .filter(|s| !s.is_empty())
+ .collect::<Vec<_>>()
+ .join(" ")
+ }
+
+ fn is_pressable(&self) -> bool {
+ PRESSABLE_ROLES.contains(&self.role.as_str())
+ }
+}
+
+fn kind(role: &str) -> &str {
+ match role {
+ "AXMenuItem" | "AXMenuBarItem" => "menu item",
+ "AXLink" => "link",
+ "AXTextField" | "AXTextArea" | "AXComboBox" => "field",
+ _ => "button",
+ }
+}
+
+/// A pressable element in `chain` (the element under the point or an ancestor) whose own
+/// text, or the text of the elements inside it, is risky.
+fn risky_click(chain: &[AxElement]) -> Option<String> {
+ let pressable = chain.iter().position(AxElement::is_pressable)?;
+ let label = chain[..=pressable].iter().map(AxElement::label).collect::<Vec<_>>().join(" ");
+ risky_word(&label).map(|_| format!("the {} “{}”", kind(&chain[pressable].role), label.trim()))
+}
+
+fn risky_return(focus: &Focus) -> Option<String> {
+ let element = &focus.element;
+ let label = element.label();
+ if focus.is_default_button {
+ return Some(format!("Return presses the default button “{label}”"));
+ }
+ if element.is_pressable() && risky_word(&label).is_some() {
+ return Some(format!("Return presses the {} “{label}”", kind(&element.role)));
+ }
+ let field = TEXT_INPUT_ROLES.contains(&element.role.as_str());
+ if field && (risky_word(&label).is_some() || find_in(&label, SEND_LIKE_FIELD).is_some()) {
+ return Some(format!("Return may send the field “{label}”"));
+ }
+ let default = focus.default_button.as_ref().map(AxElement::label)?;
+ risky_word(&default).map(|_| format!("Return may press the default button “{default}”"))
+}
+
+/// Why `act` needs a confirmation, or `None` when it does not.
+pub fn assess(act: Act<'_>, intent: Option<&str>) -> Option<String> {
+ let target = match act {
+ Act::Click(chain) => risky_click(chain),
+ Act::Return(focus) => focus.and_then(risky_return),
+ Act::Other => None,
+ };
+ let intent = intent.filter(|i| risky_word(i).is_some()).map(|i| format!("the declared intent “{}”", i.trim()));
+ match (target, intent) {
+ (Some(target), Some(intent)) => Some(format!("{target}; {intent}")),
+ (target, intent) => target.or(intent),
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ fn element(role: &str, title: &str) -> AxElement {
+ AxElement { role: role.into(), title: Some(title.into()), ..AxElement::default() }
+ }
+
+ fn focus(element: AxElement) -> Focus {
+ Focus { element, ..Focus::default() }
+ }
+
+ #[test]
+ fn english_and_spanish_words_match_case_insensitively() {
+ assert_eq!(risky_word("SEND now"), Some("send"));
+ assert_eq!(risky_word("Pagar ahora"), Some("pagar"));
+ assert_eq!(risky_word("Eliminar conversación"), Some("eliminar"));
+ assert_eq!(risky_word("Proceed to Checkout"), Some("checkout"));
+ }
+
+ #[test]
+ fn words_match_whole_words_and_the_phrase_in_order() {
+ assert_eq!(risky_word("Place Order"), Some("place order"));
+ assert_eq!(risky_word("order a place"), None);
+ assert_eq!(risky_word("Sender"), None);
+ assert_eq!(risky_word("Paypal"), None);
+ assert_eq!(risky_word("send-button"), Some("send"));
+ assert_eq!(risky_word(""), None);
+ }
+
+ #[test]
+ fn clicking_a_risky_button_or_menu_item_is_risky() {
+ assert!(assess(Act::Click(&[element("AXButton", "Send")]), None).is_some());
+ assert!(assess(Act::Click(&[element("AXMenuItem", "Delete Message")]), None).is_some());
+ let described = AxElement { role: "AXButton".into(), description: Some("Comprar".into()), ..AxElement::default() };
+ assert!(assess(Act::Click(&[described]), None).is_some());
+ }
+
+ #[test]
+ fn text_inside_a_risky_button_counts() {
+ let label = AxElement { role: "AXStaticText".into(), value: Some("Buy now".into()), ..AxElement::default() };
+ let chain = [label, AxElement { role: "AXButton".into(), ..AxElement::default() }];
+ assert!(assess(Act::Click(&chain), None).is_some());
+ }
+
+ #[test]
+ fn harmless_clicks_are_not_risky() {
+ assert_eq!(assess(Act::Click(&[element("AXButton", "Open")]), None), None);
+ // A risky word outside a button (plain text, a text field) is not a press.
+ assert_eq!(assess(Act::Click(&[element("AXStaticText", "Send")]), None), None);
+ assert_eq!(assess(Act::Click(&[element("AXTextField", "Delete")]), None), None);
+ assert_eq!(assess(Act::Click(&[]), None), None);
+ }
+
+ #[test]
+ fn return_on_a_default_button_is_risky() {
+ let default = Focus { element: element("AXButton", "OK"), is_default_button: true, default_button: None };
+ assert!(assess(Act::Return(Some(&default)), None).is_some());
+ }
+
+ #[test]
+ fn return_on_a_focused_risky_button_is_risky() {
+ assert!(assess(Act::Return(Some(&focus(element("AXButton", "Transfer")))), None).is_some());
+ }
+
+ #[test]
+ fn return_in_a_send_like_field_is_risky() {
+ let composer = AxElement { role: "AXTextArea".into(), description: Some("Message #general".into()), ..AxElement::default() };
+ assert!(assess(Act::Return(Some(&focus(composer))), None).is_some());
+ let spanish = AxElement { role: "AXTextField".into(), title: Some("Escribe un mensaje".into()), ..AxElement::default() };
+ assert!(assess(Act::Return(Some(&focus(spanish))), None).is_some());
+ }
+
+ #[test]
+ fn return_with_a_risky_default_button_in_the_window_is_risky() {
+ let field = Focus {
+ element: element("AXTextField", "Amount"),
+ is_default_button: false,
+ default_button: Some(element("AXButton", "Pay")),
+ };
+ assert!(assess(Act::Return(Some(&field)), None).is_some());
+ }
+
+ #[test]
+ fn return_in_an_ordinary_field_is_not_risky() {
+ let search = Focus {
+ element: element("AXTextField", "Search"),
+ is_default_button: false,
+ default_button: Some(element("AXButton", "Done")),
+ };
+ assert_eq!(assess(Act::Return(Some(&search)), None), None);
+ assert_eq!(assess(Act::Return(None), None), None);
+ }
+
+ #[test]
+ fn a_risky_intent_makes_any_action_risky() {
+ assert!(assess(Act::Other, Some("borrar el archivo")).is_some());
+ assert!(assess(Act::Click(&[element("AXButton", "Open")]), Some("purchase the plan")).is_some());
+ }
+
+ #[test]
+ fn a_harmless_intent_never_removes_a_check() {
+ assert_eq!(assess(Act::Other, Some("open the menu")), None);
+ assert!(assess(Act::Click(&[element("AXButton", "Send")]), Some("just looking")).is_some());
+ }
+}
diff --git a/apps/desktop/src-tauri/src/computer/server.rs b/apps/desktop/src-tauri/src/computer/server.rs
new file mode 100644
index 0000000..048ab02
--- /dev/null
+++ b/apps/desktop/src-tauri/src/computer/server.rs
@@ -0,0 +1,125 @@
+//! Serves `Helper` on `127.0.0.1:<random port>/mcp` with a fresh random token per launch
+//! (S24.1). The token lives only in memory; the webview gets it from `computer_endpoint`.
+
+use super::mcp::{Helper, HttpRequest};
+use serde::Serialize;
+use std::io::Read;
+use std::sync::Arc;
+use std::thread;
+
+/// The `computer_endpoint` reply (L60).
+#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
+pub struct Endpoint {
+ pub url: String,
+ pub token: String,
+}
+
+/// 32 random bytes, hex encoded.
+pub fn new_token() -> String {
+ let mut bytes = [0u8; 32];
+ getrandom::fill(&mut bytes).expect("the OS random source is unavailable");
+ bytes.iter().map(|b| format!("{b:02x}")).collect()
+}
+
+/// Binds a random loopback port and serves each request on its own thread (actions are
+/// serialized inside `Helper`).
+pub fn start(helper: Arc<Helper>, token: String) -> std::io::Result<Endpoint> {
+ let server = tiny_http::Server::http("127.0.0.1:0").map_err(std::io::Error::other)?;
+ let port = server.server_addr().to_ip().map(|a| a.port()).ok_or_else(|| std::io::Error::other("no port"))?;
+ thread::Builder::new().name("computer-mcp".into()).spawn(move || {
+ for request in server.incoming_requests() {
+ let helper = helper.clone();
+ thread::spawn(move || serve(&helper, request));
+ }
+ })?;
+ Ok(Endpoint { url: format!("http://127.0.0.1:{port}/mcp"), token })
+}
+
+/// Requests larger than this are refused: tool calls are small.
+const MAX_BODY: u64 = 1 << 20;
+
+fn serve(helper: &Helper, mut request: tiny_http::Request) {
+ let authorization = request
+ .headers()
+ .iter()
+ .find(|h| h.field.equiv("Authorization"))
+ .map(|h| h.value.as_str().to_string());
+ let mut body = String::new();
+ let read = request.as_reader().take(MAX_BODY).read_to_string(&mut body);
+ let method = request.method().as_str().to_string();
+ let path = request.url().split('?').next().unwrap_or_default().to_string();
+ let reply = match read {
+ Ok(_) => helper.handle(&HttpRequest {
+ method: &method,
+ path: &path,
+ authorization: authorization.as_deref(),
+ body: &body,
+ }),
+ Err(_) => super::mcp::HttpReply { status: 400, body: None },
+ };
+ let is_json = reply.body.is_some();
+ let text = reply.body.map(|b| b.to_string()).unwrap_or_default();
+ let mut response = tiny_http::Response::from_string(text).with_status_code(reply.status);
+ if is_json {
+ response.add_header(tiny_http::Header::from_bytes("Content-Type", "application/json").expect("valid header"));
+ }
+ if reply.status == 401 {
+ response.add_header(tiny_http::Header::from_bytes("WWW-Authenticate", "Bearer").expect("valid header"));
+ }
+ let _ = request.respond(response);
+}
+
+#[cfg(test)]
+mod tests {
+ use super::super::control::Control;
+ use super::super::fake::{FakeClock, FakeDesktop, FakeDialogs};
+ use super::*;
+ use std::io::Write;
+ use std::net::TcpStream;
+
+ #[test]
+ fn tokens_are_long_random_and_fresh() {
+ let (a, b) = (new_token(), new_token());
+ assert_eq!(a.len(), 64);
+ assert!(a.chars().all(|c| c.is_ascii_hexdigit()));
+ assert_ne!(a, b);
+ }
+
+ fn exchange(url: &str, request: &str) -> String {
+ let address = url.trim_start_matches("http://").trim_end_matches("/mcp");
+ let mut stream = TcpStream::connect(address).unwrap();
+ stream.write_all(request.as_bytes()).unwrap();
+ let mut reply = String::new();
+ stream.read_to_string(&mut reply).unwrap();
+ reply
+ }
+
+ #[test]
+ fn the_loopback_server_checks_the_token_and_answers_json() {
+ let token = new_token();
+ let helper = Helper::new(
+ Arc::new(FakeDesktop::default()),
+ Arc::new(FakeDialogs::new(false, false)),
+ Arc::new(FakeClock::new(0)),
+ Arc::new(Control::new(|_| {})),
+ token.clone(),
+ 1,
+ );
+ let endpoint = start(Arc::new(helper), token.clone()).unwrap();
+ assert!(endpoint.url.starts_with("http://127.0.0.1:") && endpoint.url.ends_with("/mcp"));
+
+ let body = r#"{"jsonrpc":"2.0","id":1,"method":"tools/list"}"#;
+ let request = |auth: &str| {
+ format!(
+ "POST /mcp HTTP/1.1\r\nHost: 127.0.0.1\r\n{auth}Content-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}",
+ body.len()
+ )
+ };
+ let denied = exchange(&endpoint.url, &request(""));
+ assert!(denied.starts_with("HTTP/1.1 401"), "{denied}");
+ let allowed = exchange(&endpoint.url, &request(&format!("Authorization: Bearer {token}\r\n")));
+ assert!(allowed.starts_with("HTTP/1.1 200"), "{allowed}");
+ assert!(allowed.to_ascii_lowercase().contains("content-type: application/json"), "{allowed}");
+ assert!(allowed.contains("\"screenshot\""), "{allowed}");
+ }
+}
diff --git a/apps/desktop/src-tauri/src/computer/session.rs b/apps/desktop/src-tauri/src/computer/session.rs
new file mode 100644
index 0000000..be3952b
--- /dev/null
+++ b/apps/desktop/src-tauri/src/computer/session.rs
@@ -0,0 +1,138 @@
+//! The computer-control session (S24.3): idle until the user allows it, then granted until
+//! `ends_at`, ended by the Stop button, the tray item, the panic shortcut, or the timeout.
+//! Every end bumps `epoch`, so actions queued under an earlier epoch are dropped.
+
+use serde::Serialize;
+
+pub const SESSION_MS: u64 = 30 * 60 * 1000;
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
+#[serde(rename_all = "lowercase")]
+pub enum Reason {
+ Granted,
+ Stopped,
+ Panic,
+ Timeout,
+}
+
+/// The `computer://state` event and the `computer_status` reply (L60). `ends_at` is in
+/// Unix epoch milliseconds.
+#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
+#[serde(rename_all = "camelCase")]
+pub struct StateEvent {
+ pub active: bool,
+ #[serde(skip_serializing_if = "Option::is_none")]
+ pub ends_at: Option<u64>,
+ #[serde(skip_serializing_if = "Option::is_none")]
+ pub reason: Option<Reason>,
+}
+
+#[derive(Debug, Default)]
+pub struct Session {
+ ends_at: Option<u64>,
+ epoch: u64,
+}
+
+impl Session {
+ pub fn is_active(&self, now_ms: u64) -> bool {
+ self.ends_at.is_some_and(|end| now_ms < end)
+ }
+
+ pub fn epoch(&self) -> u64 {
+ self.epoch
+ }
+
+ pub fn status(&self, now_ms: u64) -> StateEvent {
+ let active = self.is_active(now_ms);
+ StateEvent { active, ends_at: self.ends_at.filter(|_| active), reason: None }
+ }
+
+ /// Starts (or restarts) a session of `SESSION_MS`.
+ pub fn grant(&mut self, now_ms: u64) -> StateEvent {
+ let ends_at = now_ms + SESSION_MS;
+ self.ends_at = Some(ends_at);
+ StateEvent { active: true, ends_at: Some(ends_at), reason: Some(Reason::Granted) }
+ }
+
+ /// Ends the session for `reason`. `None` when there was none to end.
+ pub fn end(&mut self, reason: Reason) -> Option<StateEvent> {
+ self.ends_at.take()?;
+ self.epoch += 1;
+ Some(StateEvent { active: false, ends_at: None, reason: Some(reason) })
+ }
+
+ /// Ends the session with `Timeout` once `now_ms` reaches its end.
+ pub fn expire(&mut self, now_ms: u64) -> Option<StateEvent> {
+ if self.ends_at.is_some_and(|end| now_ms >= end) {
+ self.end(Reason::Timeout)
+ } else {
+ None
+ }
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ const NOW: u64 = 1_700_000_000_000;
+
+ #[test]
+ fn a_new_session_is_idle() {
+ let session = Session::default();
+ assert!(!session.is_active(NOW));
+ assert_eq!(session.status(NOW), StateEvent { active: false, ends_at: None, reason: None });
+ }
+
+ #[test]
+ fn granting_starts_thirty_minutes() {
+ let mut session = Session::default();
+ let event = session.grant(NOW);
+ assert_eq!(event, StateEvent { active: true, ends_at: Some(NOW + SESSION_MS), reason: Some(Reason::Granted) });
+ assert!(session.is_active(NOW + SESSION_MS - 1));
+ assert_eq!(session.status(NOW + 5).ends_at, Some(NOW + SESSION_MS));
+ }
+
+ #[test]
+ fn stop_and_panic_end_the_session_and_bump_the_epoch() {
+ for reason in [Reason::Stopped, Reason::Panic] {
+ let mut session = Session::default();
+ session.grant(NOW);
+ let epoch = session.epoch();
+ assert_eq!(session.end(reason), Some(StateEvent { active: false, ends_at: None, reason: Some(reason) }));
+ assert!(!session.is_active(NOW));
+ assert_eq!(session.epoch(), epoch + 1);
+ }
+ }
+
+ #[test]
+ fn ending_an_idle_session_reports_nothing() {
+ let mut session = Session::default();
+ assert_eq!(session.end(Reason::Stopped), None);
+ assert_eq!(session.epoch(), 0);
+ }
+
+ #[test]
+ fn the_session_times_out_at_its_end() {
+ let mut session = Session::default();
+ session.grant(NOW);
+ assert_eq!(session.expire(NOW + SESSION_MS - 1), None);
+ assert!(!session.is_active(NOW + SESSION_MS));
+ assert_eq!(
+ session.expire(NOW + SESSION_MS),
+ Some(StateEvent { active: false, ends_at: None, reason: Some(Reason::Timeout) })
+ );
+ assert_eq!(session.epoch(), 1);
+ assert_eq!(session.expire(NOW + SESSION_MS + 1), None);
+ }
+
+ #[test]
+ fn the_event_serializes_like_the_l60_contract() {
+ let mut session = Session::default();
+ let granted = serde_json::to_value(session.grant(NOW)).unwrap();
+ assert_eq!(granted, serde_json::json!({"active": true, "endsAt": NOW + SESSION_MS, "reason": "granted"}));
+ let panic = serde_json::to_value(session.end(Reason::Panic).unwrap()).unwrap();
+ assert_eq!(panic, serde_json::json!({"active": false, "reason": "panic"}));
+ assert_eq!(serde_json::to_value(session.status(NOW)).unwrap(), serde_json::json!({"active": false}));
+ }
+}
diff --git a/apps/desktop/src-tauri/src/lib.rs b/apps/desktop/src-tauri/src/lib.rs
index 677e03c..37e517c 100644
--- a/apps/desktop/src-tauri/src/lib.rs
+++ b/apps/desktop/src-tauri/src/lib.rs
@@ -1,3 +1,4 @@
+pub mod computer;
pub mod config;
pub mod daemon;
pub mod geometry;
diff --git a/apps/desktop/src-tauri/src/shell.rs b/apps/desktop/src-tauri/src/shell.rs
index d9f7477..be80bb1 100644
--- a/apps/desktop/src-tauri/src/shell.rs
+++ b/apps/desktop/src-tauri/src/shell.rs
@@ -1,6 +1,8 @@
//! Tauri wiring: windows, commands, tray menu, global shortcut, Dot snapping,
//! and the daemon lifecycle (design §9).
+use crate::computer::app as computer;
+use crate::computer::session::Reason;
use crate::config::{self, ConnectionInfo, DesktopConfig, DEFAULT_SHORTCUT};
use crate::daemon::{Daemon, RestartOutcome};
use crate::geometry::{self, Rect};
@@ -258,12 +260,17 @@ fn conversations_on(value: Option<&str>) -> bool {
/// The tray menu, top to bottom (`-` is a separator). "New conversation" only exists with the
/// conversations list on; in the single continuous chat there is nothing new to start.
-fn tray_menu_ids(conversations: bool) -> Vec<&'static str> {
+/// "Stop computer control" exists where computer control does (macOS, S24.3).
+fn tray_menu_ids(conversations: bool, computer_control: bool) -> Vec<&'static str> {
let mut ids = vec!["open"];
if conversations {
ids.push("new");
}
- ids.extend(["browser", "-", "restart", "autostart", "-", "quit"]);
+ ids.push("browser");
+ if computer_control {
+ ids.push("computer-stop");
+ }
+ ids.extend(["-", "restart", "autostart", "-", "quit"]);
ids
}
@@ -274,11 +281,18 @@ fn build_tray(app: &AppHandle, config: &DesktopConfig) -> tauri::Result<TrayIcon
.or_else(|_| MenuItem::with_id(app, "open", "Open", true, None::<&str>))?;
let menu = Menu::new(app)?;
let conversations = conversations_on(std::env::var("GENTLE_DOT_CONVERSATIONS").ok().as_deref());
- for id in tray_menu_ids(conversations) {
+ for id in tray_menu_ids(conversations, cfg!(target_os = "macos")) {
match id {
"open" => menu.append(&open)?,
"new" => menu.append(&MenuItem::with_id(app, "new", "New conversation", true, None::<&str>)?)?,
"browser" => menu.append(&MenuItem::with_id(app, "browser", "Open in browser", true, None::<&str>)?)?,
+ "computer-stop" => menu.append(&MenuItem::with_id(
+ app,
+ "computer-stop",
+ "Stop computer control",
+ true,
+ None::<&str>,
+ )?)?,
"restart" => {
menu.append(&MenuItem::with_id(app, "restart", "Restart assistant", true, None::<&str>)?)?;
}
@@ -305,6 +319,10 @@ fn build_tray(app: &AppHandle, config: &DesktopConfig) -> tauri::Result<TrayIcon
"open" => show_panel(app),
"new" => show_panel(app).and_then(|()| app.emit("dot://new-conversation", ()).map_err(err)),
"browser" => open_in_browser(app),
+ "computer-stop" => {
+ computer::stop(app, Reason::Stopped);
+ Ok(())
+ }
"restart" => {
restart_daemon(app.clone());
Ok(())
@@ -360,6 +378,19 @@ fn restart_daemon(app: AppHandle) {
});
}
+/// ⌥⇧Esc ends computer control from anywhere, even while another app is in front.
+#[cfg(target_os = "macos")]
+fn register_panic_shortcut(app: &AppHandle) {
+ let handler = |app: &AppHandle, _: &_, event: tauri_plugin_global_shortcut::ShortcutEvent| {
+ if event.state() == ShortcutState::Pressed {
+ computer::stop(app, Reason::Panic);
+ }
+ };
+ if let Err(error) = app.global_shortcut().on_shortcut(computer::PANIC_SHORTCUT, handler) {
+ eprintln!("gentle-dot: the computer-control panic shortcut is unavailable: {error}");
+ }
+}
+
fn register_shortcut(app: &AppHandle, shortcut: &str) {
let handler = |app: &AppHandle, _: &_, event: tauri_plugin_global_shortcut::ShortcutEvent| {
if event.state() == ShortcutState::Pressed {
@@ -403,7 +434,17 @@ pub fn run() {
.plugin(tauri_plugin_autostart::init(MacosLauncher::LaunchAgent, None))
.plugin(tauri_plugin_opener::init())
.plugin(tauri_plugin_dialog::init())
- .invoke_handler(tauri::generate_handler![connection_info, toggle_panel, hide_panel, set_dot_state])
+ .invoke_handler(tauri::generate_handler![
+ connection_info,
+ toggle_panel,
+ hide_panel,
+ set_dot_state,
+ computer::computer_endpoint,
+ computer::computer_permissions,
+ computer::computer_request_permission,
+ computer::computer_stop,
+ computer::computer_status
+ ])
.setup(move |app| {
#[cfg(target_os = "macos")]
app.set_activation_policy(tauri::ActivationPolicy::Accessory);
@@ -417,6 +458,9 @@ pub fn run() {
}
});
app.manage(Shell { config: config.clone(), daemon, snap: spawn_snapper(handle.clone()), places_windows });
+ app.manage(computer::start(&handle));
+ #[cfg(target_os = "macos")]
+ register_panic_shortcut(&handle);
build_windows(&handle, &config, places_windows)?;
build_tray(&handle, &config)?;
@@ -457,7 +501,7 @@ mod tests {
#[test]
fn tray_hides_new_conversation_in_the_single_chat() {
assert_eq!(
- tray_menu_ids(false),
+ tray_menu_ids(false, false),
vec!["open", "browser", "-", "restart", "autostart", "-", "quit"]
);
}
@@ -465,11 +509,19 @@ mod tests {
#[test]
fn tray_shows_new_conversation_with_the_conversations_list() {
assert_eq!(
- tray_menu_ids(true),
+ tray_menu_ids(true, false),
vec!["open", "new", "browser", "-", "restart", "autostart", "-", "quit"]
);
}
+ #[test]
+ fn tray_offers_stop_computer_control_where_it_exists() {
+ assert_eq!(
+ tray_menu_ids(false, true),
+ vec!["open", "browser", "computer-stop", "-", "restart", "autostart", "-", "quit"]
+ );
+ }
+
#[test]
fn conversations_flag_reads_like_the_daemon() {
assert!(conversations_on(Some("1")));
diff --git a/apps/desktop/src-tauri/tauri.conf.json b/apps/desktop/src-tauri/tauri.conf.json
index 36589f3..f2cc106 100644
--- a/apps/desktop/src-tauri/tauri.conf.json
+++ b/apps/desktop/src-tauri/tauri.conf.json
@@ -28,7 +28,7 @@
"icons/icon.png"
],
"macOS": {
- "minimumSystemVersion": "11.0"
+ "minimumSystemVersion": "12.3"
}
}
}