A personal assistant that lives on your desktop, remembers what matters, and can use your Mac for you.
TypeScript
fix(desktop): let the helper word a backend refusal from what it already sent
Marcelito73bd20051835
diff --git a/apps/desktop/src-tauri/src/computer/fake.rs b/apps/desktop/src-tauri/src/computer/fake.rsindex b87a457..2ca8805 100644--- a/apps/desktop/src-tauri/src/computer/fake.rs+++ b/apps/desktop/src-tauri/src/computer/fake.rs@@ -3,7 +3,7 @@use super::blocklist::AppIdentity;use super::coords::{Point, ScreenshotGeometry};use super::risk::{AxElement, Focus};-use super::{AppInfo, Capture, Clock, Desktop, Dialogs, InputEvent, KeyboardRecipient, Permissions};+use super::{AppInfo, Capture, Clock, Desktop, Dialogs, InputEvent, KeyboardRecipient, Permissions, TypedError};use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering};use std::sync::Mutex;@@ -21,6 +21,9 @@ pub struct FakeDesktop {pub focused_app: Mutex<Option<AppIdentity>>,/// Changes the foreground app after this many recorded posts; disabled by default.pub front_after_posts: Mutex<Option<(usize, AppIdentity)>>,+ /// The backend refuses this typed post (1-based), sending nothing of it; disabled by default.+ pub refuse_typed_at: Mutex<Option<(usize, TypedError)>>,+ pub typed_calls: Mutex<usize>,/// Apps that run as administrator, refused for keyboard input.pub elevated: Mutex<Vec<i32>>,/// Marks an app as elevated after this many recorded posts; disabled by default.@@ -58,6 +61,8 @@ impl Default for FakeDesktop {frontmost: Mutex::new(Some(safari)),focused_app: Mutex::new(None),front_after_posts: Mutex::new(None),+ refuse_typed_at: Mutex::new(None),+ typed_calls: Mutex::new(0),elevated: Mutex::new(Vec::new()),elevated_after_posts: Mutex::new(None),front_during_capture: Mutex::new(None),@@ -130,9 +135,19 @@ impl Desktop for FakeDesktop {Ok(())}- fn post_typed(&self, event: &InputEvent, recipient: &KeyboardRecipient) -> Result<(), String> {+ fn post_typed(&self, event: &InputEvent, recipient: &KeyboardRecipient) -> Result<(), TypedError> {+ let call = {+ let mut calls = self.typed_calls.lock().unwrap();+ *calls += 1;+ *calls+ };+ if let Some((at, error)) = self.refuse_typed_at.lock().unwrap().as_ref() {+ if call == *at {+ return Err(error.clone());+ }+ }// Only posts that went through, so this stays in step with `events`.- self.post(event)?;+ self.post(event).map_err(TypedError::Failed)?;self.typed_posts.lock().unwrap().push((event.clone(), *recipient));Ok(())}diff --git a/apps/desktop/src-tauri/src/computer/mcp.rs b/apps/desktop/src-tauri/src/computer/mcp.rsindex 6a22a25..a33b347 100644--- a/apps/desktop/src-tauri/src/computer/mcp.rs+++ b/apps/desktop/src-tauri/src/computer/mcp.rs@@ -26,7 +26,7 @@ use super::rate::{RateLimiter, MAX_ACTIONS_PER_SECOND};use super::risk::{assess, Act};use super::working::{NoPreparation, Preparation};use super::{- Clock, Desktop, Dialogs, InputEvent, KeyboardRecipient, MouseButton, KEYS_CHANGED_BEFORE_TYPING,+ Clock, Desktop, Dialogs, InputEvent, KeyboardRecipient, MouseButton, TypedError, KEYS_CHANGED_BEFORE_TYPING,KEYS_CHANGED_WHILE_TYPING,};use base64::Engine as _;@@ -828,8 +828,14 @@ impl Helper {// A stop before the post sent nothing more; only a failed post leaves a range uncertain.self.still_running(epoch).map_err(|error| format!("{error} Typing stopped after {posted} of {total} characters."))?;- self.desktop.post_typed(&event, &recipient).map_err(|error| {- format!("{error} Typing stopped within characters {}-{} of {total}.", posted + 1, posted + characters)+ self.desktop.post_typed(&event, &recipient).map_err(|error| match error {+ TypedError::Failed(error) => format!(+ "{error} Typing stopped within characters {}-{} of {total}.",+ posted + 1,+ posted + characters+ ),+ // Refused before any of this post went out: what was sent is certain.+ refused => format!("{} Typing stopped after {posted} of {total} characters.", refused.message(posted > 0)),})?;posted += characters;Ok(())@@ -868,7 +874,7 @@ impl Helper {return Err("The app receiving the keys changed before the key was sent, so it was not sent.".into());}self.still_running(epoch)?;- self.desktop.post_typed(&InputEvent::Key(combo), &recipient)?;+ self.desktop.post_typed(&InputEvent::Key(combo), &recipient).map_err(|error| error.message(false))?;(self.glow)(Mark::Key);Ok((format!("{action} in {target}: done."), SETTLE_MS))}@@ -1622,6 +1628,41 @@ mod tests {elevated_front_refuses("key", json!({"combo": "cmd+a"}));}+ /// The backend refusing a post before sending any of it (#167): the helper words it from what+ /// earlier posts already sent, and reports no uncertain range.+ #[test]+ fn a_backend_refusal_is_reported_after_the_characters_already_sent() {+ for (at, error, expected, sent) in [+ (2, TypedError::RecipientChanged,+ "The app receiving the keys changed while typing, so the rest was not sent. Typing stopped after 20 of 45 characters.", 1),+ (1, TypedError::RecipientChanged,+ "The app receiving the keys changed before typing, so nothing was sent. Typing stopped after 0 of 45 characters.", 0),+ (2, TypedError::FrontUnknown,+ "Cannot tell which window is in front, so the rest was not sent. Typing stopped after 20 of 45 characters.", 1),+ (1, TypedError::FrontUnknown,+ "Cannot tell which window is in front, so nothing was sent. Typing stopped after 0 of 45 characters.", 0),+ ] {+ let rig = rig();+ rig.ready();+ *rig.desktop.refuse_typed_at.lock().unwrap() = Some((at, error.clone()));+ let result = rig.call("type", json!({"text": "a".repeat(45)}));+ assert!(is_error(&result), "{error:?} at {at}: {result}");+ assert_eq!(first_text(&result), expected, "{error:?} at {at}");+ assert_eq!(rig.desktop.events(), vec![InputEvent::Text("a".repeat(20)); sent], "{error:?} at {at}");+ }+ }++ #[test]+ fn a_backend_refusal_of_a_key_says_nothing_was_sent() {+ let rig = rig();+ rig.ready();+ *rig.desktop.refuse_typed_at.lock().unwrap() = Some((1, TypedError::RecipientChanged));+ let result = rig.call("key", json!({"combo": "tab"}));+ assert!(is_error(&result), "{result}");+ assert_eq!(first_text(&result), KEYS_CHANGED_BEFORE_TYPING);+ assert!(rig.desktop.events().is_empty());+ }+#[test]fn elevated_front_refuses_key_at_the_recheck() {let rig = rig();diff --git a/apps/desktop/src-tauri/src/computer/mod.rs b/apps/desktop/src-tauri/src/computer/mod.rsindex ece8b19..a3caf4b 100644--- a/apps/desktop/src-tauri/src/computer/mod.rs+++ b/apps/desktop/src-tauri/src/computer/mod.rs@@ -107,6 +107,33 @@ pub enum PermissionKind {pub const KEYS_CHANGED_WHILE_TYPING: &str = "The app receiving the keys changed while typing, so the rest was not sent.";pub const KEYS_CHANGED_BEFORE_TYPING: &str = "The app receiving the keys changed before typing, so nothing was sent.";+/// Why a typed post (`Desktop::post_typed`) did not go through. The first two are refusals+/// before anything of that post was sent; only the helper knows how much of the text went out+/// before it, so it words them (`message`).+#[derive(Debug, Clone, PartialEq, Eq)]+pub enum TypedError {+ /// The windows in front no longer belong to the approved recipient.+ RecipientChanged,+ /// The system did not say which window is in front.+ FrontUnknown,+ /// Posting failed; part of the post may have been sent.+ Failed(String),+}++impl TypedError {+ /// The error for the agent, given whether earlier posts of the same action already sent part+ /// of it.+ pub fn message(&self, sent: bool) -> String {+ match (self, sent) {+ (TypedError::RecipientChanged, false) => KEYS_CHANGED_BEFORE_TYPING.into(),+ (TypedError::RecipientChanged, true) => KEYS_CHANGED_WHILE_TYPING.into(),+ (TypedError::FrontUnknown, false) => "Cannot tell which window is in front, so nothing was sent.".into(),+ (TypedError::FrontUnknown, true) => "Cannot tell which window is in front, so the rest was not sent.".into(),+ (TypedError::Failed(error), _) => error.clone(),+ }+ }+}+/// Why keys are not sent to an app that runs as administrator (Windows drops them silently).pub fn runs_as_administrator(name: &str) -> String {format!("{name} runs as administrator, so Gentle Dot cannot send it keys.")@@ -132,8 +159,8 @@ pub trait Desktop: Send + Sync {fn post(&self, event: &InputEvent) -> Result<(), String>;/// Posts a type chunk or Return bound to the recipient approved by the shared guard./// Platforms with targeted input checks can validate it immediately before sending.- fn post_typed(&self, event: &InputEvent, _recipient: &KeyboardRecipient) -> Result<(), String> {- self.post(event)+ fn post_typed(&self, event: &InputEvent, _recipient: &KeyboardRecipient) -> Result<(), TypedError> {+ self.post(event).map_err(TypedError::Failed)}fn list_apps(&self) -> Result<Vec<AppInfo>, String>;/// Opens (or activates) an app by name or bundle id.diff --git a/apps/desktop/src-tauri/src/computer/win.rs b/apps/desktop/src-tauri/src/computer/win.rsindex 9b86d82..4fd3df2 100644--- a/apps/desktop/src-tauri/src/computer/win.rs+++ b/apps/desktop/src-tauri/src/computer/win.rs@@ -30,14 +30,16 @@ use super::winapps::{};use super::winput::{checks_typing_target, releases, send_batches, settles, steps, Step, VirtualScreen};use super::{- runs_as_administrator, AppInfo, Capture, InputEvent, KeyboardRecipient, MouseButton, KEYS_CHANGED_BEFORE_TYPING,+ runs_as_administrator, AppInfo, Capture, InputEvent, KeyboardRecipient, MouseButton, TypedError,KEYS_CHANGED_WHILE_TYPING,};use std::ffi::c_void;use std::mem::{size_of, zeroed};use std::ptr::null_mut;use std::time::Duration;-use windows_sys::Win32::Foundation::{CloseHandle, GetLastError, SetLastError, ERROR_TIMEOUT, HWND, LPARAM, POINT, RECT};+use windows_sys::Win32::Foundation::{+ CloseHandle, GetLastError, SetLastError, ERROR_ACCESS_DENIED, ERROR_TIMEOUT, HWND, LPARAM, POINT, RECT,+};use windows_sys::Win32::Graphics::Dwm::{DwmFlush, DwmGetWindowAttribute, DWMWA_CLOAKED};use windows_sys::Win32::Storage::FileSystem::{GetFileVersionInfoSizeW, GetFileVersionInfoW, VerQueryValueW};use windows_sys::Win32::System::Com::{CoInitializeEx, CoUninitialize, COINIT_APARTMENTTHREADED, COINIT_DISABLE_OLE1DDE};@@ -224,27 +226,27 @@ fn typing_windows() -> Option<(HWND, HWND)> {}}-const CANNOT_TELL_FRONT: &str = "Cannot tell which window is in front, so nothing was sent.";-/// Posts `event` with a short gap between steps (surrogate pairs form one batch), waiting after/// key releases or a complete pair for the window in front to take it. Text is bound to the/// window in front and its keyboard focus when it starts, and stops when either changes. A/// failure releases the keys and buttons left pressed by the successfully sent prefix before/// reporting the error.pub fn post(event: &InputEvent) -> Result<(), String> {- post_input(event, None)+ post_input(event, None).map_err(|error| error.message(false))}/// Posts a typed chunk, a Return, or a key only to the recipient the shared guard approved: the-/// windows it is bound to must still belong to the approved front app and focused app.-pub fn post_typed(event: &InputEvent, recipient: &KeyboardRecipient) -> Result<(), String> {+/// windows it is bound to must still belong to the approved front app and focused app. A refusal+/// before anything of it was sent is `RecipientChanged` or `FrontUnknown`, which the helper words+/// from what earlier chunks sent.+pub fn post_typed(event: &InputEvent, recipient: &KeyboardRecipient) -> Result<(), TypedError> {post_input(event, Some(recipient))}-fn post_input(event: &InputEvent, approved: Option<&KeyboardRecipient>) -> Result<(), String> {+fn post_input(event: &InputEvent, approved: Option<&KeyboardRecipient>) -> Result<(), TypedError> {// Bind both type chunks and their Returns. Standalone combos remain free to switch apps.let typing_target = if approved.is_some() || matches!(event, InputEvent::Text(_)) {- Some(typing_windows().ok_or(CANNOT_TELL_FRONT)?)+ Some(typing_windows().ok_or(TypedError::FrontUnknown)?)} else {None};@@ -267,26 +269,27 @@ fn post_input(event: &InputEvent, approved: Option<&KeyboardRecipient>) -> Resul&& focused == approved.focused_pid&& typing_windows() == Some((root, focus));if !valid {- return Err(KEYS_CHANGED_BEFORE_TYPING.into());+ return Err(TypedError::RecipientChanged);}}- let plan = steps(event, &virtual_screen())?;+ let plan = steps(event, &virtual_screen()).map_err(TypedError::Failed)?;let mut done = 0;for batch in send_batches(&plan) {let checked = checks_typing_target(&batch[0])|| (approved.is_some() && matches!(batch[0], Step::Key { down: true, .. }));let changed = checked && typing_target.is_some_and(|target| typing_windows() != Some(target));let failed = if changed {- let message = if done == 0 { KEYS_CHANGED_BEFORE_TYPING } else { KEYS_CHANGED_WHILE_TYPING };- Some((done, message.to_string()))+ // Before the first batch nothing of this post went out; after it, part of it did.+ let error = if done == 0 { TypedError::RecipientChanged } else { TypedError::Failed(KEYS_CHANGED_WHILE_TYPING.into()) };+ Some((done, error))} else {match send_batch(batch) {- Err((sent, error)) => Some((done + sent, error)),+ Err((sent, error)) => Some((done + sent, TypedError::Failed(error))),Ok(()) => {// No sleep or catch_up inside a surrogate pair's single SendInput call.std::thread::sleep(EVENT_GAP);if settles(batch.last().unwrap()) {- catch_up().err().map(|error| (done + batch.len(), error))+ catch_up().err().map(|error| (done + batch.len(), TypedError::Failed(error)))} else {None}@@ -1006,15 +1009,25 @@ fn resolve_name(wanted: &str, shortcuts: &[String]) -> Result<String, String> {}}-/// The integrity level of a process (the last sub-authority of its token's mandatory label), or-/// none when its token cannot be read.-fn integrity_level(process: *mut c_void) -> Option<u32> {+/// What reading a process's integrity level gave.+#[derive(Debug, PartialEq, Eq)]+enum Integrity {+ /// The last sub-authority of its token's mandatory label.+ Level(u32),+ /// Its token refused to be opened: a process of this user and session refuses it when it runs+ /// at a higher level (elevated).+ Denied,+ /// It could not be read.+ Unknown,+}++fn integrity_level(process: *mut c_void) -> Integrity {// SAFETY: the token is checked and closed; the label is read from a buffer of the size Windows// asked for, aligned for it, and its SID is validated before its sub-authorities are read.unsafe {let mut token = null_mut();if OpenProcessToken(process, TOKEN_QUERY, &mut token) == 0 {- return None;+ return if GetLastError() == ERROR_ACCESS_DENIED { Integrity::Denied } else { Integrity::Unknown };}let mut size = 0u32;GetTokenInformation(token, TokenIntegrityLevel, null_mut(), 0, &mut size);@@ -1023,27 +1036,31 @@ fn integrity_level(process: *mut c_void) -> Option<u32> {&& GetTokenInformation(token, TokenIntegrityLevel, buffer.as_mut_ptr() as *mut c_void, size, &mut size) != 0;CloseHandle(token);if !read {- return None;+ return Integrity::Unknown;}let sid = (*(buffer.as_ptr() as *const TOKEN_MANDATORY_LABEL)).Label.Sid;if sid.is_null() || IsValidSid(sid) == 0 {- return None;+ return Integrity::Unknown;}let count = *GetSidSubAuthorityCount(sid);- (count > 0).then(|| *GetSidSubAuthority(sid, u32::from(count) - 1))+ if count == 0 {+ return Integrity::Unknown;+ }+ Integrity::Level(*GetSidSubAuthority(sid, u32::from(count) - 1))}}/// Why keys must not go to `app`: Windows drops input sent to a process with a higher integrity/// level than ours (an app run as administrator) without an error, so typing there would report-/// keys that never arrived. A process whose level cannot be read is refused the same way.+/// keys that never arrived. A token that refuses to be opened is taken as elevated; a process+/// whose level cannot be read at all is refused too.pub fn keyboard_refusal(app: &AppIdentity) -> Option<String> {// SAFETY: the pseudo handle needs no closing; the target's handle is checked and closed.let (own, target) = unsafe {let own = integrity_level(GetCurrentProcess());let process = OpenProcess(PROCESS_QUERY_LIMITED_INFORMATION, 0, app.pid as u32);let target = if process.is_null() {- None+ Integrity::Unknown} else {let level = integrity_level(process);CloseHandle(process);@@ -1052,8 +1069,8 @@ pub fn keyboard_refusal(app: &AppIdentity) -> Option<String> {(own, target)};match (own, target) {- (Some(own), Some(target)) if target <= own => None,- (Some(_), Some(_)) => Some(runs_as_administrator(&app.name)),+ (Integrity::Level(own), Integrity::Level(target)) if target <= own => None,+ (Integrity::Level(_), Integrity::Level(_) | Integrity::Denied) => Some(runs_as_administrator(&app.name)),_ => Some(format!("Gentle Dot cannot tell whether {} runs as administrator, so it does not send it keys.",app.name@@ -1170,13 +1187,14 @@ mod tests {}/// A recipient the windows in front do not belong to gets nothing; on a desktop with nothing- /// in front (CI), that is said instead.+ /// in front, as on CI, the front is unknown instead, so CI rarely reaches the recipient check:+ /// the interactive run covers it.#[test]fn keys_for_another_recipient_are_not_posted() {let elsewhere = KeyboardRecipient { front_pid: i32::MAX, focused_pid: None };let result = post_typed(&InputEvent::Text("x".into()), &elsewhere);assert!(- result == Err(KEYS_CHANGED_BEFORE_TYPING.into()) || result == Err(CANNOT_TELL_FRONT.into()),+ result == Err(TypedError::RecipientChanged) || result == Err(TypedError::FrontUnknown),"{result:?}");}