Skip to content

A personal assistant that lives on your desktop, remembers what matters, and can use your Mac for you.

TypeScript

fix(desktop): let the helper word a backend refusal from what it already sent

Marcelito73bd20051835
diff --git a/apps/desktop/src-tauri/src/computer/fake.rs b/apps/desktop/src-tauri/src/computer/fake.rs
index b87a457..2ca8805 100644
--- a/apps/desktop/src-tauri/src/computer/fake.rs
+++ b/apps/desktop/src-tauri/src/computer/fake.rs
@@ -3,7 +3,7 @@
use super::blocklist::AppIdentity;
use super::coords::{Point, ScreenshotGeometry};
use super::risk::{AxElement, Focus};
-use super::{AppInfo, Capture, Clock, Desktop, Dialogs, InputEvent, KeyboardRecipient, Permissions};
+use super::{AppInfo, Capture, Clock, Desktop, Dialogs, InputEvent, KeyboardRecipient, Permissions, TypedError};
use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering};
use std::sync::Mutex;
@@ -21,6 +21,9 @@ pub struct FakeDesktop {
pub focused_app: Mutex<Option<AppIdentity>>,
/// Changes the foreground app after this many recorded posts; disabled by default.
pub front_after_posts: Mutex<Option<(usize, AppIdentity)>>,
+ /// The backend refuses this typed post (1-based), sending nothing of it; disabled by default.
+ pub refuse_typed_at: Mutex<Option<(usize, TypedError)>>,
+ pub typed_calls: Mutex<usize>,
/// Apps that run as administrator, refused for keyboard input.
pub elevated: Mutex<Vec<i32>>,
/// Marks an app as elevated after this many recorded posts; disabled by default.
@@ -58,6 +61,8 @@ impl Default for FakeDesktop {
frontmost: Mutex::new(Some(safari)),
focused_app: Mutex::new(None),
front_after_posts: Mutex::new(None),
+ refuse_typed_at: Mutex::new(None),
+ typed_calls: Mutex::new(0),
elevated: Mutex::new(Vec::new()),
elevated_after_posts: Mutex::new(None),
front_during_capture: Mutex::new(None),
@@ -130,9 +135,19 @@ impl Desktop for FakeDesktop {
Ok(())
}
- fn post_typed(&self, event: &InputEvent, recipient: &KeyboardRecipient) -> Result<(), String> {
+ fn post_typed(&self, event: &InputEvent, recipient: &KeyboardRecipient) -> Result<(), TypedError> {
+ let call = {
+ let mut calls = self.typed_calls.lock().unwrap();
+ *calls += 1;
+ *calls
+ };
+ if let Some((at, error)) = self.refuse_typed_at.lock().unwrap().as_ref() {
+ if call == *at {
+ return Err(error.clone());
+ }
+ }
// Only posts that went through, so this stays in step with `events`.
- self.post(event)?;
+ self.post(event).map_err(TypedError::Failed)?;
self.typed_posts.lock().unwrap().push((event.clone(), *recipient));
Ok(())
}
diff --git a/apps/desktop/src-tauri/src/computer/mcp.rs b/apps/desktop/src-tauri/src/computer/mcp.rs
index 6a22a25..a33b347 100644
--- a/apps/desktop/src-tauri/src/computer/mcp.rs
+++ b/apps/desktop/src-tauri/src/computer/mcp.rs
@@ -26,7 +26,7 @@ use super::rate::{RateLimiter, MAX_ACTIONS_PER_SECOND};
use super::risk::{assess, Act};
use super::working::{NoPreparation, Preparation};
use super::{
- Clock, Desktop, Dialogs, InputEvent, KeyboardRecipient, MouseButton, KEYS_CHANGED_BEFORE_TYPING,
+ Clock, Desktop, Dialogs, InputEvent, KeyboardRecipient, MouseButton, TypedError, KEYS_CHANGED_BEFORE_TYPING,
KEYS_CHANGED_WHILE_TYPING,
};
use base64::Engine as _;
@@ -828,8 +828,14 @@ impl Helper {
// A stop before the post sent nothing more; only a failed post leaves a range uncertain.
self.still_running(epoch)
.map_err(|error| format!("{error} Typing stopped after {posted} of {total} characters."))?;
- self.desktop.post_typed(&event, &recipient).map_err(|error| {
- format!("{error} Typing stopped within characters {}-{} of {total}.", posted + 1, posted + characters)
+ self.desktop.post_typed(&event, &recipient).map_err(|error| match error {
+ TypedError::Failed(error) => format!(
+ "{error} Typing stopped within characters {}-{} of {total}.",
+ posted + 1,
+ posted + characters
+ ),
+ // Refused before any of this post went out: what was sent is certain.
+ refused => format!("{} Typing stopped after {posted} of {total} characters.", refused.message(posted > 0)),
})?;
posted += characters;
Ok(())
@@ -868,7 +874,7 @@ impl Helper {
return Err("The app receiving the keys changed before the key was sent, so it was not sent.".into());
}
self.still_running(epoch)?;
- self.desktop.post_typed(&InputEvent::Key(combo), &recipient)?;
+ self.desktop.post_typed(&InputEvent::Key(combo), &recipient).map_err(|error| error.message(false))?;
(self.glow)(Mark::Key);
Ok((format!("{action} in {target}: done."), SETTLE_MS))
}
@@ -1622,6 +1628,41 @@ mod tests {
elevated_front_refuses("key", json!({"combo": "cmd+a"}));
}
+ /// The backend refusing a post before sending any of it (#167): the helper words it from what
+ /// earlier posts already sent, and reports no uncertain range.
+ #[test]
+ fn a_backend_refusal_is_reported_after_the_characters_already_sent() {
+ for (at, error, expected, sent) in [
+ (2, TypedError::RecipientChanged,
+ "The app receiving the keys changed while typing, so the rest was not sent. Typing stopped after 20 of 45 characters.", 1),
+ (1, TypedError::RecipientChanged,
+ "The app receiving the keys changed before typing, so nothing was sent. Typing stopped after 0 of 45 characters.", 0),
+ (2, TypedError::FrontUnknown,
+ "Cannot tell which window is in front, so the rest was not sent. Typing stopped after 20 of 45 characters.", 1),
+ (1, TypedError::FrontUnknown,
+ "Cannot tell which window is in front, so nothing was sent. Typing stopped after 0 of 45 characters.", 0),
+ ] {
+ let rig = rig();
+ rig.ready();
+ *rig.desktop.refuse_typed_at.lock().unwrap() = Some((at, error.clone()));
+ let result = rig.call("type", json!({"text": "a".repeat(45)}));
+ assert!(is_error(&result), "{error:?} at {at}: {result}");
+ assert_eq!(first_text(&result), expected, "{error:?} at {at}");
+ assert_eq!(rig.desktop.events(), vec![InputEvent::Text("a".repeat(20)); sent], "{error:?} at {at}");
+ }
+ }
+
+ #[test]
+ fn a_backend_refusal_of_a_key_says_nothing_was_sent() {
+ let rig = rig();
+ rig.ready();
+ *rig.desktop.refuse_typed_at.lock().unwrap() = Some((1, TypedError::RecipientChanged));
+ let result = rig.call("key", json!({"combo": "tab"}));
+ assert!(is_error(&result), "{result}");
+ assert_eq!(first_text(&result), KEYS_CHANGED_BEFORE_TYPING);
+ assert!(rig.desktop.events().is_empty());
+ }
+
#[test]
fn elevated_front_refuses_key_at_the_recheck() {
let rig = rig();
diff --git a/apps/desktop/src-tauri/src/computer/mod.rs b/apps/desktop/src-tauri/src/computer/mod.rs
index ece8b19..a3caf4b 100644
--- a/apps/desktop/src-tauri/src/computer/mod.rs
+++ b/apps/desktop/src-tauri/src/computer/mod.rs
@@ -107,6 +107,33 @@ pub enum PermissionKind {
pub const KEYS_CHANGED_WHILE_TYPING: &str = "The app receiving the keys changed while typing, so the rest was not sent.";
pub const KEYS_CHANGED_BEFORE_TYPING: &str = "The app receiving the keys changed before typing, so nothing was sent.";
+/// Why a typed post (`Desktop::post_typed`) did not go through. The first two are refusals
+/// before anything of that post was sent; only the helper knows how much of the text went out
+/// before it, so it words them (`message`).
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub enum TypedError {
+ /// The windows in front no longer belong to the approved recipient.
+ RecipientChanged,
+ /// The system did not say which window is in front.
+ FrontUnknown,
+ /// Posting failed; part of the post may have been sent.
+ Failed(String),
+}
+
+impl TypedError {
+ /// The error for the agent, given whether earlier posts of the same action already sent part
+ /// of it.
+ pub fn message(&self, sent: bool) -> String {
+ match (self, sent) {
+ (TypedError::RecipientChanged, false) => KEYS_CHANGED_BEFORE_TYPING.into(),
+ (TypedError::RecipientChanged, true) => KEYS_CHANGED_WHILE_TYPING.into(),
+ (TypedError::FrontUnknown, false) => "Cannot tell which window is in front, so nothing was sent.".into(),
+ (TypedError::FrontUnknown, true) => "Cannot tell which window is in front, so the rest was not sent.".into(),
+ (TypedError::Failed(error), _) => error.clone(),
+ }
+ }
+}
+
/// Why keys are not sent to an app that runs as administrator (Windows drops them silently).
pub fn runs_as_administrator(name: &str) -> String {
format!("{name} runs as administrator, so Gentle Dot cannot send it keys.")
@@ -132,8 +159,8 @@ pub trait Desktop: Send + Sync {
fn post(&self, event: &InputEvent) -> Result<(), String>;
/// Posts a type chunk or Return bound to the recipient approved by the shared guard.
/// Platforms with targeted input checks can validate it immediately before sending.
- fn post_typed(&self, event: &InputEvent, _recipient: &KeyboardRecipient) -> Result<(), String> {
- self.post(event)
+ fn post_typed(&self, event: &InputEvent, _recipient: &KeyboardRecipient) -> Result<(), TypedError> {
+ self.post(event).map_err(TypedError::Failed)
}
fn list_apps(&self) -> Result<Vec<AppInfo>, String>;
/// Opens (or activates) an app by name or bundle id.
diff --git a/apps/desktop/src-tauri/src/computer/win.rs b/apps/desktop/src-tauri/src/computer/win.rs
index 9b86d82..4fd3df2 100644
--- a/apps/desktop/src-tauri/src/computer/win.rs
+++ b/apps/desktop/src-tauri/src/computer/win.rs
@@ -30,14 +30,16 @@ use super::winapps::{
};
use super::winput::{checks_typing_target, releases, send_batches, settles, steps, Step, VirtualScreen};
use super::{
- runs_as_administrator, AppInfo, Capture, InputEvent, KeyboardRecipient, MouseButton, KEYS_CHANGED_BEFORE_TYPING,
+ runs_as_administrator, AppInfo, Capture, InputEvent, KeyboardRecipient, MouseButton, TypedError,
KEYS_CHANGED_WHILE_TYPING,
};
use std::ffi::c_void;
use std::mem::{size_of, zeroed};
use std::ptr::null_mut;
use std::time::Duration;
-use windows_sys::Win32::Foundation::{CloseHandle, GetLastError, SetLastError, ERROR_TIMEOUT, HWND, LPARAM, POINT, RECT};
+use windows_sys::Win32::Foundation::{
+ CloseHandle, GetLastError, SetLastError, ERROR_ACCESS_DENIED, ERROR_TIMEOUT, HWND, LPARAM, POINT, RECT,
+};
use windows_sys::Win32::Graphics::Dwm::{DwmFlush, DwmGetWindowAttribute, DWMWA_CLOAKED};
use windows_sys::Win32::Storage::FileSystem::{GetFileVersionInfoSizeW, GetFileVersionInfoW, VerQueryValueW};
use windows_sys::Win32::System::Com::{CoInitializeEx, CoUninitialize, COINIT_APARTMENTTHREADED, COINIT_DISABLE_OLE1DDE};
@@ -224,27 +226,27 @@ fn typing_windows() -> Option<(HWND, HWND)> {
}
}
-const CANNOT_TELL_FRONT: &str = "Cannot tell which window is in front, so nothing was sent.";
-
/// Posts `event` with a short gap between steps (surrogate pairs form one batch), waiting after
/// key releases or a complete pair for the window in front to take it. Text is bound to the
/// window in front and its keyboard focus when it starts, and stops when either changes. A
/// failure releases the keys and buttons left pressed by the successfully sent prefix before
/// reporting the error.
pub fn post(event: &InputEvent) -> Result<(), String> {
- post_input(event, None)
+ post_input(event, None).map_err(|error| error.message(false))
}
/// Posts a typed chunk, a Return, or a key only to the recipient the shared guard approved: the
-/// windows it is bound to must still belong to the approved front app and focused app.
-pub fn post_typed(event: &InputEvent, recipient: &KeyboardRecipient) -> Result<(), String> {
+/// windows it is bound to must still belong to the approved front app and focused app. A refusal
+/// before anything of it was sent is `RecipientChanged` or `FrontUnknown`, which the helper words
+/// from what earlier chunks sent.
+pub fn post_typed(event: &InputEvent, recipient: &KeyboardRecipient) -> Result<(), TypedError> {
post_input(event, Some(recipient))
}
-fn post_input(event: &InputEvent, approved: Option<&KeyboardRecipient>) -> Result<(), String> {
+fn post_input(event: &InputEvent, approved: Option<&KeyboardRecipient>) -> Result<(), TypedError> {
// Bind both type chunks and their Returns. Standalone combos remain free to switch apps.
let typing_target = if approved.is_some() || matches!(event, InputEvent::Text(_)) {
- Some(typing_windows().ok_or(CANNOT_TELL_FRONT)?)
+ Some(typing_windows().ok_or(TypedError::FrontUnknown)?)
} else {
None
};
@@ -267,26 +269,27 @@ fn post_input(event: &InputEvent, approved: Option<&KeyboardRecipient>) -> Resul
&& focused == approved.focused_pid
&& typing_windows() == Some((root, focus));
if !valid {
- return Err(KEYS_CHANGED_BEFORE_TYPING.into());
+ return Err(TypedError::RecipientChanged);
}
}
- let plan = steps(event, &virtual_screen())?;
+ let plan = steps(event, &virtual_screen()).map_err(TypedError::Failed)?;
let mut done = 0;
for batch in send_batches(&plan) {
let checked = checks_typing_target(&batch[0])
|| (approved.is_some() && matches!(batch[0], Step::Key { down: true, .. }));
let changed = checked && typing_target.is_some_and(|target| typing_windows() != Some(target));
let failed = if changed {
- let message = if done == 0 { KEYS_CHANGED_BEFORE_TYPING } else { KEYS_CHANGED_WHILE_TYPING };
- Some((done, message.to_string()))
+ // Before the first batch nothing of this post went out; after it, part of it did.
+ let error = if done == 0 { TypedError::RecipientChanged } else { TypedError::Failed(KEYS_CHANGED_WHILE_TYPING.into()) };
+ Some((done, error))
} else {
match send_batch(batch) {
- Err((sent, error)) => Some((done + sent, error)),
+ Err((sent, error)) => Some((done + sent, TypedError::Failed(error))),
Ok(()) => {
// No sleep or catch_up inside a surrogate pair's single SendInput call.
std::thread::sleep(EVENT_GAP);
if settles(batch.last().unwrap()) {
- catch_up().err().map(|error| (done + batch.len(), error))
+ catch_up().err().map(|error| (done + batch.len(), TypedError::Failed(error)))
} else {
None
}
@@ -1006,15 +1009,25 @@ fn resolve_name(wanted: &str, shortcuts: &[String]) -> Result<String, String> {
}
}
-/// The integrity level of a process (the last sub-authority of its token's mandatory label), or
-/// none when its token cannot be read.
-fn integrity_level(process: *mut c_void) -> Option<u32> {
+/// What reading a process's integrity level gave.
+#[derive(Debug, PartialEq, Eq)]
+enum Integrity {
+ /// The last sub-authority of its token's mandatory label.
+ Level(u32),
+ /// Its token refused to be opened: a process of this user and session refuses it when it runs
+ /// at a higher level (elevated).
+ Denied,
+ /// It could not be read.
+ Unknown,
+}
+
+fn integrity_level(process: *mut c_void) -> Integrity {
// SAFETY: the token is checked and closed; the label is read from a buffer of the size Windows
// asked for, aligned for it, and its SID is validated before its sub-authorities are read.
unsafe {
let mut token = null_mut();
if OpenProcessToken(process, TOKEN_QUERY, &mut token) == 0 {
- return None;
+ return if GetLastError() == ERROR_ACCESS_DENIED { Integrity::Denied } else { Integrity::Unknown };
}
let mut size = 0u32;
GetTokenInformation(token, TokenIntegrityLevel, null_mut(), 0, &mut size);
@@ -1023,27 +1036,31 @@ fn integrity_level(process: *mut c_void) -> Option<u32> {
&& GetTokenInformation(token, TokenIntegrityLevel, buffer.as_mut_ptr() as *mut c_void, size, &mut size) != 0;
CloseHandle(token);
if !read {
- return None;
+ return Integrity::Unknown;
}
let sid = (*(buffer.as_ptr() as *const TOKEN_MANDATORY_LABEL)).Label.Sid;
if sid.is_null() || IsValidSid(sid) == 0 {
- return None;
+ return Integrity::Unknown;
}
let count = *GetSidSubAuthorityCount(sid);
- (count > 0).then(|| *GetSidSubAuthority(sid, u32::from(count) - 1))
+ if count == 0 {
+ return Integrity::Unknown;
+ }
+ Integrity::Level(*GetSidSubAuthority(sid, u32::from(count) - 1))
}
}
/// Why keys must not go to `app`: Windows drops input sent to a process with a higher integrity
/// level than ours (an app run as administrator) without an error, so typing there would report
-/// keys that never arrived. A process whose level cannot be read is refused the same way.
+/// keys that never arrived. A token that refuses to be opened is taken as elevated; a process
+/// whose level cannot be read at all is refused too.
pub fn keyboard_refusal(app: &AppIdentity) -> Option<String> {
// SAFETY: the pseudo handle needs no closing; the target's handle is checked and closed.
let (own, target) = unsafe {
let own = integrity_level(GetCurrentProcess());
let process = OpenProcess(PROCESS_QUERY_LIMITED_INFORMATION, 0, app.pid as u32);
let target = if process.is_null() {
- None
+ Integrity::Unknown
} else {
let level = integrity_level(process);
CloseHandle(process);
@@ -1052,8 +1069,8 @@ pub fn keyboard_refusal(app: &AppIdentity) -> Option<String> {
(own, target)
};
match (own, target) {
- (Some(own), Some(target)) if target <= own => None,
- (Some(_), Some(_)) => Some(runs_as_administrator(&app.name)),
+ (Integrity::Level(own), Integrity::Level(target)) if target <= own => None,
+ (Integrity::Level(_), Integrity::Level(_) | Integrity::Denied) => Some(runs_as_administrator(&app.name)),
_ => Some(format!(
"Gentle Dot cannot tell whether {} runs as administrator, so it does not send it keys.",
app.name
@@ -1170,13 +1187,14 @@ mod tests {
}
/// A recipient the windows in front do not belong to gets nothing; on a desktop with nothing
- /// in front (CI), that is said instead.
+ /// in front, as on CI, the front is unknown instead, so CI rarely reaches the recipient check:
+ /// the interactive run covers it.
#[test]
fn keys_for_another_recipient_are_not_posted() {
let elsewhere = KeyboardRecipient { front_pid: i32::MAX, focused_pid: None };
let result = post_typed(&InputEvent::Text("x".into()), &elsewhere);
assert!(
- result == Err(KEYS_CHANGED_BEFORE_TYPING.into()) || result == Err(CANNOT_TELL_FRONT.into()),
+ result == Err(TypedError::RecipientChanged) || result == Err(TypedError::FrontUnknown),
"{result:?}"
);
}