A personal assistant that lives on your desktop, remembers what matters, and can use your Mac for you.
TypeScript
feat(computer): show a pink glow where the agent acts
Alan Buscagliaf01dfdf84a6e
diff --git a/apps/desktop/src-tauri/Cargo.toml b/apps/desktop/src-tauri/Cargo.tomlindex 19b43e8..87ea1e0 100644--- a/apps/desktop/src-tauri/Cargo.toml+++ b/apps/desktop/src-tauri/Cargo.toml@@ -35,7 +35,7 @@ objc2-foundation = { version = "0.3", default-features = false, features = ["stdobjc2-core-foundation = { version = "0.3", default-features = false, features = ["std", "CFArray", "CFCGTypes", "CFData", "CFDictionary", "CFNumber", "CFString"] }objc2-core-graphics = { version = "0.3", default-features = false, features = ["std", "CGDataProvider", "CGDirectDisplay", "CGError", "CGEvent", "CGEventTypes", "CGImage", "CGRemoteOperation", "CGWindow"] }objc2-app-kit = { version = "0.3", default-features = false, features = ["std", "libc", "NSRunningApplication", "NSWorkspace"] }-objc2-application-services = { version = "0.3", default-features = false, features = ["std", "libc", "AXError", "AXUIElement", "HIServices"] }+objc2-application-services = { version = "0.3", default-features = false, features = ["std", "libc", "AXError", "AXUIElement", "AXValue", "HIServices"] }objc2-screen-capture-kit = { version = "0.3", default-features = false, features = ["std", "block2", "libc", "objc2-core-foundation", "objc2-core-graphics", "SCScreenshotManager", "SCShareableContent", "SCStream"] }# Debug builds encode screenshots fast too (S24.8): `image` holds the JPEG encoder and `zune-*`diff --git a/apps/desktop/src-tauri/capabilities/default.json b/apps/desktop/src-tauri/capabilities/default.jsonindex bac9a4a..3f09df0 100644--- a/apps/desktop/src-tauri/capabilities/default.json+++ b/apps/desktop/src-tauri/capabilities/default.json@@ -1,25 +1,35 @@{"$schema": "../gen/schemas/desktop-schema.json",- "identifier": "default",- "description": "What the Dot and panel UI may call: event listening, Dot dragging, opening web pages (sign-in), the shell commands (hiding the rose and the full-screen panel included), and the computer-control commands (yolo mode included: its native confirmation decides).",- "windows": ["dot", "panel"],- "permissions": [- "core:event:allow-listen",- "core:event:allow-unlisten",- "core:window:allow-start-dragging",- "opener:default",- "allow-connection-info",- "allow-toggle-panel",- "allow-hide-panel",- "allow-set-dot-state",- "allow-rose-hidden",- "allow-set-rose-hidden",- "allow-set-panel-fullscreen",- "allow-computer-endpoint",- "allow-computer-permissions",- "allow-computer-request-permission",- "allow-computer-stop",- "allow-computer-status",- "allow-computer-set-yolo"+ "capabilities": [+ {+ "identifier": "default",+ "description": "What the Dot and panel UI may call: event listening, Dot dragging, opening web pages (sign-in), the shell commands (hiding the rose and the full-screen panel included), and the computer-control commands (yolo mode included: its native confirmation decides).",+ "windows": ["dot", "panel"],+ "permissions": [+ "core:event:allow-listen",+ "core:event:allow-unlisten",+ "core:window:allow-start-dragging",+ "opener:default",+ "allow-connection-info",+ "allow-toggle-panel",+ "allow-hide-panel",+ "allow-set-dot-state",+ "allow-rose-hidden",+ "allow-set-rose-hidden",+ "allow-set-panel-fullscreen",+ "allow-computer-endpoint",+ "allow-computer-permissions",+ "allow-computer-request-permission",+ "allow-computer-stop",+ "allow-computer-status",+ "allow-computer-set-yolo"+ ]+ },+ {+ "identifier": "glow",+ "description": "What the computer-control glow overlay (S28) may call: listening to the glow and session events, nothing else.",+ "windows": ["glow"],+ "permissions": ["core:event:allow-listen", "core:event:allow-unlisten"]+ }]}diff --git a/apps/desktop/src-tauri/src/computer/app.rs b/apps/desktop/src-tauri/src/computer/app.rsindex 107bee7..34fb486 100644--- a/apps/desktop/src-tauri/src/computer/app.rs+++ b/apps/desktop/src-tauri/src/computer/app.rs@@ -2,6 +2,11 @@//! event, the panic shortcut, the timeout watcher, and the native dialogs. Off macOS the//! commands answer "unavailable" and nothing starts. Yolo mode (S24.9) is switched only from//! here (`computer_set_yolo` and the tray item), never from the MCP helper or the daemon.+//!+//! The glow (S28): a click-through overlay window (`glow`) is ordered in while a session is+//! active and out when it ends. The helper's marks reach it through a channel, so the helper+//! never waits on the overlay; a thread of its own looks up the display (and, for keys, the+//! focused element), moves the overlay over that display, and emits `computer://glow`.use super::control::Control;use super::server::Endpoint;@@ -11,6 +16,10 @@ use std::sync::Arc;use tauri::{AppHandle, Emitter, Manager, State};pub const STATE_EVENT: &str = "computer://state";+/// Where the agent just acted, sent to the overlay only (S28).+pub const GLOW_EVENT: &str = "computer://glow";+pub const GLOW_WINDOW: &str = "glow";+pub const GLOW_TITLE: &str = "Gentle Dot Glow";/// ⌥⇧Esc ends the session from anywhere (S24.3).pub const PANIC_SHORTCUT: &str = "Alt+Shift+Escape";pub const GRANT_TEXT: &str = "Allow Gentle Dot to see and control this Mac for 30 minutes?";@@ -30,9 +39,19 @@ pub struct Computer {desktop: Arc<dyn super::Desktop>,}-/// Creates the session and, on macOS, starts the MCP helper and the timeout watcher.+/// What the overlay thread hears: the session started or ended, or the agent acted.+#[cfg_attr(not(target_os = "macos"), allow(dead_code))]+enum Overlay {+ Session(bool),+ Mark(super::glow::Mark),+}++/// Creates the session and, on macOS, starts the MCP helper, the timeout watcher, and the+/// glow overlay. Runs in the app's setup, on the main thread.pub fn start(app: &AppHandle) -> Computer {+ let (overlay, marks) = std::sync::mpsc::channel::<Overlay>();let events = app.clone();+ let session = overlay.clone();let control = Arc::new(Control::new(move |event: &StateEvent| {if let Err(error) = events.emit(STATE_EVENT, event) {eprintln!("gentle-dot: cannot report the computer-control state: {error}");@@ -40,18 +59,30 @@ pub fn start(app: &AppHandle) -> Computer {if let Some(item) = events.try_state::<YoloMenuItem>() {let _ = item.0.set_checked(event.yolo);}+ let _ = session.send(Overlay::Session(event.active));}));#[cfg(target_os = "macos")]{- let desktop: Arc<dyn super::Desktop> = Arc::new(super::MacDesktop::default());- let endpoint = start_helper(app, control.clone(), desktop.clone())+ let number = Arc::new(std::sync::atomic::AtomicU32::new(0));+ let desktop: Arc<dyn super::Desktop> = Arc::new(super::MacDesktop::new(number.clone()));+ let glow = move |mark| {+ let _ = overlay.send(Overlay::Mark(mark));+ };+ let endpoint = start_helper(app, control.clone(), desktop.clone(), glow).map_err(|error| eprintln!("gentle-dot: computer control is unavailable: {error}")).ok();watch_timeout(control.clone());+ match build_overlay(app, &number) {+ Ok(window) => run_overlay(app.clone(), window, number, control.clone(), marks),+ Err(error) => eprintln!("gentle-dot: the computer-control glow is unavailable: {error}"),+ }Computer { control, endpoint, desktop }}#[cfg(not(target_os = "macos"))]- Computer { control, endpoint: None }+ {+ drop((overlay, marks));+ Computer { control, endpoint: None }+ }}#[cfg(target_os = "macos")]@@ -59,6 +90,7 @@ fn start_helper(app: &AppHandle,control: Arc<Control>,desktop: Arc<dyn super::Desktop>,+ glow: impl Fn(super::glow::Mark) + Send + Sync + 'static,) -> std::io::Result<Endpoint> {let token = super::server::new_token();let helper = super::mcp::Helper::new(@@ -68,10 +100,112 @@ fn start_helper(control,token.clone(),std::process::id() as i32,- );+ )+ .with_glow(glow);super::server::start(Arc::new(helper), token)}+/// The overlay's NSWindow, used on the main thread only (the window lives as long as the app).+#[cfg(target_os = "macos")]+#[derive(Clone, Copy)]+struct NsWindow(usize);++/// The click-through overlay (S28): transparent, undecorated, never focusable, on every Space,+/// above menus, ignoring the mouse, and hidden until a session starts. Its window number lets+/// the helper skip it when it resolves which app owns a point.+#[cfg(target_os = "macos")]+fn build_overlay(+ app: &AppHandle,+ number: &std::sync::atomic::AtomicU32,+) -> tauri::Result<(tauri::WebviewWindow, NsWindow)> {+ let url = tauri::WebviewUrl::App("index.html?surface=glow".into());+ let window = tauri::WebviewWindowBuilder::new(app, GLOW_WINDOW, url)+ .title(GLOW_TITLE)+ .inner_size(1.0, 1.0)+ .position(0.0, 0.0)+ .transparent(true)+ .decorations(false)+ .always_on_top(true)+ .resizable(false)+ .skip_taskbar(true)+ .visible_on_all_workspaces(true)+ .shadow(false)+ .focused(false)+ .focusable(false)+ .visible(false)+ .build()?;+ window.set_ignore_cursor_events(true)?;+ let ns_window = window.ns_window()?;+ // SAFETY: setup runs on the main thread, and `ns_window` is the live window just built.+ number.store(unsafe { super::macos::prepare_overlay(ns_window) }, std::sync::atomic::Ordering::SeqCst);+ Ok((window, NsWindow(ns_window as usize)))+}++/// Orders the overlay in or out on the main thread and records its window number.+#[cfg(target_os = "macos")]+fn order_overlay(app: &AppHandle, ns_window: NsWindow, number: &Arc<std::sync::atomic::AtomicU32>, visible: bool) {+ let number = number.clone();+ let ordered = app.run_on_main_thread(move || {+ // SAFETY: on the main thread, with the overlay's live NSWindow.+ let n = unsafe { super::macos::order_overlay(ns_window.0 as *mut std::ffi::c_void, visible) };+ if n != 0 {+ number.store(n, std::sync::atomic::Ordering::SeqCst);+ }+ });+ if let Err(error) = ordered {+ eprintln!("gentle-dot: cannot show or hide the glow: {error}");+ }+}++/// The overlay's own thread: follows the session and draws each mark while it lasts.+#[cfg(target_os = "macos")]+fn run_overlay(+ app: AppHandle,+ (window, ns_window): (tauri::WebviewWindow, NsWindow),+ number: Arc<std::sync::atomic::AtomicU32>,+ control: Arc<Control>,+ marks: std::sync::mpsc::Receiver<Overlay>,+) {+ use tauri::{LogicalPosition, LogicalSize};+ let spawned = std::thread::Builder::new().name("computer-glow".into()).spawn(move || {+ let (mut shown, mut covering) = (false, None);+ for message in marks {+ match message {+ Overlay::Session(active) => {+ if active != shown {+ order_overlay(&app, ns_window, &number, active);+ shown = active;+ }+ }+ Overlay::Mark(mark) => {+ // A mark that arrives after Stop or panic is dropped.+ if !control.is_active(SystemClock.now_ms()) {+ continue;+ }+ let placed =+ super::glow::place(mark, super::macos::display_frame_at, super::macos::focused_frame);+ let Some((display, event)) = placed else { continue };+ if covering != Some(display) {+ let _ = window.set_position(LogicalPosition::new(display.x, display.y));+ let _ = window.set_size(LogicalSize::new(display.width, display.height));+ covering = Some(display);+ }+ if !shown {+ order_overlay(&app, ns_window, &number, true);+ shown = true;+ }+ if let Err(error) = app.emit_to(GLOW_WINDOW, GLOW_EVENT, event) {+ eprintln!("gentle-dot: cannot draw the glow: {error}");+ }+ }+ }+ }+ });+ if let Err(error) = spawned {+ eprintln!("gentle-dot: the computer-control glow is unavailable: {error}");+ }+}+/// Reports the timeout when it happens, not only at the next tool call.#[cfg(target_os = "macos")]fn watch_timeout(control: Arc<Control>) {diff --git a/apps/desktop/src-tauri/src/computer/glow.rs b/apps/desktop/src-tauri/src/computer/glow.rsnew file mode 100644index 0000000..60884d0--- /dev/null+++ b/apps/desktop/src-tauri/src/computer/glow.rs@@ -0,0 +1,173 @@+//! Where the agent acts (S28): after each action that went through, the helper reports a+//! `Mark` in global points, and the app draws a pink glow there in a click-through overlay.+//! `place` picks the display the overlay covers and turns the mark into the overlay's own+//! coordinates (points from the display's top-left corner).++use super::coords::Point;+use serde::Serialize;++/// A rectangle in global points.+#[derive(Debug, Clone, Copy, PartialEq)]+pub struct Frame {+ pub x: f64,+ pub y: f64,+ pub width: f64,+ pub height: f64,+}++impl Frame {+ pub fn contains(&self, at: Point) -> bool {+ at.x >= self.x && at.y >= self.y && at.x < self.x + self.width && at.y < self.y + self.height+ }++ fn center(&self) -> Point {+ Point { x: self.x + self.width / 2.0, y: self.y + self.height / 2.0 }+ }+}++/// What the agent just did, and where, in global points.+#[derive(Debug, Clone, Copy, PartialEq)]+pub enum Mark {+ /// A pulse.+ Click(Point),+ /// A short marker.+ Move(Point),+ /// A short marker.+ Scroll(Point),+ /// A trail from one point to the other.+ Drag { from: Point, to: Point },+ /// Keys went to the focused element; the app looks up where it is.+ Key,+}++#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]+#[serde(rename_all = "lowercase")]+pub enum GlowKind {+ Click,+ Move,+ Scroll,+ Drag,+ Key,+}++/// The `computer://glow` event for the overlay, in points from its top-left corner.+#[derive(Debug, Clone, Copy, PartialEq, Serialize)]+#[serde(rename_all = "camelCase")]+pub struct GlowEvent {+ pub kind: GlowKind,+ pub x: f64,+ pub y: f64,+ #[serde(skip_serializing_if = "Option::is_none")]+ pub to_x: Option<f64>,+ #[serde(skip_serializing_if = "Option::is_none")]+ pub to_y: Option<f64>,+ #[serde(skip_serializing_if = "Option::is_none")]+ pub width: Option<f64>,+ #[serde(skip_serializing_if = "Option::is_none")]+ pub height: Option<f64>,+}++/// The display the overlay covers for `mark` and the event to draw on it, or `None` when the+/// position is unknown (a focused element without a frame, a point on no display). A drag+/// glows on the display where it starts; `focus` runs only for keyboard marks.+pub fn place(+ mark: Mark,+ display_at: impl Fn(Point) -> Option<Frame>,+ focus: impl FnOnce() -> Option<Frame>,+) -> Option<(Frame, GlowEvent)> {+ let at = |kind, point: Point| GlowEvent {+ kind,+ x: point.x,+ y: point.y,+ to_x: None,+ to_y: None,+ width: None,+ height: None,+ };+ let (anchor, event) = match mark {+ Mark::Click(point) => (point, at(GlowKind::Click, point)),+ Mark::Move(point) => (point, at(GlowKind::Move, point)),+ Mark::Scroll(point) => (point, at(GlowKind::Scroll, point)),+ Mark::Drag { from, to } => {+ (from, GlowEvent { to_x: Some(to.x), to_y: Some(to.y), ..at(GlowKind::Drag, from) })+ }+ Mark::Key => {+ let frame = focus().filter(|f| f.width > 0.0 && f.height > 0.0)?;+ let origin = Point { x: frame.x, y: frame.y };+ let event = GlowEvent { width: Some(frame.width), height: Some(frame.height), ..at(GlowKind::Key, origin) };+ (frame.center(), event)+ }+ };+ let display = display_at(anchor)?;+ let local = |x: f64, y: f64| (x - display.x, y - display.y);+ let (x, y) = local(event.x, event.y);+ let to = event.to_x.zip(event.to_y).map(|(tx, ty)| local(tx, ty));+ Some((display, GlowEvent { x, y, to_x: to.map(|t| t.0), to_y: to.map(|t| t.1), ..event }))+}++#[cfg(test)]+mod tests {+ use super::*;+ use serde_json::json;++ /// Two displays side by side: the main one at the origin, a second one to its right.+ fn display_at(at: Point) -> Option<Frame> {+ let main = Frame { x: 0.0, y: 0.0, width: 1440.0, height: 900.0 };+ let second = Frame { x: 1440.0, y: -100.0, width: 1920.0, height: 1080.0 };+ [main, second].into_iter().find(|frame| frame.contains(at))+ }++ fn no_focus() -> Option<Frame> {+ None+ }++ fn p(x: f64, y: f64) -> Point {+ Point { x, y }+ }++ #[test]+ fn a_click_glows_at_its_point_on_the_display_it_is_on() {+ let (display, event) = place(Mark::Click(p(1500.0, 20.0)), display_at, no_focus).unwrap();+ assert_eq!(display, Frame { x: 1440.0, y: -100.0, width: 1920.0, height: 1080.0 });+ assert_eq!(serde_json::to_value(event).unwrap(), json!({"kind": "click", "x": 60.0, "y": 120.0}));+ }++ #[test]+ fn move_and_scroll_are_short_markers() {+ let (_, moved) = place(Mark::Move(p(10.0, 20.0)), display_at, no_focus).unwrap();+ let (_, scrolled) = place(Mark::Scroll(p(30.0, 40.0)), display_at, no_focus).unwrap();+ assert_eq!(serde_json::to_value(moved).unwrap(), json!({"kind": "move", "x": 10.0, "y": 20.0}));+ assert_eq!(serde_json::to_value(scrolled).unwrap(), json!({"kind": "scroll", "x": 30.0, "y": 40.0}));+ }++ #[test]+ fn a_drag_is_a_trail_on_the_display_where_it_starts() {+ let drag = Mark::Drag { from: p(100.0, 100.0), to: p(300.0, 250.0) };+ let (display, event) = place(drag, display_at, no_focus).unwrap();+ assert_eq!(display.x, 0.0);+ assert_eq!(+ serde_json::to_value(event).unwrap(),+ json!({"kind": "drag", "x": 100.0, "y": 100.0, "toX": 300.0, "toY": 250.0})+ );+ }++ #[test]+ fn a_keyboard_action_glows_around_the_focused_element() {+ let field = || Some(Frame { x: 1600.0, y: 0.0, width: 200.0, height: 30.0 });+ let (display, event) = place(Mark::Key, display_at, field).unwrap();+ assert_eq!(display.x, 1440.0);+ assert_eq!(+ serde_json::to_value(event).unwrap(),+ json!({"kind": "key", "x": 160.0, "y": 100.0, "width": 200.0, "height": 30.0})+ );+ }++ #[test]+ fn nothing_glows_where_the_position_is_unknown() {+ // The focused element has no frame, or the point is on no display.+ assert_eq!(place(Mark::Key, display_at, no_focus), None);+ assert_eq!(place(Mark::Click(p(-5000.0, 0.0)), display_at, no_focus), None);+ let empty = || Some(Frame { x: 10.0, y: 10.0, width: 0.0, height: 0.0 });+ assert_eq!(place(Mark::Key, display_at, empty), None);+ }+}diff --git a/apps/desktop/src-tauri/src/computer/macos.rs b/apps/desktop/src-tauri/src/computer/macos.rsindex 97b5543..1282163 100644--- a/apps/desktop/src-tauri/src/computer/macos.rs+++ b/apps/desktop/src-tauri/src/computer/macos.rs@@ -1,22 +1,28 @@//! The real Mac behind `Desktop`: ScreenCaptureKit screenshots (macOS 14+), Quartz events,//! the Accessibility API, and NSWorkspace. Nothing here decides policy; `mcp` does.+//! It also reads what the glow overlay (S28) needs: display frames, the focused element's+//! frame, and the overlay window's own setup.use super::blocklist::AppIdentity;use super::coords::{fit_long_edge, Point, ScreenshotGeometry, MAX_LONG_EDGE};use super::encode::{bgra_to_jpeg, JPEG_QUALITY};+use super::glow::Frame;use super::keys::KeyCombo;use super::risk::{AxElement, Focus};-use super::windows::{owner_row_at, WindowRow};+use super::windows::{hit_test_app, owner_row_at, WindowRow};use super::{AppInfo, Capture, Desktop, InputEvent, MouseButton, PermissionKind, Permissions};use block2::RcBlock;use objc2::rc::Retained;-use objc2::{available, AnyThread};+use objc2::runtime::AnyObject;+use objc2::{available, msg_send, AnyThread};use objc2_app_kit::{NSApplicationActivationOptions, NSApplicationActivationPolicy, NSRunningApplication, NSWorkspace};-use objc2_application_services::{AXError, AXIsProcessTrusted, AXIsProcessTrustedWithOptions, AXUIElement};+use objc2_application_services::{+ AXError, AXIsProcessTrusted, AXIsProcessTrustedWithOptions, AXUIElement, AXValue, AXValueType,+};use objc2_core_foundation::{CFBoolean, CFDictionary, CFRetained, CFString, CFType, CGPoint, CGRect, CGSize};use objc2_core_graphics::{- kCGNullWindowID, kCGWindowAlpha, kCGWindowBounds, kCGWindowLayer, kCGWindowOwnerName, kCGWindowOwnerPID,- CGDataProvider, CGDirectDisplayID, CGError, CGEvent, CGEventField, CGEventFlags,+ kCGNullWindowID, kCGWindowAlpha, kCGWindowBounds, kCGWindowLayer, kCGWindowNumber, kCGWindowOwnerName,+ kCGWindowOwnerPID, CGDataProvider, CGDirectDisplayID, CGDisplayBounds, CGError, CGEvent, CGEventField, CGEventFlags,CGEventTapLocation, CGEventType, CGGetDisplaysWithPoint, CGImage, CGMouseButton,CGPreflightScreenCaptureAccess, CGRequestScreenCaptureAccess, CGScrollEventUnit, CGWindowListCopyWindowInfo,CGWindowListOption,@@ -25,8 +31,10 @@ use objc2_foundation::{NSArray, NSDictionary, NSError, NSNumber, NSString};use objc2_screen_capture_kit::{SCContentFilter, SCScreenshotManager, SCShareableContent, SCStreamConfiguration, SCWindow,};+use std::ffi::c_void;use std::ptr::NonNull;-use std::sync::mpsc;+use std::sync::atomic::{AtomicU32, Ordering};+use std::sync::{mpsc, Arc};use std::thread;use std::time::Duration;@@ -46,11 +54,33 @@ unsafe impl<T> Send for Handoff<T> {}pub struct MacDesktop {own_pid: i32,+ /// The glow overlay's window number (S28), 0 until it exists.+ overlay: Arc<AtomicU32>,}-impl Default for MacDesktop {- fn default() -> Self {- MacDesktop { own_pid: std::process::id() as i32 }+impl MacDesktop {+ pub fn new(overlay: Arc<AtomicU32>) -> Self {+ MacDesktop { own_pid: std::process::id() as i32, overlay }+ }++ fn overlay(&self) -> Option<u32> {+ Some(self.overlay.load(Ordering::SeqCst)).filter(|&n| n != 0)+ }++ /// The element under `at`. A hit test that went through the click-through overlay lands on+ /// Gentle Dot, so it is redone inside the app that owns the window there.+ fn hit_test(&self, at: Point) -> Option<CFRetained<AXUIElement>> {+ let element = ax_element_at(&system_wide(), at)?;+ let Some(hit_pid) = ax_pid(&element) else {+ return Some(element);+ };+ let rows = window_rows();+ let owner = owner_row_at(&rows, at, self.overlay()).map(|row| row.pid);+ match hit_test_app(self.own_pid, hit_pid, owner) {+ // SAFETY: plain constructor.+ Some(pid) => ax_element_at(&with_timeout(unsafe { AXUIElement::new_application(pid) }), at),+ None => Some(element),+ }}}@@ -256,12 +286,13 @@ fn ax_pid(element: &AXUIElement) -> Option<i32> {(error == AXError::Success && pid > 0).then_some(pid)}-/// The Accessibility element at a global point, as hit-tested by the app that owns it.-fn ax_element_at(at: Point) -> Option<CFRetained<AXUIElement>> {+/// The Accessibility element at a global point, as hit-tested by the app that owns it+/// (`scope` is the system-wide element, or one app's element to test only that app).+fn ax_element_at(scope: &AXUIElement, at: Point) -> Option<CFRetained<AXUIElement>> {let mut found: *const AXUIElement = std::ptr::null();// SAFETY: `found` is a valid out-pointer; on success it holds a +1 reference.let out = NonNull::from(&mut found);- let error = unsafe { system_wide().copy_element_at_position(at.x as f32, at.y as f32, out) };+ let error = unsafe { scope.copy_element_at_position(at.x as f32, at.y as f32, out) };let found = NonNull::new(found.cast_mut()).filter(|_| error == AXError::Success)?;Some(unsafe { CFRetained::from_raw(found) })}@@ -277,12 +308,82 @@ fn describe(element: &AXUIElement) -> AxElement {fn system_wide() -> CFRetained<AXUIElement> {// SAFETY: plain constructor.- let element = unsafe { AXUIElement::new_system_wide() };- // A hung app must not stall the helper.+ with_timeout(unsafe { AXUIElement::new_system_wide() })+}++/// A hung app must not stall the helper.+fn with_timeout(element: CFRetained<AXUIElement>) -> CFRetained<AXUIElement> {+ // SAFETY: setting a timeout on a valid element.unsafe { element.set_messaging_timeout(0.5) };element}+/// A `CGPoint` or `CGSize` attribute (`AXPosition`, `AXSize`).+fn ax_struct<T: Default>(element: &AXUIElement, attribute: &str, kind: AXValueType) -> Option<T> {+ let value = ax_value(element, attribute)?.downcast::<AXValue>().ok()?;+ let mut out = T::default();+ // SAFETY: `out` has the layout of `kind` (CGPoint or CGSize).+ let ok = unsafe { value.value(kind, NonNull::from(&mut out).cast()) };+ ok.then_some(out)+}++/// The focused element's frame in global points, when it reports one (S28).+pub fn focused_frame() -> Option<Frame> {+ let focused = ax_element(&system_wide(), "AXFocusedUIElement")?;+ let origin: CGPoint = ax_struct(&focused, "AXPosition", AXValueType::CGPoint)?;+ let size: CGSize = ax_struct(&focused, "AXSize", AXValueType::CGSize)?;+ Some(Frame { x: origin.x, y: origin.y, width: size.width, height: size.height })+}++/// The frame of the display showing `at`, in global points (S28).+pub fn display_frame_at(at: Point) -> Option<Frame> {+ let bounds = CGDisplayBounds(display_at(at)?);+ Some(Frame { x: bounds.origin.x, y: bounds.origin.y, width: bounds.size.width, height: bounds.size.height })+}++/// `NSPopUpMenuWindowLevel + 1`: the glow shows above menus the agent clicks.+const OVERLAY_LEVEL: isize = 102;+/// `canJoinAllSpaces | stationary | ignoresCycle | fullScreenAuxiliary`.+const OVERLAY_COLLECTION: usize = (1 << 0) | (1 << 4) | (1 << 6) | (1 << 8);++/// Readies the glow overlay's NSWindow: above menus, on every Space (full-screen ones too),+/// out of window cycling, never taking mouse events. Main thread only. Returns its number.+///+/// # Safety+/// `ns_window` must be a live NSWindow.+pub unsafe fn prepare_overlay(ns_window: *mut c_void) -> u32 {+ let window = unsafe { &*ns_window.cast::<AnyObject>() };+ unsafe {+ let _: () = msg_send![window, setLevel: OVERLAY_LEVEL];+ let _: () = msg_send![window, setCollectionBehavior: OVERLAY_COLLECTION];+ let _: () = msg_send![window, setIgnoresMouseEvents: true];+ let _: () = msg_send![window, setHasShadow: false];+ overlay_number(window)+ }+}++/// Orders the overlay in without making it key or activating the app, or orders it out.+/// Main thread only. Returns its window number.+///+/// # Safety+/// `ns_window` must be a live NSWindow.+pub unsafe fn order_overlay(ns_window: *mut c_void, visible: bool) -> u32 {+ let window = unsafe { &*ns_window.cast::<AnyObject>() };+ unsafe {+ if visible {+ let _: () = msg_send![window, orderFrontRegardless];+ } else {+ let _: () = msg_send![window, orderOut: std::ptr::null::<AnyObject>()];+ }+ overlay_number(window)+ }+}++unsafe fn overlay_number(window: &AnyObject) -> u32 {+ let number: isize = unsafe { msg_send![window, windowNumber] };+ u32::try_from(number).unwrap_or(0)+}+type WindowInfo = NSDictionary<NSString, objc2::runtime::AnyObject>;/// A window-list key (a `CFString` constant) as the bridged `NSString`.@@ -312,14 +413,21 @@ fn window_rows() -> Vec<WindowRow> {// SAFETY: CFArray of CFDictionary is toll-free bridged to NSArray of NSDictionary.let list: &NSArray<WindowInfo> = unsafe { &*(CFRetained::as_ptr(&list).as_ptr() as *const _) };// SAFETY: extern constants provided by CoreGraphics.- let (alpha_key, layer_key, pid_key, name_key) = unsafe {- (info_key(kCGWindowAlpha), info_key(kCGWindowLayer), info_key(kCGWindowOwnerPID), info_key(kCGWindowOwnerName))+ let (alpha_key, layer_key, pid_key, name_key, number_key) = unsafe {+ (+ info_key(kCGWindowAlpha),+ info_key(kCGWindowLayer),+ info_key(kCGWindowOwnerPID),+ info_key(kCGWindowOwnerName),+ info_key(kCGWindowNumber),+ )};let number = |entry: &WindowInfo, key| entry.objectForKey(key).and_then(|n| n.downcast::<NSNumber>().ok());list.iter().filter_map(|entry| {let bounds = window_bounds(&entry)?;Some(WindowRow {+ number: number(&entry, number_key).map_or(0, |n| n.unsignedIntValue()),pid: number(&entry, pid_key)?.intValue(),owner_name: entry.objectForKey(name_key)@@ -421,7 +529,7 @@ impl Desktop for MacDesktop {}fn element_at(&self, at: Point) -> Vec<AxElement> {- let Some(mut element) = ax_element_at(at) else {+ let Some(mut element) = self.hit_test(at) else {return Vec::new();};let mut chain = vec![describe(&element)];@@ -449,13 +557,14 @@ impl Desktop for MacDesktop {ax_pid(&focused).map(|pid| identity_of(pid, ""))}- /// The owner of the frontmost app window under the point (the pointer and other window- /// server layers skipped), else the app whose Accessibility element is there.+ /// The owner of the frontmost app window under the point (the pointer, other window+ /// server layers, and the glow overlay skipped), else the app whose Accessibility element+ /// is there (Gentle Dot, and so refused, if that lands on the overlay).fn window_owner_at(&self, at: Point) -> Option<AppIdentity> {- if let Some(row) = owner_row_at(&window_rows(), at) {+ if let Some(row) = owner_row_at(&window_rows(), at, self.overlay()) {return Some(identity_of(row.pid, &row.owner_name));}- let element = ax_element_at(at)?;+ let element = ax_element_at(&system_wide(), at)?;ax_pid(&element).map(|pid| identity_of(pid, ""))}diff --git a/apps/desktop/src-tauri/src/computer/mcp.rs b/apps/desktop/src-tauri/src/computer/mcp.rsindex eba4469..c8700ca 100644--- a/apps/desktop/src-tauri/src/computer/mcp.rs+++ b/apps/desktop/src-tauri/src/computer/mcp.rs@@ -7,10 +7,15 @@//! (`perform`) and counts as one action for the rate limit; a full window delays the step//! instead of refusing it. Coordinates refer to the screenshot taken before the batch, and one//! screenshot follows it. The first refused, declined, or failed step stops the batch.+//!+//! The glow (S28): once an action's input is posted, `perform` reports where it acted to the+//! glow sink, so a refused, declined, stopped, or failed action (or batch step) reports+//! nothing. The sink only queues the mark; it never changes a check, the order, or a result.use super::blocklist::{blocked_name, blocked_reason, AppIdentity};use super::control::Control;use super::coords::{Point, ScreenshotGeometry};+use super::glow::Mark;use super::keys::{parse_combo, KeyCombo};use super::rate::{RateLimiter, MAX_ACTIONS_PER_SECOND};use super::risk::{assess, Act};@@ -307,6 +312,8 @@ pub struct Helper {limiter: Mutex<RateLimiter>,/// Serializes actions and holds the geometry of the latest screenshot.actions: Mutex<Option<ScreenshotGeometry>>,+ /// Where an action that went through acted (S28); must return at once.+ glow: Box<dyn Fn(Mark) + Send + Sync>,}impl Helper {@@ -327,9 +334,16 @@ impl Helper {own_pid,limiter: Mutex::new(RateLimiter::new(MAX_ACTIONS_PER_SECOND, 1000)),actions: Mutex::new(None),+ glow: Box::new(|_| {}),}}+ /// Reports each action that went through to `glow`, which must only queue it (S28).+ pub fn with_glow(mut self, glow: impl Fn(Mark) + Send + Sync + 'static) -> Self {+ self.glow = Box::new(glow);+ self+ }+fn authorized(&self, header: Option<&str>) -> bool {let Some((scheme, token)) = header.and_then(|h| h.trim().split_once(' ')) else {return false;@@ -637,6 +651,7 @@ impl Helper {};self.confirm_if_risky(assess(Act::Click(&chain), intent), &action, &target)?;self.post(epoch, InputEvent::Click { at, button, count })?;+ (self.glow)(Mark::Click(at));Ok((format!("{action} in {target}: done."), SETTLE_MS))}Tool::Move { x, y } => {@@ -645,6 +660,7 @@ impl Helper {let action = format!("Move the pointer to ({x}, {y})");self.confirm_if_risky(assess(Act::Other, intent), &action, &target)?;self.post(epoch, InputEvent::Move(at))?;+ (self.glow)(Mark::Move(at));Ok((format!("{action}: done."), SETTLE_MS))}Tool::Drag { from, to } => {@@ -656,6 +672,7 @@ impl Helper {let (from_chain, to_chain) = (self.desktop.element_at(start), self.desktop.element_at(end));self.confirm_if_risky(assess(Act::Drag(&from_chain, &to_chain), intent), &action, &target)?;self.post(epoch, InputEvent::Drag { from: start, to: end })?;+ (self.glow)(Mark::Drag { from: start, to: end });Ok((format!("{action}: done."), SETTLE_MS))}Tool::Scroll { x, y, dx, dy } => {@@ -664,6 +681,7 @@ impl Helper {let action = format!("Scroll by ({dx}, {dy}) at ({x}, {y})");self.confirm_if_risky(assess(Act::Other, intent), &action, &target)?;self.post(epoch, InputEvent::Scroll { at, dx, dy })?;+ (self.glow)(Mark::Scroll(at));Ok((format!("{action}: done."), SETTLE_MS))}Tool::Type { text: typed } => {@@ -681,6 +699,7 @@ impl Helper {self.post(epoch, InputEvent::Text(chunk))?;}}+ (self.glow)(Mark::Key);Ok((format!("{action} in {target}: done."), SETTLE_MS))}Tool::Key { combo } => {@@ -696,6 +715,7 @@ impl Helper {let action = format!("Press {}", combo.key);self.confirm_if_risky(assess(act, intent), &action, &target)?;self.post(epoch, InputEvent::Key(combo))?;+ (self.glow)(Mark::Key);Ok((format!("{action} in {target}: done."), SETTLE_MS))}Tool::OpenApp { name } => {@@ -727,6 +747,7 @@ mod tests {use super::super::blocklist::AppIdentity;use super::super::coords::Point;use super::super::fake::{app, FakeClock, FakeDesktop, FakeDialogs};+ use super::super::glow::Mark;use super::super::keys::parse_combo;use super::super::risk::{AxElement, Focus};use super::super::session::{Reason, StateEvent, SESSION_MS};@@ -747,6 +768,8 @@ mod tests {clock: Arc<FakeClock>,control: Arc<Control>,states: Arc<Mutex<Vec<StateEvent>>>,+ /// Each glow mark (S28), with how many input events were posted when it came.+ marks: Arc<Mutex<Vec<(Mark, usize)>>>,}fn rig_with(allow: bool, confirm: bool) -> Rig {@@ -756,9 +779,26 @@ mod tests {let states = Arc::new(Mutex::new(Vec::new()));let sink = states.clone();let control = Arc::new(Control::new(move |event| sink.lock().unwrap().push(event.clone())));+ let marks = Arc::new(Mutex::new(Vec::new()));+ let (seen, posted) = (marks.clone(), desktop.clone());let helper =- Helper::new(desktop.clone(), dialogs.clone(), clock.clone(), control.clone(), TOKEN.into(), OWN_PID);- Rig { helper, desktop, dialogs, clock, control, states }+ Helper::new(desktop.clone(), dialogs.clone(), clock.clone(), control.clone(), TOKEN.into(), OWN_PID)+ .with_glow(move |mark| seen.lock().unwrap().push((mark, posted.events().len())));+ Rig { helper, desktop, dialogs, clock, control, states, marks }+ }++ /// A rig whose helper has no glow sink, as before S28.+ fn rig_without_glow(allow: bool, confirm: bool) -> Rig {+ let rig = rig_with(allow, confirm);+ let helper = Helper::new(+ rig.desktop.clone(),+ rig.dialogs.clone(),+ rig.clock.clone(),+ rig.control.clone(),+ TOKEN.into(),+ OWN_PID,+ );+ Rig { helper, ..rig }}fn rig() -> Rig {@@ -1649,4 +1689,139 @@ mod tests {let apps: Value = serde_json::from_str(&first_text(&result)).unwrap();assert_eq!(apps[0], json!({"name": "Safari", "bundleId": "com.apple.Safari", "active": true}));}++ // --- The pink glow (S28) ---++ fn marks(rig: &Rig) -> Vec<(Mark, usize)> {+ rig.marks.lock().unwrap().clone()+ }++ #[test]+ fn each_action_that_went_through_marks_where_it_acted_after_posting() {+ let rig = rig();+ rig.ready();+ let p = |x, y| Point { x, y };+ rig.call("click", json!({"x": 100, "y": 50}));+ rig.call("move", json!({"x": 1, "y": 2}));+ rig.call("drag", json!({"fromX": 10, "fromY": 20, "toX": 30, "toY": 40}));+ rig.call("scroll", json!({"x": 5, "y": 6, "dx": 0, "dy": 120}));+ rig.call("key", json!({"combo": "tab"}));+ // A text with a newline is three input events and one mark, after the last of them.+ rig.call("type", json!({"text": "hi\nthere"}));+ assert_eq!(+ marks(&rig),+ vec![+ (Mark::Click(p(200.0, 100.0)), 1),+ (Mark::Move(p(2.0, 4.0)), 2),+ (Mark::Drag { from: p(20.0, 40.0), to: p(60.0, 80.0) }, 3),+ (Mark::Scroll(p(10.0, 12.0)), 4),+ (Mark::Key, 5),+ (Mark::Key, 8),+ ]+ );+ }++ #[test]+ fn refused_declined_and_stopped_actions_mark_nothing() {+ // Declined: a risky click the user says no to.+ let declined = rig_with(true, false);+ declined.ready();+ *declined.desktop.elements.lock().unwrap() = send_button();+ assert!(first_text(&declined.call("click", json!({"x": 10, "y": 10}))).contains("declined"));+ assert!(is_error(&declined.call("type", json!({"text": "hello\t "}))));+ // Refused: a blocked window, a point no app owns, a blocked app in front, invalid arguments.+ let rig_blocked = rig();+ rig_blocked.ready();+ *rig_blocked.desktop.owner_at.lock().unwrap() =+ Box::new(|_| Some(app(300, "com.apple.systempreferences", "System Settings")));+ assert!(is_error(&rig_blocked.call("click", json!({"x": 10, "y": 10}))));+ assert!(is_error(&rig_blocked.call("drag", json!({"fromX": 1, "fromY": 1, "toX": 2, "toY": 2}))));+ *rig_blocked.desktop.owner_at.lock().unwrap() = Box::new(|_| None);+ assert!(is_error(&rig_blocked.call("scroll", json!({"x": 1, "y": 1, "dx": 0, "dy": 1}))));+ *rig_blocked.desktop.frontmost.lock().unwrap() = Some(app(302, "com.apple.keychainaccess", "Keychain Access"));+ assert!(is_error(&rig_blocked.call("key", json!({"combo": "cmd+a"}))));+ assert!(is_error(&rig_blocked.call("click", json!({"x": 1}))));+ // Stopped: queued before a panic, no session (denied), no screenshot yet.+ let rig_stopped = rig();+ rig_stopped.ready();+ let queued_at = rig_stopped.control.epoch();+ rig_stopped.control.end(Reason::Panic);+ assert!(is_error(&rig_stopped.helper.call_tool_from(queued_at, "move", &json!({"x": 1, "y": 1})).unwrap()));+ let rig_denied = rig_with(false, true);+ assert!(is_error(&rig_denied.call("key", json!({"combo": "tab"}))));+ let rig_fresh = rig();+ assert!(is_error(&rig_fresh.call("click", json!({"x": 1, "y": 1}))));+ for r in [&declined, &rig_blocked, &rig_stopped, &rig_denied, &rig_fresh] {+ assert!(r.desktop.events().is_empty());+ assert!(marks(r).is_empty(), "{:?}", marks(r));+ }+ }++ #[test]+ fn screenshots_lists_open_app_and_wait_mark_nothing() {+ let rig = rig();+ rig.ready();+ rig.call("list_apps", json!({}));+ rig.call("open_app", json!({"name": "Mail"}));+ rig.call("wait", json!({"ms": 200}));+ assert!(marks(&rig).is_empty());+ }++ #[test]+ fn a_batch_marks_each_step_that_ran_and_none_after_it_stopped() {+ let rig = rig();+ rig.ready();+ *rig.desktop.owner_at.lock().unwrap() = Box::new(|at: Point| {+ if at.x > 100.0 {+ Some(app(300, "com.apple.systempreferences", "System Settings"))+ } else {+ Some(safari())+ }+ });+ let result = rig.call(+ "batch",+ batch(json!([+ {"tool": "click", "x": 10, "y": 10},+ {"tool": "wait", "ms": 0},+ {"tool": "type", "text": "Ana"},+ {"tool": "click", "x": 200, "y": 10},+ {"tool": "click", "x": 20, "y": 20},+ ])),+ );+ assert!(is_error(&result));+ assert_eq!(marks(&rig), vec![(Mark::Click(Point { x: 20.0, y: 20.0 }), 1), (Mark::Key, 2)]);+ }++ #[test]+ fn the_glow_changes_no_check_order_or_result() {+ // The same calls, with and without the glow, give the same results, dialogs, and events.+ let script = |rig: &Rig| -> Vec<Value> {+ *rig.desktop.elements.lock().unwrap() = send_button();+ let mut results = vec![rig.call("screenshot", json!({}))];+ for (name, args) in [+ ("click", json!({"x": 10, "y": 10})),+ ("move", json!({"x": 3, "y": 3})),+ ("type", json!({"text": "a\nb"})),+ (+ "batch",+ batch(json!([+ {"tool": "scroll", "x": 1, "y": 1, "dx": 0, "dy": 5},+ {"tool": "click", "x": 9, "y": 9},+ ])),+ ),+ ("click", json!({"x": 5000, "y": 1})),+ ] {+ results.push(rig.call(name, args));+ }+ results+ };+ for confirm in [true, false] {+ let (with, without) = (rig_with(true, confirm), rig_without_glow(true, confirm));+ assert_eq!(script(&with), script(&without), "confirm {confirm}");+ assert_eq!(with.desktop.events(), without.desktop.events());+ assert_eq!(*with.dialogs.confirmations.lock().unwrap(), *without.dialogs.confirmations.lock().unwrap());+ assert_eq!(with.clock.now_ms(), without.clock.now_ms());+ assert!(without.marks.lock().unwrap().is_empty());+ }+ }}diff --git a/apps/desktop/src-tauri/src/computer/mod.rs b/apps/desktop/src-tauri/src/computer/mod.rsindex 390c7c7..5e619b5 100644--- a/apps/desktop/src-tauri/src/computer/mod.rs+++ b/apps/desktop/src-tauri/src/computer/mod.rs@@ -10,6 +10,7 @@ pub mod blocklist;pub mod control;pub mod coords;pub mod encode;+pub mod glow;#[cfg(test)]mod fake;pub mod keys;diff --git a/apps/desktop/src-tauri/src/computer/windows.rs b/apps/desktop/src-tauri/src/computer/windows.rsindex bf7ecf2..5343386 100644--- a/apps/desktop/src-tauri/src/computer/windows.rs+++ b/apps/desktop/src-tauri/src/computer/windows.rs@@ -1,6 +1,7 @@//! Which window owns a point (S24.4), decided from the on-screen window list, front to back.//! The list also holds windows no app owns (the pointer, the menu bar backdrop), which must-//! never stand in for the app underneath.+//! never stand in for the app underneath. Neither may Gentle Dot's click-through glow overlay+//! (S28), skipped by its window number alone: the panel and the Dot still own their points.use super::coords::Point;@@ -10,6 +11,8 @@ pub const CURSOR_WINDOW_LEVEL: i64 = 2_147_483_630;/// One entry of `CGWindowListCopyWindowInfo`, in global points.#[derive(Debug, Clone, PartialEq)]pub struct WindowRow {+ /// `kCGWindowNumber`.+ pub number: u32,pub pid: i32,pub owner_name: String,pub layer: i64,@@ -37,8 +40,18 @@ impl WindowRow {}/// The frontmost visible window under `at` that belongs to an app, or `None` when there is none.-pub fn owner_row_at(rows: &[WindowRow], at: Point) -> Option<&WindowRow> {- rows.iter().find(|row| row.alpha > 0.0 && row.belongs_to_an_app() && row.contains(at))+/// `overlay` is the glow overlay's window number, once it exists.+pub fn owner_row_at(rows: &[WindowRow], at: Point, overlay: Option<u32>) -> Option<&WindowRow> {+ rows.iter().find(|row| {+ Some(row.number) != overlay && row.alpha > 0.0 && row.belongs_to_an_app() && row.contains(at)+ })+}++/// The app to hit-test again when the system-wide Accessibility hit test answered with+/// Gentle Dot (`hit_pid == own_pid`) while another app owns the window under the point: the+/// test went through the click-through overlay. `None` keeps the system-wide answer.+pub fn hit_test_app(own_pid: i32, hit_pid: i32, owner_pid: Option<i32>) -> Option<i32> {+ owner_pid.filter(|&owner| hit_pid == own_pid && owner != own_pid)}#[cfg(test)]@@ -48,7 +61,15 @@ mod tests {const AT: Point = Point { x: 500.0, y: 400.0 };fn row(pid: i32, owner: &str, layer: i64, (x, y, width, height): (f64, f64, f64, f64)) -> WindowRow {- WindowRow { pid, owner_name: owner.into(), layer, alpha: 1.0, x, y, width, height }+ WindowRow { number: 0, pid, owner_name: owner.into(), layer, alpha: 1.0, x, y, width, height }+ }++ const OWN_PID: i32 = 4242;+ const GLOW_WINDOW: u32 = 9001;++ /// The glow overlay (S28) as a real Mac would list it: Gentle Dot's, over the whole display.+ fn glow() -> WindowRow {+ WindowRow { number: GLOW_WINDOW, ..row(OWN_PID, "Gentle Dot", 102, (0.0, 0.0, 1440.0, 900.0)) }}/// The pointer window as a real Mac lists it, right under the pointer and first in the list.@@ -63,14 +84,14 @@ mod tests {#[test]fn the_pointer_window_is_skipped_for_the_app_underneath() {let rows = [cursor(), safari()];- assert_eq!(owner_row_at(&rows, AT).map(|r| r.pid), Some(200));+ assert_eq!(owner_row_at(&rows, AT, None).map(|r| r.pid), Some(200));}#[test]fn windows_owned_by_the_window_server_are_skipped() {let backdrop = row(412, "WindowServer", 0, (0.0, 0.0, 1440.0, 900.0));let rows = [row(412, "Window Server", 24, (0.0, 0.0, 1440.0, 900.0)), backdrop, safari()];- assert_eq!(owner_row_at(&rows, AT).map(|r| r.pid), Some(200));+ assert_eq!(owner_row_at(&rows, AT, None).map(|r| r.pid), Some(200));}#[test]@@ -78,13 +99,13 @@ mod tests {let clear = WindowRow { alpha: 0.0, ..row(300, "Overlay", 0, (0.0, 0.0, 1440.0, 900.0)) };let elsewhere = row(301, "Notes", 0, (900.0, 600.0, 200.0, 200.0));let rows = [clear, elsewhere, safari()];- assert_eq!(owner_row_at(&rows, AT).map(|r| r.pid), Some(200));+ assert_eq!(owner_row_at(&rows, AT, None).map(|r| r.pid), Some(200));}#[test]fn rows_without_a_process_are_skipped() {let rows = [row(0, "", 0, (0.0, 0.0, 1440.0, 900.0)), safari()];- assert_eq!(owner_row_at(&rows, AT).map(|r| r.pid), Some(200));+ assert_eq!(owner_row_at(&rows, AT, None).map(|r| r.pid), Some(200));}#[test]@@ -92,12 +113,47 @@ mod tests {// A lock or authorization prompt sits above every app window but below the pointer.let prompt = row(150, "loginwindow", 2_147_483_628, (0.0, 0.0, 1440.0, 900.0));let rows = [cursor(), prompt, safari()];- assert_eq!(owner_row_at(&rows, AT).map(|r| r.pid), Some(150));+ assert_eq!(owner_row_at(&rows, AT, None).map(|r| r.pid), Some(150));}#[test]fn only_the_pointer_under_the_point_resolves_to_nothing() {- assert_eq!(owner_row_at(&[cursor()], AT), None);- assert_eq!(owner_row_at(&[], AT), None);+ assert_eq!(owner_row_at(&[cursor()], AT, None), None);+ assert_eq!(owner_row_at(&[], AT, None), None);+ }++ #[test]+ fn the_glow_overlay_is_skipped_by_its_window_number() {+ let rows = [cursor(), glow(), safari()];+ assert_eq!(owner_row_at(&rows, AT, Some(GLOW_WINDOW)).map(|r| r.pid), Some(200));+ }++ #[test]+ fn the_panel_and_the_dot_under_the_glow_still_resolve_to_gentle_dot() {+ let panel = WindowRow { number: 9002, ..row(OWN_PID, "Gentle Dot", 3, (400.0, 300.0, 420.0, 640.0)) };+ let dot = WindowRow { number: 9003, ..row(OWN_PID, "Gentle Dot", 3, (490.0, 390.0, 72.0, 72.0)) };+ for rows in [[cursor(), glow(), panel.clone(), safari()], [cursor(), glow(), dot, safari()]] {+ assert_eq!(owner_row_at(&rows, AT, Some(GLOW_WINDOW)).map(|r| r.pid), Some(OWN_PID));+ }+ }++ #[test]+ fn only_the_overlay_number_is_skipped() {+ // Another window of the same owner and level, or the overlay before its number is known,+ // still owns the point (and Gentle Dot's windows are refused).+ let other = WindowRow { number: 9004, ..glow() };+ assert_eq!(owner_row_at(&[other, safari()], AT, Some(GLOW_WINDOW)).map(|r| r.pid), Some(OWN_PID));+ assert_eq!(owner_row_at(&[glow(), safari()], AT, None).map(|r| r.pid), Some(OWN_PID));+ }++ #[test]+ fn a_hit_test_that_lands_on_the_overlay_is_redone_in_the_app_underneath() {+ // The Accessibility hit test may land on the overlay; the risk check needs the element+ // of the app that owns the window under the point.+ assert_eq!(hit_test_app(OWN_PID, OWN_PID, Some(200)), Some(200));+ assert_eq!(hit_test_app(OWN_PID, 200, Some(200)), None);+ // Over the panel or the Dot (or nothing known), the system-wide answer stands.+ assert_eq!(hit_test_app(OWN_PID, OWN_PID, Some(OWN_PID)), None);+ assert_eq!(hit_test_app(OWN_PID, OWN_PID, None), None);}}diff --git a/docs/design.md b/docs/design.mdindex 316dd2f..036865f 100644--- a/docs/design.md+++ b/docs/design.md@@ -171,6 +171,11 @@ The assistant can see the screen and operate apps through a built-in MCP server- S24.7 Wiring: on every connection the desktop panel calls `computer_endpoint` and, when it returns `{url, token}`, sends `computer_register {url, token}` (the protocol accepts only `http://127.0.0.1:<port>/mcp` and a header-safe key). The daemon keeps the endpoint in memory, never in `connectors.json`, and renders `computer: {url, headers: {Authorization: "Bearer <key>"}, exposure: "direct"}` into `mcp.json`, then restarts the engine when idle. The guard's policy lists it under `builtin`, so its tools run without in-engine approval cards. It is removed on `computer_unregister` or when the window that registered it disconnects. The key is never logged or sent to a window: `connectors` lists only a built-in "Computer" entry (`builtin: true`) that the connector messages cannot change or remove, no saved or imported connector can take the name `computer`, and the import scan never offers the helper.- S24.8 Speed: `batch {steps: [{tool, ...args}], intent?}` runs 1 to 20 steps of `click`, `move`, `drag`, `scroll`, `type`, `key`, `open_app`, or `wait` in one call and returns one screenshot. Each step takes the single tool's path (session and epoch, blocklist, risky confirmation, N1/N2 typing rules, coordinates mapped with the screenshot taken before the batch) and counts as one action for the rate limit; a full window delays the step instead of refusing it. Steps are 100 ms apart (1.5 s after `open_app`) so each step's guards read the screen the previous one left. The batch's `intent` applies to every step on top of the step's own. The first refused, declined, or failed step stops the batch; the error says which steps ran and comes with a fresh screenshot while the session lasts (none after Stop or panic). Malformed batches (empty, over 20, a nested `batch`, `screenshot`, `list_apps`, unknown tools, invalid arguments) run nothing. Actions settle 150 ms before their screenshot (`open_app` 1.5 s); screenshots are JPEG quality 65; debug builds optimize `image`, `zune-*`, and this crate (opt-level 1, because the generic encoder is compiled here).- S24.9 Yolo mode: in memory only, off at launch. `computer_set_yolo {enabled}` and the tray item "Yolo mode" switch it; turning it on asks a native confirmation ("Turn on yolo mode? Gentle Dot will send, pay, delete, and submit without asking you first, for up to 1 hour. Stop and ⌥⇧Esc still work.", Turn On / Cancel). While on, only the risky-action confirmation is skipped; the grant, blocklist, rate limit, Stop, and panic are unchanged, and Stop or panic end the session, not yolo mode. It turns itself off after 1 hour. `computer://state` and `computer_status` add `yolo` and `yoloEndsAt?`; every switch reports the resulting state, so a declined confirmation flips the UI switch back. Connectors → Computer shows a "Yolo mode" switch, the banner shows "Yolo", and the rose carries a yellow mark (`dot-yolo`).+- S28 The pink glow where the agent acts:+ - What it shows: a pulse on a click, a trail along a drag (drawn on the display where the drag starts), short markers for moves and scrolls, and an outline around the focused element after `type` and `key`, when that element reports a frame (`AXPosition`, `AXSize`, read right after the keys); without one, nothing shows. The glow uses the accent `#f095c8`. It fades about 1.5 s after the last action (1.2 s lit, then a 300 ms fade) and clears at once when the session ends (Stop, panic, timeout). With "Reduce motion" it stays still: no pulse and no trail, only the drag's end.+ - Where the marks come from: once an action's input is posted, `perform` hands a `Mark` (global points and a kind) to the helper's glow sink. Each batch step that runs gets its own mark. A refused, declined, stopped, or failed action or step reports nothing. `open_app`, `wait`, `screenshot`, and `list_apps` report nothing either. The sink only queues the mark on a channel, so no check, order, timing, or result changes; a rig test runs the same calls with and without the sink and compares them.+ - The overlay: a Tauri window `glow` (`index.html?surface=glow`, title `Gentle Dot Glow`). It is transparent, undecorated, not focusable, out of the taskbar, and ignores cursor events. Its NSWindow sits at level 102, above menus, with collection behavior `canJoinAllSpaces | stationary | ignoresCycle | fullScreenAuxiliary`. It is ordered in with `orderFrontRegardless` when a session starts, so it never becomes key or activates the app, and ordered out when the session ends. A thread of its own receives the marks and drops any that arrive after the session ended. That thread looks up the display (`CGDisplayBounds`), moves the overlay over the whole display, and emits `computer://glow` to that window only. The UI also ignores marks that come after a `computer://state` without a session.+ - Guards and screenshots: the overlay is a Gentle Dot window covering the display, so pointer-owner resolution skips it by its window number (`kCGWindowNumber`) and nothing else. The panel and the Dot still resolve to Gentle Dot and are refused. When the system-wide Accessibility hit test lands on the overlay while another app owns the window under the point, the risk check hit-tests that app's own element instead (`AXUIElementCreateApplication`). Screenshots leave the overlay out with every other window of our process (ScreenCaptureKit `excludingWindows`). The overlay is on screen for the whole session, so it is in the capture's window list before it draws anything.Why enforcement lives in the app: everything inside the engine, and the daemon's WebSocket, is within the agent's reach (it has a shell, and with the daemon's access key it could answer its own approval cards, L49). The session grant, panic stop, blocklist, rate limit, and risky-action confirmations therefore run in the helper, behind native dialogs the agent cannot answer, and the in-engine guard adds nothing for these tools. macOS also attributes Accessibility and Screen Recording to the app that holds them, Gentle Dot.app. The key in `mcp.json` only reaches the helper, which still asks the user before anything happens.@@ -260,6 +265,7 @@ The desktop app bundles the built UI (`packages/ui/dist/app`) as its frontend, s|---|---|---|---|| `dot` | 72 × 72 pt (the rose in a 66 pt black disc) | `index.html?surface=dot` | title `Gentle Dot`; transparent outside the disc, no decorations, always on top, not resizable, skip taskbar, visible on all workspaces, shadow off; placed in logical points || `panel` | 420 × 640 | `index.html?surface=panel` (Linux: `&effects=none`) | title `Gentle Dot Panel`; transparent with macOS vibrancy (`HudWindow`), no decorations, always on top, hidden at start, skip taskbar. With `effects=none` the UI sets `data-effects="none"` on `<html>` and paints the panel opaque (`--bg`, no backdrop blur). |+| `glow` (macOS) | the display where the agent acts | `index.html?surface=glow` | title `Gentle Dot Glow`; the computer-control glow overlay (S28, §4): transparent, no decorations, not focusable, ignores cursor events, above menus, on every Space, shown only during a control session. Its own capability allows only `core:event:allow-listen` and `allow-unlisten`; it never connects to the daemon. |When the panel opens, it is placed next to the Dot on the side with more room, and it stays inside the monitor.@@ -284,6 +290,7 @@ The Dot drags with `getCurrentWindow().startDragging()` (permission `core:window| `dot://new-conversation` | — | the tray item "New conversation" was chosen; with the conversations list on, the panel starts a new conversation and opens; in the single chat (default) the UI ignores it || `dot://panel-shown` | — | the panel became visible; the UI focuses the composer || `dot://rose` | `{ "hidden": boolean }` | the rose was hidden or shown (panel header or tray) |+| `computer://glow` (to `glow` only) | `{ "kind": "click" \| "move" \| "scroll" \| "drag" \| "key", "x", "y", "toX"?, "toY"?, "width"?, "height"? }` in points from the overlay's top-left corner | the agent just acted there (S28); `drag` adds the end point, `key` the focused element's size |### Menu bar and shortcutdiff --git a/packages/ui/src/App.tsx b/packages/ui/src/App.tsxindex fface04..038e950 100644--- a/packages/ui/src/App.tsx+++ b/packages/ui/src/App.tsx@@ -3,6 +3,7 @@ import { type ConnectionInfo, tokenFromLocation } from "./client.ts";import { ChatSurface } from "./components/ChatSurface.tsx";import type { ComputerControls } from "./components/Computer.tsx";import { DotSurface } from "./components/DotSurface.tsx";+import { GlowSurface } from "./components/GlowSurface.tsx";import { computerPermissions, requestComputerPermission, setComputerYolo, stopComputer } from "./computer.ts";import {connectionInfo,@@ -19,9 +20,9 @@ import { usePanelWindow } from "./usePanelWindow.ts";type Surface = "dot" | "panel" | "web";-function currentSurface(): Surface {+function currentSurface(): Surface | "glow" {const surface = new URLSearchParams(window.location.search).get("surface");- return surface === "dot" || surface === "panel" ? surface : "web";+ return surface === "dot" || surface === "panel" || surface === "glow" ? surface : "web";}/**@@ -50,8 +51,13 @@ function isFullscreenShortcut(event: KeyboardEvent): boolean {);}+/** The desktop's glow overlay (S28) is its own surface: it never connects to the assistant. */export function App() {const surface = currentSurface();+ return surface === "glow" ? <GlowSurface /> : <Assistant surface={surface} />;+}++function Assistant({ surface }: { surface: Surface }) {const [info, setInfo] = useState<ConnectionInfo | undefined>(undefined);const [missingToken, setMissingToken] = useState(false);const [focusKey, setFocusKey] = useState(0);diff --git a/packages/ui/src/components/GlowSurface.tsx b/packages/ui/src/components/GlowSurface.tsxnew file mode 100644index 0000000..42ec08b--- /dev/null+++ b/packages/ui/src/components/GlowSurface.tsx@@ -0,0 +1,113 @@+import { type CSSProperties, useEffect, useRef, useState } from "react";+import { type ComputerGlow, onComputerGlow, onComputerState } from "../computer.ts";++/** The glow stays lit this long after the last action, then fades for `FADE_MS` (about 1.5 s in all). */+const HOLD_MS = 1200;+const FADE_MS = 300;+/** Older marks drop off so a long run of actions never piles up. */+const MAX_MARKS = 6;++interface Lit {+ id: number;+ glow: ComputerGlow;+}++function prefersReducedMotion(): boolean {+ return window.matchMedia?.("(prefers-reduced-motion: reduce)").matches ?? false;+}++const px = (value: number) => `${value}px`;++function Mark({ glow, still }: { glow: ComputerGlow; still: boolean }) {+ if (glow.kind === "key") {+ const frame: CSSProperties = {+ left: px(glow.x),+ top: px(glow.y),+ width: px(glow.width),+ height: px(glow.height),+ };+ return <span className="glow-mark glow-key" style={frame} />;+ }+ if (glow.kind === "drag") {+ const end = <span className="glow-mark glow-drag" style={{ left: px(glow.toX), top: px(glow.toY) }} />;+ if (still) return end;+ const length = Math.hypot(glow.toX - glow.x, glow.toY - glow.y);+ const angle = Math.atan2(glow.toY - glow.y, glow.toX - glow.x);+ const trail: CSSProperties = {+ left: px(glow.x),+ top: px(glow.y),+ width: px(length),+ transform: `rotate(${angle}rad)`,+ };+ return (+ <>+ <span className="glow-trail" style={trail} />+ {end}+ </>+ );+ }+ return <span className={`glow-mark glow-${glow.kind}`} style={{ left: px(glow.x), top: px(glow.y) }} />;+}++/**+ * The click-through overlay where the agent acts (S28): a pink pulse on a click, a trail along a+ * drag, short markers for moves and scrolls, and an outline around the focused element for keys.+ * It shows only while the agent acts, fades out about 1.5 s after the last action, and clears at+ * once when the session ends. With reduced motion the glow is static: no pulse and no trail.+ */+export function GlowSurface() {+ const [marks, setMarks] = useState<Lit[]>([]);+ const [fading, setFading] = useState(false);+ const [still] = useState(prefersReducedMotion);+ const timers = useRef<ReturnType<typeof setTimeout>[]>([]);+ const ended = useRef(false);+ const next = useRef(0);++ useEffect(() => {+ document.documentElement.dataset.surface = "glow";+ let live = true;+ const stopTimers = () => {+ for (const timer of timers.current) clearTimeout(timer);+ timers.current = [];+ };+ const clear = () => {+ stopTimers();+ setFading(false);+ setMarks([]);+ };+ const show = (glow: ComputerGlow) => {+ // A mark that arrives after Stop, panic, or the timeout stays dark.+ if (!live || ended.current) return;+ stopTimers();+ setFading(false);+ next.current += 1;+ const lit = { id: next.current, glow };+ setMarks((current) => [...current, lit].slice(-MAX_MARKS));+ timers.current = [+ setTimeout(() => setFading(true), HOLD_MS),+ setTimeout(() => setMarks([]), HOLD_MS + FADE_MS),+ ];+ };+ const subscriptions = [+ onComputerGlow(show),+ onComputerState((state) => {+ ended.current = state.session === undefined;+ if (live && ended.current) clear();+ }),+ ];+ return () => {+ live = false;+ stopTimers();+ for (const s of subscriptions) void s.then((unlisten) => unlisten());+ };+ }, []);++ const className = `glow-layer${fading ? " glow-fading" : ""}${still ? " glow-static" : ""}`;+ return (+ <div className={className} aria-hidden="true">+ {marks.map(({ id, glow }) => (+ <Mark key={id} glow={glow} still={still} />+ ))}+ </div>+ );+}diff --git a/packages/ui/src/computer.ts b/packages/ui/src/computer.tsindex 4bb649b..d7fdace 100644--- a/packages/ui/src/computer.ts+++ b/packages/ui/src/computer.ts@@ -81,12 +81,62 @@ export async function computerStatus(): Promise<ComputerState> {return stateOf(await invoke<ComputerStateEvent>("computer_status"));}+/** The event API, imported once: the glow overlay subscribes to two events at the same time. */+let eventApi: Promise<typeof import("@tauri-apps/api/event")> | undefined;+function events() {+ eventApi ??= import("@tauri-apps/api/event");+ return eventApi;+}+export async function onComputerState(handler: (state: ComputerState) => void) {if (!isTauri()) return () => {};- const { listen } = await import("@tauri-apps/api/event");+ const { listen } = await events();return listen<ComputerStateEvent>("computer://state", (event) => handler(stateOf(event.payload)));}+/**+ * Where the agent just acted (S28), in points from the top-left corner of the glow overlay. A drag+ * runs from `x, y` to `toX, toY`; a keyboard action outlines the focused element (`width × height`).+ */+export type ComputerGlow =+ | { kind: "click" | "move" | "scroll"; x: number; y: number }+ | { kind: "drag"; x: number; y: number; toX: number; toY: number }+ | { kind: "key"; x: number; y: number; width: number; height: number };++/** The overlay window's `computer://glow` events; anything it cannot place is dropped. */+export async function onComputerGlow(handler: (glow: ComputerGlow) => void) {+ if (!isTauri()) return () => {};+ const { listen } = await events();+ return listen<unknown>("computer://glow", (event) => {+ const glow = glowOf(event.payload);+ if (glow) handler(glow);+ });+}++function glowOf(payload: unknown): ComputerGlow | undefined {+ if (typeof payload !== "object" || payload === null) return undefined;+ const event = payload as Record<string, unknown>;+ const finite = (value: unknown): value is number => typeof value === "number" && Number.isFinite(value);+ const { kind, x, y } = event;+ if (!finite(x) || !finite(y)) return undefined;+ switch (kind) {+ case "click":+ case "move":+ case "scroll":+ return { kind, x, y };+ case "drag":+ return finite(event.toX) && finite(event.toY)+ ? { kind, x, y, toX: event.toX, toY: event.toY }+ : undefined;+ case "key":+ return finite(event.width) && finite(event.height) && event.width > 0 && event.height > 0+ ? { kind, x, y, width: event.width, height: event.height }+ : undefined;+ default:+ return undefined;+ }+}+function stateOf(state: ComputerStateEvent | null | undefined): ComputerState {const session = state?.active === true ? { ...timeOf(state.endsAt) } : undefined;const yolo = state?.yolo === true ? { ...timeOf(state.yoloEndsAt) } : undefined;diff --git a/packages/ui/src/styles.css b/packages/ui/src/styles.cssindex b4a5e38..0c08ce2 100644--- a/packages/ui/src/styles.css+++ b/packages/ui/src/styles.css@@ -1348,3 +1348,123 @@ button.link:hover:not(:disabled) {transition: none !important;}}++/* The glow where the agent acts (S28), in the desktop's click-through overlay: the accent pink, like+ the neon rose. A pulse on a click, a trail along a drag, short markers for moves and scrolls, and an+ outline around the focused element for keys. Shown while the agent acts, faded about 1.5 s later. */+html[data-surface="glow"] body {+ background: transparent;+ overflow: hidden;+ pointer-events: none;+}++.glow-layer {+ position: fixed;+ inset: 0;+ pointer-events: none;+ transition: opacity 300ms var(--ease);+}++.glow-layer.glow-fading {+ opacity: 0;+}++.glow-mark {+ position: absolute;+ width: 28px;+ height: 28px;+ margin: -14px 0 0 -14px;+ border-radius: 50%;+ background: radial-gradient(+ circle,+ rgba(255, 255, 255, 0.9) 0 12%,+ var(--accent) 28%,+ rgba(240, 149, 200, 0) 70%+ );+ box-shadow:+ 0 0 12px 4px rgba(240, 149, 200, 0.55),+ 0 0 32px 10px rgba(240, 149, 200, 0.25);+}++.glow-click::after {+ content: "";+ position: absolute;+ inset: -6px;+ border: 2px solid var(--accent);+ border-radius: 50%;+ animation: glow-pulse 700ms var(--ease) both;+}++.glow-move,+.glow-scroll {+ width: 16px;+ height: 16px;+ margin: -8px 0 0 -8px;+ box-shadow: 0 0 10px 3px rgba(240, 149, 200, 0.45);+ animation: glow-blip 600ms var(--ease) both;+}++.glow-trail {+ position: absolute;+ height: 6px;+ margin-top: -3px;+ border-radius: var(--radius-pill);+ transform-origin: 0 50%;+ background: linear-gradient(90deg, rgba(240, 149, 200, 0), var(--accent));+ box-shadow: 0 0 14px 3px rgba(240, 149, 200, 0.45);+ animation: glow-trail 450ms var(--ease) both;+}++.glow-key {+ margin: -4px 0 0 -4px;+ padding: 4px;+ box-sizing: content-box;+ border: 2px solid var(--accent);+ border-radius: var(--radius-sm);+ background: var(--accent-soft);+ box-shadow:+ 0 0 14px 2px rgba(240, 149, 200, 0.5),+ inset 0 0 10px rgba(240, 149, 200, 0.3);+ animation: glow-blip 600ms var(--ease) both;+}++/* Reduce motion: the glow stays still (no pulse, no trail; the overlay draws only the drag's end). */+.glow-static .glow-mark,+.glow-static .glow-mark::after {+ animation: none;+}++.glow-static .glow-click::after {+ display: none;+}++@keyframes glow-pulse {+ from {+ opacity: 1;+ transform: scale(0.4);+ }+ to {+ opacity: 0;+ transform: scale(1.8);+ }+}++@keyframes glow-blip {+ from {+ opacity: 0;+ transform: scale(0.6);+ }+ to {+ opacity: 1;+ transform: scale(1);+ }+}++@keyframes glow-trail {+ from {+ clip-path: inset(0 100% 0 0);+ }+ to {+ clip-path: inset(0 0 0 0);+ }+}diff --git a/packages/ui/test/glow.test.tsx b/packages/ui/test/glow.test.tsxnew file mode 100644index 0000000..8622022--- /dev/null+++ b/packages/ui/test/glow.test.tsx@@ -0,0 +1,158 @@+import { act, render, waitFor } from "@testing-library/react";+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";++const h = vi.hoisted(() => ({+ invoke: vi.fn(),+ listeners: new Map<string, (event: { payload: unknown }) => void>(),+ reducedMotion: false,+}));++vi.mock("@tauri-apps/api/core", () => ({+ isTauri: () => true,+ invoke: (command: string, args?: unknown) => h.invoke(command, args),+}));++vi.mock("@tauri-apps/api/event", () => ({+ listen: async (name: string, handler: (event: { payload: unknown }) => void) => {+ h.listeners.set(name, handler);+ return () => h.listeners.delete(name);+ },+}));++const { GlowSurface } = await import("../src/components/GlowSurface.tsx");+const { App } = await import("../src/App.tsx");++function emit(name: string, payload: unknown) {+ act(() => h.listeners.get(name)?.({ payload }));+}++const glow = (payload: unknown) => emit("computer://glow", payload);+const session = (active: boolean) => emit("computer://state", { active, yolo: false });++async function renderGlow() {+ const view = render(<GlowSurface />);+ await waitFor(() => expect(h.listeners.has("computer://glow")).toBe(true));+ await waitFor(() => expect(h.listeners.has("computer://state")).toBe(true));+ return view.container;+}++const marks = (root: HTMLElement) => root.querySelectorAll(".glow-mark");++beforeEach(() => {+ h.invoke.mockReset();+ h.invoke.mockImplementation(() => new Promise(() => {}));+ h.listeners.clear();+ h.reducedMotion = false;+ window.matchMedia = ((query: string) => ({+ matches: query.includes("reduced-motion") && h.reducedMotion,+ media: query,+ addEventListener: () => {},+ removeEventListener: () => {},+ })) as unknown as typeof window.matchMedia;+});++afterEach(() => {+ vi.useRealTimers();+ window.history.replaceState(null, "", "/");+});++describe("GlowSurface (S28)", () => {+ it("shows nothing while the agent is idle", async () => {+ const root = await renderGlow();+ expect(marks(root)).toHaveLength(0);+ session(true);+ expect(marks(root)).toHaveLength(0);+ });++ it("pulses where the agent clicked, in the overlay's points", async () => {+ const root = await renderGlow();+ glow({ kind: "click", x: 60, y: 120 });+ const click = root.querySelector(".glow-mark.glow-click") as HTMLElement;+ expect(click).not.toBeNull();+ expect(click.style.left).toBe("60px");+ expect(click.style.top).toBe("120px");+ });++ it("draws a trail along a drag and short markers for move and scroll", async () => {+ const root = await renderGlow();+ glow({ kind: "drag", x: 100, y: 100, toX: 400, toY: 500 });+ const trail = root.querySelector(".glow-trail") as HTMLElement;+ expect(trail).not.toBeNull();+ expect(trail.style.width).toBe("500px");+ expect(root.querySelector(".glow-mark.glow-drag")).not.toBeNull();+ glow({ kind: "move", x: 1, y: 2 });+ glow({ kind: "scroll", x: 3, y: 4 });+ expect(root.querySelector(".glow-mark.glow-move")).not.toBeNull();+ expect(root.querySelector(".glow-mark.glow-scroll")).not.toBeNull();+ });++ it("outlines the focused element for keyboard actions", async () => {+ const root = await renderGlow();+ glow({ kind: "key", x: 160, y: 100, width: 200, height: 30 });+ const field = root.querySelector(".glow-mark.glow-key") as HTMLElement;+ expect(field.style.width).toBe("200px");+ expect(field.style.height).toBe("30px");+ });++ it("ignores events it cannot place", async () => {+ const root = await renderGlow();+ for (const payload of [+ null,+ { kind: "click" },+ { kind: "explode", x: 1, y: 1 },+ { kind: "key", x: 1, y: 1 },+ { kind: "drag", x: 1, y: 1 },+ { kind: "click", x: Number.NaN, y: 1 },+ ]) {+ glow(payload);+ }+ expect(marks(root)).toHaveLength(0);+ });++ it("fades out about 1.5 s after the last action", async () => {+ const root = await renderGlow();+ vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout"] });+ glow({ kind: "click", x: 10, y: 10 });+ act(() => vi.advanceTimersByTime(1000));+ // Another action keeps it lit.+ glow({ kind: "move", x: 20, y: 20 });+ act(() => vi.advanceTimersByTime(1200));+ expect(marks(root)).toHaveLength(2);+ expect(root.querySelector(".glow-layer")).toHaveClass("glow-fading");+ act(() => vi.advanceTimersByTime(300));+ expect(marks(root)).toHaveLength(0);+ });++ it("disappears at once when the session ends and stays dark until the next one", async () => {+ const root = await renderGlow();+ session(true);+ glow({ kind: "click", x: 10, y: 10 });+ session(false);+ expect(marks(root)).toHaveLength(0);+ // A mark that arrives after Stop, panic, or the timeout is dropped.+ glow({ kind: "click", x: 10, y: 10 });+ expect(marks(root)).toHaveLength(0);+ session(true);+ glow({ kind: "click", x: 10, y: 10 });+ expect(marks(root)).toHaveLength(1);+ });++ it("stays still with reduced motion: a static glow, no pulse or trail", async () => {+ h.reducedMotion = true;+ const root = await renderGlow();+ glow({ kind: "drag", x: 100, y: 100, toX: 400, toY: 500 });+ expect(root.querySelector(".glow-layer")).toHaveClass("glow-static");+ expect(root.querySelector(".glow-trail")).toBeNull();+ const end = root.querySelector(".glow-mark.glow-drag") as HTMLElement;+ expect([end.style.left, end.style.top]).toEqual(["400px", "500px"]);+ });++ it("is the app's glow window, which never connects to the assistant", async () => {+ window.history.replaceState(null, "", "/?surface=glow");+ const view = render(<App />);+ await waitFor(() => expect(h.listeners.has("computer://glow")).toBe(true));+ expect(view.container.querySelector(".glow-layer")).not.toBeNull();+ expect(document.documentElement.dataset.surface).toBe("glow");+ expect(h.invoke).not.toHaveBeenCalledWith("connection_info", undefined);+ });+});