Skip to content

A personal assistant that lives on your desktop, remembers what matters, and can use your Mac for you.

TypeScript

feat(computer): show a pink glow where the agent acts

Alan Buscagliaf01dfdf84a6e
diff --git a/apps/desktop/src-tauri/Cargo.toml b/apps/desktop/src-tauri/Cargo.toml
index 19b43e8..87ea1e0 100644
--- a/apps/desktop/src-tauri/Cargo.toml
+++ b/apps/desktop/src-tauri/Cargo.toml
@@ -35,7 +35,7 @@ objc2-foundation = { version = "0.3", default-features = false, features = ["std
objc2-core-foundation = { version = "0.3", default-features = false, features = ["std", "CFArray", "CFCGTypes", "CFData", "CFDictionary", "CFNumber", "CFString"] }
objc2-core-graphics = { version = "0.3", default-features = false, features = ["std", "CGDataProvider", "CGDirectDisplay", "CGError", "CGEvent", "CGEventTypes", "CGImage", "CGRemoteOperation", "CGWindow"] }
objc2-app-kit = { version = "0.3", default-features = false, features = ["std", "libc", "NSRunningApplication", "NSWorkspace"] }
-objc2-application-services = { version = "0.3", default-features = false, features = ["std", "libc", "AXError", "AXUIElement", "HIServices"] }
+objc2-application-services = { version = "0.3", default-features = false, features = ["std", "libc", "AXError", "AXUIElement", "AXValue", "HIServices"] }
objc2-screen-capture-kit = { version = "0.3", default-features = false, features = ["std", "block2", "libc", "objc2-core-foundation", "objc2-core-graphics", "SCScreenshotManager", "SCShareableContent", "SCStream"] }
# Debug builds encode screenshots fast too (S24.8): `image` holds the JPEG encoder and `zune-*`
diff --git a/apps/desktop/src-tauri/capabilities/default.json b/apps/desktop/src-tauri/capabilities/default.json
index bac9a4a..3f09df0 100644
--- a/apps/desktop/src-tauri/capabilities/default.json
+++ b/apps/desktop/src-tauri/capabilities/default.json
@@ -1,25 +1,35 @@
{
"$schema": "../gen/schemas/desktop-schema.json",
- "identifier": "default",
- "description": "What the Dot and panel UI may call: event listening, Dot dragging, opening web pages (sign-in), the shell commands (hiding the rose and the full-screen panel included), and the computer-control commands (yolo mode included: its native confirmation decides).",
- "windows": ["dot", "panel"],
- "permissions": [
- "core:event:allow-listen",
- "core:event:allow-unlisten",
- "core:window:allow-start-dragging",
- "opener:default",
- "allow-connection-info",
- "allow-toggle-panel",
- "allow-hide-panel",
- "allow-set-dot-state",
- "allow-rose-hidden",
- "allow-set-rose-hidden",
- "allow-set-panel-fullscreen",
- "allow-computer-endpoint",
- "allow-computer-permissions",
- "allow-computer-request-permission",
- "allow-computer-stop",
- "allow-computer-status",
- "allow-computer-set-yolo"
+ "capabilities": [
+ {
+ "identifier": "default",
+ "description": "What the Dot and panel UI may call: event listening, Dot dragging, opening web pages (sign-in), the shell commands (hiding the rose and the full-screen panel included), and the computer-control commands (yolo mode included: its native confirmation decides).",
+ "windows": ["dot", "panel"],
+ "permissions": [
+ "core:event:allow-listen",
+ "core:event:allow-unlisten",
+ "core:window:allow-start-dragging",
+ "opener:default",
+ "allow-connection-info",
+ "allow-toggle-panel",
+ "allow-hide-panel",
+ "allow-set-dot-state",
+ "allow-rose-hidden",
+ "allow-set-rose-hidden",
+ "allow-set-panel-fullscreen",
+ "allow-computer-endpoint",
+ "allow-computer-permissions",
+ "allow-computer-request-permission",
+ "allow-computer-stop",
+ "allow-computer-status",
+ "allow-computer-set-yolo"
+ ]
+ },
+ {
+ "identifier": "glow",
+ "description": "What the computer-control glow overlay (S28) may call: listening to the glow and session events, nothing else.",
+ "windows": ["glow"],
+ "permissions": ["core:event:allow-listen", "core:event:allow-unlisten"]
+ }
]
}
diff --git a/apps/desktop/src-tauri/src/computer/app.rs b/apps/desktop/src-tauri/src/computer/app.rs
index 107bee7..34fb486 100644
--- a/apps/desktop/src-tauri/src/computer/app.rs
+++ b/apps/desktop/src-tauri/src/computer/app.rs
@@ -2,6 +2,11 @@
//! event, the panic shortcut, the timeout watcher, and the native dialogs. Off macOS the
//! commands answer "unavailable" and nothing starts. Yolo mode (S24.9) is switched only from
//! here (`computer_set_yolo` and the tray item), never from the MCP helper or the daemon.
+//!
+//! The glow (S28): a click-through overlay window (`glow`) is ordered in while a session is
+//! active and out when it ends. The helper's marks reach it through a channel, so the helper
+//! never waits on the overlay; a thread of its own looks up the display (and, for keys, the
+//! focused element), moves the overlay over that display, and emits `computer://glow`.
use super::control::Control;
use super::server::Endpoint;
@@ -11,6 +16,10 @@ use std::sync::Arc;
use tauri::{AppHandle, Emitter, Manager, State};
pub const STATE_EVENT: &str = "computer://state";
+/// Where the agent just acted, sent to the overlay only (S28).
+pub const GLOW_EVENT: &str = "computer://glow";
+pub const GLOW_WINDOW: &str = "glow";
+pub const GLOW_TITLE: &str = "Gentle Dot Glow";
/// ⌥⇧Esc ends the session from anywhere (S24.3).
pub const PANIC_SHORTCUT: &str = "Alt+Shift+Escape";
pub const GRANT_TEXT: &str = "Allow Gentle Dot to see and control this Mac for 30 minutes?";
@@ -30,9 +39,19 @@ pub struct Computer {
desktop: Arc<dyn super::Desktop>,
}
-/// Creates the session and, on macOS, starts the MCP helper and the timeout watcher.
+/// What the overlay thread hears: the session started or ended, or the agent acted.
+#[cfg_attr(not(target_os = "macos"), allow(dead_code))]
+enum Overlay {
+ Session(bool),
+ Mark(super::glow::Mark),
+}
+
+/// Creates the session and, on macOS, starts the MCP helper, the timeout watcher, and the
+/// glow overlay. Runs in the app's setup, on the main thread.
pub fn start(app: &AppHandle) -> Computer {
+ let (overlay, marks) = std::sync::mpsc::channel::<Overlay>();
let events = app.clone();
+ let session = overlay.clone();
let control = Arc::new(Control::new(move |event: &StateEvent| {
if let Err(error) = events.emit(STATE_EVENT, event) {
eprintln!("gentle-dot: cannot report the computer-control state: {error}");
@@ -40,18 +59,30 @@ pub fn start(app: &AppHandle) -> Computer {
if let Some(item) = events.try_state::<YoloMenuItem>() {
let _ = item.0.set_checked(event.yolo);
}
+ let _ = session.send(Overlay::Session(event.active));
}));
#[cfg(target_os = "macos")]
{
- let desktop: Arc<dyn super::Desktop> = Arc::new(super::MacDesktop::default());
- let endpoint = start_helper(app, control.clone(), desktop.clone())
+ let number = Arc::new(std::sync::atomic::AtomicU32::new(0));
+ let desktop: Arc<dyn super::Desktop> = Arc::new(super::MacDesktop::new(number.clone()));
+ let glow = move |mark| {
+ let _ = overlay.send(Overlay::Mark(mark));
+ };
+ let endpoint = start_helper(app, control.clone(), desktop.clone(), glow)
.map_err(|error| eprintln!("gentle-dot: computer control is unavailable: {error}"))
.ok();
watch_timeout(control.clone());
+ match build_overlay(app, &number) {
+ Ok(window) => run_overlay(app.clone(), window, number, control.clone(), marks),
+ Err(error) => eprintln!("gentle-dot: the computer-control glow is unavailable: {error}"),
+ }
Computer { control, endpoint, desktop }
}
#[cfg(not(target_os = "macos"))]
- Computer { control, endpoint: None }
+ {
+ drop((overlay, marks));
+ Computer { control, endpoint: None }
+ }
}
#[cfg(target_os = "macos")]
@@ -59,6 +90,7 @@ fn start_helper(
app: &AppHandle,
control: Arc<Control>,
desktop: Arc<dyn super::Desktop>,
+ glow: impl Fn(super::glow::Mark) + Send + Sync + 'static,
) -> std::io::Result<Endpoint> {
let token = super::server::new_token();
let helper = super::mcp::Helper::new(
@@ -68,10 +100,112 @@ fn start_helper(
control,
token.clone(),
std::process::id() as i32,
- );
+ )
+ .with_glow(glow);
super::server::start(Arc::new(helper), token)
}
+/// The overlay's NSWindow, used on the main thread only (the window lives as long as the app).
+#[cfg(target_os = "macos")]
+#[derive(Clone, Copy)]
+struct NsWindow(usize);
+
+/// The click-through overlay (S28): transparent, undecorated, never focusable, on every Space,
+/// above menus, ignoring the mouse, and hidden until a session starts. Its window number lets
+/// the helper skip it when it resolves which app owns a point.
+#[cfg(target_os = "macos")]
+fn build_overlay(
+ app: &AppHandle,
+ number: &std::sync::atomic::AtomicU32,
+) -> tauri::Result<(tauri::WebviewWindow, NsWindow)> {
+ let url = tauri::WebviewUrl::App("index.html?surface=glow".into());
+ let window = tauri::WebviewWindowBuilder::new(app, GLOW_WINDOW, url)
+ .title(GLOW_TITLE)
+ .inner_size(1.0, 1.0)
+ .position(0.0, 0.0)
+ .transparent(true)
+ .decorations(false)
+ .always_on_top(true)
+ .resizable(false)
+ .skip_taskbar(true)
+ .visible_on_all_workspaces(true)
+ .shadow(false)
+ .focused(false)
+ .focusable(false)
+ .visible(false)
+ .build()?;
+ window.set_ignore_cursor_events(true)?;
+ let ns_window = window.ns_window()?;
+ // SAFETY: setup runs on the main thread, and `ns_window` is the live window just built.
+ number.store(unsafe { super::macos::prepare_overlay(ns_window) }, std::sync::atomic::Ordering::SeqCst);
+ Ok((window, NsWindow(ns_window as usize)))
+}
+
+/// Orders the overlay in or out on the main thread and records its window number.
+#[cfg(target_os = "macos")]
+fn order_overlay(app: &AppHandle, ns_window: NsWindow, number: &Arc<std::sync::atomic::AtomicU32>, visible: bool) {
+ let number = number.clone();
+ let ordered = app.run_on_main_thread(move || {
+ // SAFETY: on the main thread, with the overlay's live NSWindow.
+ let n = unsafe { super::macos::order_overlay(ns_window.0 as *mut std::ffi::c_void, visible) };
+ if n != 0 {
+ number.store(n, std::sync::atomic::Ordering::SeqCst);
+ }
+ });
+ if let Err(error) = ordered {
+ eprintln!("gentle-dot: cannot show or hide the glow: {error}");
+ }
+}
+
+/// The overlay's own thread: follows the session and draws each mark while it lasts.
+#[cfg(target_os = "macos")]
+fn run_overlay(
+ app: AppHandle,
+ (window, ns_window): (tauri::WebviewWindow, NsWindow),
+ number: Arc<std::sync::atomic::AtomicU32>,
+ control: Arc<Control>,
+ marks: std::sync::mpsc::Receiver<Overlay>,
+) {
+ use tauri::{LogicalPosition, LogicalSize};
+ let spawned = std::thread::Builder::new().name("computer-glow".into()).spawn(move || {
+ let (mut shown, mut covering) = (false, None);
+ for message in marks {
+ match message {
+ Overlay::Session(active) => {
+ if active != shown {
+ order_overlay(&app, ns_window, &number, active);
+ shown = active;
+ }
+ }
+ Overlay::Mark(mark) => {
+ // A mark that arrives after Stop or panic is dropped.
+ if !control.is_active(SystemClock.now_ms()) {
+ continue;
+ }
+ let placed =
+ super::glow::place(mark, super::macos::display_frame_at, super::macos::focused_frame);
+ let Some((display, event)) = placed else { continue };
+ if covering != Some(display) {
+ let _ = window.set_position(LogicalPosition::new(display.x, display.y));
+ let _ = window.set_size(LogicalSize::new(display.width, display.height));
+ covering = Some(display);
+ }
+ if !shown {
+ order_overlay(&app, ns_window, &number, true);
+ shown = true;
+ }
+ if let Err(error) = app.emit_to(GLOW_WINDOW, GLOW_EVENT, event) {
+ eprintln!("gentle-dot: cannot draw the glow: {error}");
+ }
+ }
+ }
+ }
+ });
+ if let Err(error) = spawned {
+ eprintln!("gentle-dot: the computer-control glow is unavailable: {error}");
+ }
+}
+
/// Reports the timeout when it happens, not only at the next tool call.
#[cfg(target_os = "macos")]
fn watch_timeout(control: Arc<Control>) {
diff --git a/apps/desktop/src-tauri/src/computer/glow.rs b/apps/desktop/src-tauri/src/computer/glow.rs
new file mode 100644
index 0000000..60884d0
--- /dev/null
+++ b/apps/desktop/src-tauri/src/computer/glow.rs
@@ -0,0 +1,173 @@
+//! Where the agent acts (S28): after each action that went through, the helper reports a
+//! `Mark` in global points, and the app draws a pink glow there in a click-through overlay.
+//! `place` picks the display the overlay covers and turns the mark into the overlay's own
+//! coordinates (points from the display's top-left corner).
+
+use super::coords::Point;
+use serde::Serialize;
+
+/// A rectangle in global points.
+#[derive(Debug, Clone, Copy, PartialEq)]
+pub struct Frame {
+ pub x: f64,
+ pub y: f64,
+ pub width: f64,
+ pub height: f64,
+}
+
+impl Frame {
+ pub fn contains(&self, at: Point) -> bool {
+ at.x >= self.x && at.y >= self.y && at.x < self.x + self.width && at.y < self.y + self.height
+ }
+
+ fn center(&self) -> Point {
+ Point { x: self.x + self.width / 2.0, y: self.y + self.height / 2.0 }
+ }
+}
+
+/// What the agent just did, and where, in global points.
+#[derive(Debug, Clone, Copy, PartialEq)]
+pub enum Mark {
+ /// A pulse.
+ Click(Point),
+ /// A short marker.
+ Move(Point),
+ /// A short marker.
+ Scroll(Point),
+ /// A trail from one point to the other.
+ Drag { from: Point, to: Point },
+ /// Keys went to the focused element; the app looks up where it is.
+ Key,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
+#[serde(rename_all = "lowercase")]
+pub enum GlowKind {
+ Click,
+ Move,
+ Scroll,
+ Drag,
+ Key,
+}
+
+/// The `computer://glow` event for the overlay, in points from its top-left corner.
+#[derive(Debug, Clone, Copy, PartialEq, Serialize)]
+#[serde(rename_all = "camelCase")]
+pub struct GlowEvent {
+ pub kind: GlowKind,
+ pub x: f64,
+ pub y: f64,
+ #[serde(skip_serializing_if = "Option::is_none")]
+ pub to_x: Option<f64>,
+ #[serde(skip_serializing_if = "Option::is_none")]
+ pub to_y: Option<f64>,
+ #[serde(skip_serializing_if = "Option::is_none")]
+ pub width: Option<f64>,
+ #[serde(skip_serializing_if = "Option::is_none")]
+ pub height: Option<f64>,
+}
+
+/// The display the overlay covers for `mark` and the event to draw on it, or `None` when the
+/// position is unknown (a focused element without a frame, a point on no display). A drag
+/// glows on the display where it starts; `focus` runs only for keyboard marks.
+pub fn place(
+ mark: Mark,
+ display_at: impl Fn(Point) -> Option<Frame>,
+ focus: impl FnOnce() -> Option<Frame>,
+) -> Option<(Frame, GlowEvent)> {
+ let at = |kind, point: Point| GlowEvent {
+ kind,
+ x: point.x,
+ y: point.y,
+ to_x: None,
+ to_y: None,
+ width: None,
+ height: None,
+ };
+ let (anchor, event) = match mark {
+ Mark::Click(point) => (point, at(GlowKind::Click, point)),
+ Mark::Move(point) => (point, at(GlowKind::Move, point)),
+ Mark::Scroll(point) => (point, at(GlowKind::Scroll, point)),
+ Mark::Drag { from, to } => {
+ (from, GlowEvent { to_x: Some(to.x), to_y: Some(to.y), ..at(GlowKind::Drag, from) })
+ }
+ Mark::Key => {
+ let frame = focus().filter(|f| f.width > 0.0 && f.height > 0.0)?;
+ let origin = Point { x: frame.x, y: frame.y };
+ let event = GlowEvent { width: Some(frame.width), height: Some(frame.height), ..at(GlowKind::Key, origin) };
+ (frame.center(), event)
+ }
+ };
+ let display = display_at(anchor)?;
+ let local = |x: f64, y: f64| (x - display.x, y - display.y);
+ let (x, y) = local(event.x, event.y);
+ let to = event.to_x.zip(event.to_y).map(|(tx, ty)| local(tx, ty));
+ Some((display, GlowEvent { x, y, to_x: to.map(|t| t.0), to_y: to.map(|t| t.1), ..event }))
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use serde_json::json;
+
+ /// Two displays side by side: the main one at the origin, a second one to its right.
+ fn display_at(at: Point) -> Option<Frame> {
+ let main = Frame { x: 0.0, y: 0.0, width: 1440.0, height: 900.0 };
+ let second = Frame { x: 1440.0, y: -100.0, width: 1920.0, height: 1080.0 };
+ [main, second].into_iter().find(|frame| frame.contains(at))
+ }
+
+ fn no_focus() -> Option<Frame> {
+ None
+ }
+
+ fn p(x: f64, y: f64) -> Point {
+ Point { x, y }
+ }
+
+ #[test]
+ fn a_click_glows_at_its_point_on_the_display_it_is_on() {
+ let (display, event) = place(Mark::Click(p(1500.0, 20.0)), display_at, no_focus).unwrap();
+ assert_eq!(display, Frame { x: 1440.0, y: -100.0, width: 1920.0, height: 1080.0 });
+ assert_eq!(serde_json::to_value(event).unwrap(), json!({"kind": "click", "x": 60.0, "y": 120.0}));
+ }
+
+ #[test]
+ fn move_and_scroll_are_short_markers() {
+ let (_, moved) = place(Mark::Move(p(10.0, 20.0)), display_at, no_focus).unwrap();
+ let (_, scrolled) = place(Mark::Scroll(p(30.0, 40.0)), display_at, no_focus).unwrap();
+ assert_eq!(serde_json::to_value(moved).unwrap(), json!({"kind": "move", "x": 10.0, "y": 20.0}));
+ assert_eq!(serde_json::to_value(scrolled).unwrap(), json!({"kind": "scroll", "x": 30.0, "y": 40.0}));
+ }
+
+ #[test]
+ fn a_drag_is_a_trail_on_the_display_where_it_starts() {
+ let drag = Mark::Drag { from: p(100.0, 100.0), to: p(300.0, 250.0) };
+ let (display, event) = place(drag, display_at, no_focus).unwrap();
+ assert_eq!(display.x, 0.0);
+ assert_eq!(
+ serde_json::to_value(event).unwrap(),
+ json!({"kind": "drag", "x": 100.0, "y": 100.0, "toX": 300.0, "toY": 250.0})
+ );
+ }
+
+ #[test]
+ fn a_keyboard_action_glows_around_the_focused_element() {
+ let field = || Some(Frame { x: 1600.0, y: 0.0, width: 200.0, height: 30.0 });
+ let (display, event) = place(Mark::Key, display_at, field).unwrap();
+ assert_eq!(display.x, 1440.0);
+ assert_eq!(
+ serde_json::to_value(event).unwrap(),
+ json!({"kind": "key", "x": 160.0, "y": 100.0, "width": 200.0, "height": 30.0})
+ );
+ }
+
+ #[test]
+ fn nothing_glows_where_the_position_is_unknown() {
+ // The focused element has no frame, or the point is on no display.
+ assert_eq!(place(Mark::Key, display_at, no_focus), None);
+ assert_eq!(place(Mark::Click(p(-5000.0, 0.0)), display_at, no_focus), None);
+ let empty = || Some(Frame { x: 10.0, y: 10.0, width: 0.0, height: 0.0 });
+ assert_eq!(place(Mark::Key, display_at, empty), None);
+ }
+}
diff --git a/apps/desktop/src-tauri/src/computer/macos.rs b/apps/desktop/src-tauri/src/computer/macos.rs
index 97b5543..1282163 100644
--- a/apps/desktop/src-tauri/src/computer/macos.rs
+++ b/apps/desktop/src-tauri/src/computer/macos.rs
@@ -1,22 +1,28 @@
//! The real Mac behind `Desktop`: ScreenCaptureKit screenshots (macOS 14+), Quartz events,
//! the Accessibility API, and NSWorkspace. Nothing here decides policy; `mcp` does.
+//! It also reads what the glow overlay (S28) needs: display frames, the focused element's
+//! frame, and the overlay window's own setup.
use super::blocklist::AppIdentity;
use super::coords::{fit_long_edge, Point, ScreenshotGeometry, MAX_LONG_EDGE};
use super::encode::{bgra_to_jpeg, JPEG_QUALITY};
+use super::glow::Frame;
use super::keys::KeyCombo;
use super::risk::{AxElement, Focus};
-use super::windows::{owner_row_at, WindowRow};
+use super::windows::{hit_test_app, owner_row_at, WindowRow};
use super::{AppInfo, Capture, Desktop, InputEvent, MouseButton, PermissionKind, Permissions};
use block2::RcBlock;
use objc2::rc::Retained;
-use objc2::{available, AnyThread};
+use objc2::runtime::AnyObject;
+use objc2::{available, msg_send, AnyThread};
use objc2_app_kit::{NSApplicationActivationOptions, NSApplicationActivationPolicy, NSRunningApplication, NSWorkspace};
-use objc2_application_services::{AXError, AXIsProcessTrusted, AXIsProcessTrustedWithOptions, AXUIElement};
+use objc2_application_services::{
+ AXError, AXIsProcessTrusted, AXIsProcessTrustedWithOptions, AXUIElement, AXValue, AXValueType,
+};
use objc2_core_foundation::{CFBoolean, CFDictionary, CFRetained, CFString, CFType, CGPoint, CGRect, CGSize};
use objc2_core_graphics::{
- kCGNullWindowID, kCGWindowAlpha, kCGWindowBounds, kCGWindowLayer, kCGWindowOwnerName, kCGWindowOwnerPID,
- CGDataProvider, CGDirectDisplayID, CGError, CGEvent, CGEventField, CGEventFlags,
+ kCGNullWindowID, kCGWindowAlpha, kCGWindowBounds, kCGWindowLayer, kCGWindowNumber, kCGWindowOwnerName,
+ kCGWindowOwnerPID, CGDataProvider, CGDirectDisplayID, CGDisplayBounds, CGError, CGEvent, CGEventField, CGEventFlags,
CGEventTapLocation, CGEventType, CGGetDisplaysWithPoint, CGImage, CGMouseButton,
CGPreflightScreenCaptureAccess, CGRequestScreenCaptureAccess, CGScrollEventUnit, CGWindowListCopyWindowInfo,
CGWindowListOption,
@@ -25,8 +31,10 @@ use objc2_foundation::{NSArray, NSDictionary, NSError, NSNumber, NSString};
use objc2_screen_capture_kit::{
SCContentFilter, SCScreenshotManager, SCShareableContent, SCStreamConfiguration, SCWindow,
};
+use std::ffi::c_void;
use std::ptr::NonNull;
-use std::sync::mpsc;
+use std::sync::atomic::{AtomicU32, Ordering};
+use std::sync::{mpsc, Arc};
use std::thread;
use std::time::Duration;
@@ -46,11 +54,33 @@ unsafe impl<T> Send for Handoff<T> {}
pub struct MacDesktop {
own_pid: i32,
+ /// The glow overlay's window number (S28), 0 until it exists.
+ overlay: Arc<AtomicU32>,
}
-impl Default for MacDesktop {
- fn default() -> Self {
- MacDesktop { own_pid: std::process::id() as i32 }
+impl MacDesktop {
+ pub fn new(overlay: Arc<AtomicU32>) -> Self {
+ MacDesktop { own_pid: std::process::id() as i32, overlay }
+ }
+
+ fn overlay(&self) -> Option<u32> {
+ Some(self.overlay.load(Ordering::SeqCst)).filter(|&n| n != 0)
+ }
+
+ /// The element under `at`. A hit test that went through the click-through overlay lands on
+ /// Gentle Dot, so it is redone inside the app that owns the window there.
+ fn hit_test(&self, at: Point) -> Option<CFRetained<AXUIElement>> {
+ let element = ax_element_at(&system_wide(), at)?;
+ let Some(hit_pid) = ax_pid(&element) else {
+ return Some(element);
+ };
+ let rows = window_rows();
+ let owner = owner_row_at(&rows, at, self.overlay()).map(|row| row.pid);
+ match hit_test_app(self.own_pid, hit_pid, owner) {
+ // SAFETY: plain constructor.
+ Some(pid) => ax_element_at(&with_timeout(unsafe { AXUIElement::new_application(pid) }), at),
+ None => Some(element),
+ }
}
}
@@ -256,12 +286,13 @@ fn ax_pid(element: &AXUIElement) -> Option<i32> {
(error == AXError::Success && pid > 0).then_some(pid)
}
-/// The Accessibility element at a global point, as hit-tested by the app that owns it.
-fn ax_element_at(at: Point) -> Option<CFRetained<AXUIElement>> {
+/// The Accessibility element at a global point, as hit-tested by the app that owns it
+/// (`scope` is the system-wide element, or one app's element to test only that app).
+fn ax_element_at(scope: &AXUIElement, at: Point) -> Option<CFRetained<AXUIElement>> {
let mut found: *const AXUIElement = std::ptr::null();
// SAFETY: `found` is a valid out-pointer; on success it holds a +1 reference.
let out = NonNull::from(&mut found);
- let error = unsafe { system_wide().copy_element_at_position(at.x as f32, at.y as f32, out) };
+ let error = unsafe { scope.copy_element_at_position(at.x as f32, at.y as f32, out) };
let found = NonNull::new(found.cast_mut()).filter(|_| error == AXError::Success)?;
Some(unsafe { CFRetained::from_raw(found) })
}
@@ -277,12 +308,82 @@ fn describe(element: &AXUIElement) -> AxElement {
fn system_wide() -> CFRetained<AXUIElement> {
// SAFETY: plain constructor.
- let element = unsafe { AXUIElement::new_system_wide() };
- // A hung app must not stall the helper.
+ with_timeout(unsafe { AXUIElement::new_system_wide() })
+}
+
+/// A hung app must not stall the helper.
+fn with_timeout(element: CFRetained<AXUIElement>) -> CFRetained<AXUIElement> {
+ // SAFETY: setting a timeout on a valid element.
unsafe { element.set_messaging_timeout(0.5) };
element
}
+/// A `CGPoint` or `CGSize` attribute (`AXPosition`, `AXSize`).
+fn ax_struct<T: Default>(element: &AXUIElement, attribute: &str, kind: AXValueType) -> Option<T> {
+ let value = ax_value(element, attribute)?.downcast::<AXValue>().ok()?;
+ let mut out = T::default();
+ // SAFETY: `out` has the layout of `kind` (CGPoint or CGSize).
+ let ok = unsafe { value.value(kind, NonNull::from(&mut out).cast()) };
+ ok.then_some(out)
+}
+
+/// The focused element's frame in global points, when it reports one (S28).
+pub fn focused_frame() -> Option<Frame> {
+ let focused = ax_element(&system_wide(), "AXFocusedUIElement")?;
+ let origin: CGPoint = ax_struct(&focused, "AXPosition", AXValueType::CGPoint)?;
+ let size: CGSize = ax_struct(&focused, "AXSize", AXValueType::CGSize)?;
+ Some(Frame { x: origin.x, y: origin.y, width: size.width, height: size.height })
+}
+
+/// The frame of the display showing `at`, in global points (S28).
+pub fn display_frame_at(at: Point) -> Option<Frame> {
+ let bounds = CGDisplayBounds(display_at(at)?);
+ Some(Frame { x: bounds.origin.x, y: bounds.origin.y, width: bounds.size.width, height: bounds.size.height })
+}
+
+/// `NSPopUpMenuWindowLevel + 1`: the glow shows above menus the agent clicks.
+const OVERLAY_LEVEL: isize = 102;
+/// `canJoinAllSpaces | stationary | ignoresCycle | fullScreenAuxiliary`.
+const OVERLAY_COLLECTION: usize = (1 << 0) | (1 << 4) | (1 << 6) | (1 << 8);
+
+/// Readies the glow overlay's NSWindow: above menus, on every Space (full-screen ones too),
+/// out of window cycling, never taking mouse events. Main thread only. Returns its number.
+///
+/// # Safety
+/// `ns_window` must be a live NSWindow.
+pub unsafe fn prepare_overlay(ns_window: *mut c_void) -> u32 {
+ let window = unsafe { &*ns_window.cast::<AnyObject>() };
+ unsafe {
+ let _: () = msg_send![window, setLevel: OVERLAY_LEVEL];
+ let _: () = msg_send![window, setCollectionBehavior: OVERLAY_COLLECTION];
+ let _: () = msg_send![window, setIgnoresMouseEvents: true];
+ let _: () = msg_send![window, setHasShadow: false];
+ overlay_number(window)
+ }
+}
+
+/// Orders the overlay in without making it key or activating the app, or orders it out.
+/// Main thread only. Returns its window number.
+///
+/// # Safety
+/// `ns_window` must be a live NSWindow.
+pub unsafe fn order_overlay(ns_window: *mut c_void, visible: bool) -> u32 {
+ let window = unsafe { &*ns_window.cast::<AnyObject>() };
+ unsafe {
+ if visible {
+ let _: () = msg_send![window, orderFrontRegardless];
+ } else {
+ let _: () = msg_send![window, orderOut: std::ptr::null::<AnyObject>()];
+ }
+ overlay_number(window)
+ }
+}
+
+unsafe fn overlay_number(window: &AnyObject) -> u32 {
+ let number: isize = unsafe { msg_send![window, windowNumber] };
+ u32::try_from(number).unwrap_or(0)
+}
+
type WindowInfo = NSDictionary<NSString, objc2::runtime::AnyObject>;
/// A window-list key (a `CFString` constant) as the bridged `NSString`.
@@ -312,14 +413,21 @@ fn window_rows() -> Vec<WindowRow> {
// SAFETY: CFArray of CFDictionary is toll-free bridged to NSArray of NSDictionary.
let list: &NSArray<WindowInfo> = unsafe { &*(CFRetained::as_ptr(&list).as_ptr() as *const _) };
// SAFETY: extern constants provided by CoreGraphics.
- let (alpha_key, layer_key, pid_key, name_key) = unsafe {
- (info_key(kCGWindowAlpha), info_key(kCGWindowLayer), info_key(kCGWindowOwnerPID), info_key(kCGWindowOwnerName))
+ let (alpha_key, layer_key, pid_key, name_key, number_key) = unsafe {
+ (
+ info_key(kCGWindowAlpha),
+ info_key(kCGWindowLayer),
+ info_key(kCGWindowOwnerPID),
+ info_key(kCGWindowOwnerName),
+ info_key(kCGWindowNumber),
+ )
};
let number = |entry: &WindowInfo, key| entry.objectForKey(key).and_then(|n| n.downcast::<NSNumber>().ok());
list.iter()
.filter_map(|entry| {
let bounds = window_bounds(&entry)?;
Some(WindowRow {
+ number: number(&entry, number_key).map_or(0, |n| n.unsignedIntValue()),
pid: number(&entry, pid_key)?.intValue(),
owner_name: entry
.objectForKey(name_key)
@@ -421,7 +529,7 @@ impl Desktop for MacDesktop {
}
fn element_at(&self, at: Point) -> Vec<AxElement> {
- let Some(mut element) = ax_element_at(at) else {
+ let Some(mut element) = self.hit_test(at) else {
return Vec::new();
};
let mut chain = vec![describe(&element)];
@@ -449,13 +557,14 @@ impl Desktop for MacDesktop {
ax_pid(&focused).map(|pid| identity_of(pid, ""))
}
- /// The owner of the frontmost app window under the point (the pointer and other window
- /// server layers skipped), else the app whose Accessibility element is there.
+ /// The owner of the frontmost app window under the point (the pointer, other window
+ /// server layers, and the glow overlay skipped), else the app whose Accessibility element
+ /// is there (Gentle Dot, and so refused, if that lands on the overlay).
fn window_owner_at(&self, at: Point) -> Option<AppIdentity> {
- if let Some(row) = owner_row_at(&window_rows(), at) {
+ if let Some(row) = owner_row_at(&window_rows(), at, self.overlay()) {
return Some(identity_of(row.pid, &row.owner_name));
}
- let element = ax_element_at(at)?;
+ let element = ax_element_at(&system_wide(), at)?;
ax_pid(&element).map(|pid| identity_of(pid, ""))
}
diff --git a/apps/desktop/src-tauri/src/computer/mcp.rs b/apps/desktop/src-tauri/src/computer/mcp.rs
index eba4469..c8700ca 100644
--- a/apps/desktop/src-tauri/src/computer/mcp.rs
+++ b/apps/desktop/src-tauri/src/computer/mcp.rs
@@ -7,10 +7,15 @@
//! (`perform`) and counts as one action for the rate limit; a full window delays the step
//! instead of refusing it. Coordinates refer to the screenshot taken before the batch, and one
//! screenshot follows it. The first refused, declined, or failed step stops the batch.
+//!
+//! The glow (S28): once an action's input is posted, `perform` reports where it acted to the
+//! glow sink, so a refused, declined, stopped, or failed action (or batch step) reports
+//! nothing. The sink only queues the mark; it never changes a check, the order, or a result.
use super::blocklist::{blocked_name, blocked_reason, AppIdentity};
use super::control::Control;
use super::coords::{Point, ScreenshotGeometry};
+use super::glow::Mark;
use super::keys::{parse_combo, KeyCombo};
use super::rate::{RateLimiter, MAX_ACTIONS_PER_SECOND};
use super::risk::{assess, Act};
@@ -307,6 +312,8 @@ pub struct Helper {
limiter: Mutex<RateLimiter>,
/// Serializes actions and holds the geometry of the latest screenshot.
actions: Mutex<Option<ScreenshotGeometry>>,
+ /// Where an action that went through acted (S28); must return at once.
+ glow: Box<dyn Fn(Mark) + Send + Sync>,
}
impl Helper {
@@ -327,9 +334,16 @@ impl Helper {
own_pid,
limiter: Mutex::new(RateLimiter::new(MAX_ACTIONS_PER_SECOND, 1000)),
actions: Mutex::new(None),
+ glow: Box::new(|_| {}),
}
}
+ /// Reports each action that went through to `glow`, which must only queue it (S28).
+ pub fn with_glow(mut self, glow: impl Fn(Mark) + Send + Sync + 'static) -> Self {
+ self.glow = Box::new(glow);
+ self
+ }
+
fn authorized(&self, header: Option<&str>) -> bool {
let Some((scheme, token)) = header.and_then(|h| h.trim().split_once(' ')) else {
return false;
@@ -637,6 +651,7 @@ impl Helper {
};
self.confirm_if_risky(assess(Act::Click(&chain), intent), &action, &target)?;
self.post(epoch, InputEvent::Click { at, button, count })?;
+ (self.glow)(Mark::Click(at));
Ok((format!("{action} in {target}: done."), SETTLE_MS))
}
Tool::Move { x, y } => {
@@ -645,6 +660,7 @@ impl Helper {
let action = format!("Move the pointer to ({x}, {y})");
self.confirm_if_risky(assess(Act::Other, intent), &action, &target)?;
self.post(epoch, InputEvent::Move(at))?;
+ (self.glow)(Mark::Move(at));
Ok((format!("{action}: done."), SETTLE_MS))
}
Tool::Drag { from, to } => {
@@ -656,6 +672,7 @@ impl Helper {
let (from_chain, to_chain) = (self.desktop.element_at(start), self.desktop.element_at(end));
self.confirm_if_risky(assess(Act::Drag(&from_chain, &to_chain), intent), &action, &target)?;
self.post(epoch, InputEvent::Drag { from: start, to: end })?;
+ (self.glow)(Mark::Drag { from: start, to: end });
Ok((format!("{action}: done."), SETTLE_MS))
}
Tool::Scroll { x, y, dx, dy } => {
@@ -664,6 +681,7 @@ impl Helper {
let action = format!("Scroll by ({dx}, {dy}) at ({x}, {y})");
self.confirm_if_risky(assess(Act::Other, intent), &action, &target)?;
self.post(epoch, InputEvent::Scroll { at, dx, dy })?;
+ (self.glow)(Mark::Scroll(at));
Ok((format!("{action}: done."), SETTLE_MS))
}
Tool::Type { text: typed } => {
@@ -681,6 +699,7 @@ impl Helper {
self.post(epoch, InputEvent::Text(chunk))?;
}
}
+ (self.glow)(Mark::Key);
Ok((format!("{action} in {target}: done."), SETTLE_MS))
}
Tool::Key { combo } => {
@@ -696,6 +715,7 @@ impl Helper {
let action = format!("Press {}", combo.key);
self.confirm_if_risky(assess(act, intent), &action, &target)?;
self.post(epoch, InputEvent::Key(combo))?;
+ (self.glow)(Mark::Key);
Ok((format!("{action} in {target}: done."), SETTLE_MS))
}
Tool::OpenApp { name } => {
@@ -727,6 +747,7 @@ mod tests {
use super::super::blocklist::AppIdentity;
use super::super::coords::Point;
use super::super::fake::{app, FakeClock, FakeDesktop, FakeDialogs};
+ use super::super::glow::Mark;
use super::super::keys::parse_combo;
use super::super::risk::{AxElement, Focus};
use super::super::session::{Reason, StateEvent, SESSION_MS};
@@ -747,6 +768,8 @@ mod tests {
clock: Arc<FakeClock>,
control: Arc<Control>,
states: Arc<Mutex<Vec<StateEvent>>>,
+ /// Each glow mark (S28), with how many input events were posted when it came.
+ marks: Arc<Mutex<Vec<(Mark, usize)>>>,
}
fn rig_with(allow: bool, confirm: bool) -> Rig {
@@ -756,9 +779,26 @@ mod tests {
let states = Arc::new(Mutex::new(Vec::new()));
let sink = states.clone();
let control = Arc::new(Control::new(move |event| sink.lock().unwrap().push(event.clone())));
+ let marks = Arc::new(Mutex::new(Vec::new()));
+ let (seen, posted) = (marks.clone(), desktop.clone());
let helper =
- Helper::new(desktop.clone(), dialogs.clone(), clock.clone(), control.clone(), TOKEN.into(), OWN_PID);
- Rig { helper, desktop, dialogs, clock, control, states }
+ Helper::new(desktop.clone(), dialogs.clone(), clock.clone(), control.clone(), TOKEN.into(), OWN_PID)
+ .with_glow(move |mark| seen.lock().unwrap().push((mark, posted.events().len())));
+ Rig { helper, desktop, dialogs, clock, control, states, marks }
+ }
+
+ /// A rig whose helper has no glow sink, as before S28.
+ fn rig_without_glow(allow: bool, confirm: bool) -> Rig {
+ let rig = rig_with(allow, confirm);
+ let helper = Helper::new(
+ rig.desktop.clone(),
+ rig.dialogs.clone(),
+ rig.clock.clone(),
+ rig.control.clone(),
+ TOKEN.into(),
+ OWN_PID,
+ );
+ Rig { helper, ..rig }
}
fn rig() -> Rig {
@@ -1649,4 +1689,139 @@ mod tests {
let apps: Value = serde_json::from_str(&first_text(&result)).unwrap();
assert_eq!(apps[0], json!({"name": "Safari", "bundleId": "com.apple.Safari", "active": true}));
}
+
+ // --- The pink glow (S28) ---
+
+ fn marks(rig: &Rig) -> Vec<(Mark, usize)> {
+ rig.marks.lock().unwrap().clone()
+ }
+
+ #[test]
+ fn each_action_that_went_through_marks_where_it_acted_after_posting() {
+ let rig = rig();
+ rig.ready();
+ let p = |x, y| Point { x, y };
+ rig.call("click", json!({"x": 100, "y": 50}));
+ rig.call("move", json!({"x": 1, "y": 2}));
+ rig.call("drag", json!({"fromX": 10, "fromY": 20, "toX": 30, "toY": 40}));
+ rig.call("scroll", json!({"x": 5, "y": 6, "dx": 0, "dy": 120}));
+ rig.call("key", json!({"combo": "tab"}));
+ // A text with a newline is three input events and one mark, after the last of them.
+ rig.call("type", json!({"text": "hi\nthere"}));
+ assert_eq!(
+ marks(&rig),
+ vec![
+ (Mark::Click(p(200.0, 100.0)), 1),
+ (Mark::Move(p(2.0, 4.0)), 2),
+ (Mark::Drag { from: p(20.0, 40.0), to: p(60.0, 80.0) }, 3),
+ (Mark::Scroll(p(10.0, 12.0)), 4),
+ (Mark::Key, 5),
+ (Mark::Key, 8),
+ ]
+ );
+ }
+
+ #[test]
+ fn refused_declined_and_stopped_actions_mark_nothing() {
+ // Declined: a risky click the user says no to.
+ let declined = rig_with(true, false);
+ declined.ready();
+ *declined.desktop.elements.lock().unwrap() = send_button();
+ assert!(first_text(&declined.call("click", json!({"x": 10, "y": 10}))).contains("declined"));
+ assert!(is_error(&declined.call("type", json!({"text": "hello\t "}))));
+ // Refused: a blocked window, a point no app owns, a blocked app in front, invalid arguments.
+ let rig_blocked = rig();
+ rig_blocked.ready();
+ *rig_blocked.desktop.owner_at.lock().unwrap() =
+ Box::new(|_| Some(app(300, "com.apple.systempreferences", "System Settings")));
+ assert!(is_error(&rig_blocked.call("click", json!({"x": 10, "y": 10}))));
+ assert!(is_error(&rig_blocked.call("drag", json!({"fromX": 1, "fromY": 1, "toX": 2, "toY": 2}))));
+ *rig_blocked.desktop.owner_at.lock().unwrap() = Box::new(|_| None);
+ assert!(is_error(&rig_blocked.call("scroll", json!({"x": 1, "y": 1, "dx": 0, "dy": 1}))));
+ *rig_blocked.desktop.frontmost.lock().unwrap() = Some(app(302, "com.apple.keychainaccess", "Keychain Access"));
+ assert!(is_error(&rig_blocked.call("key", json!({"combo": "cmd+a"}))));
+ assert!(is_error(&rig_blocked.call("click", json!({"x": 1}))));
+ // Stopped: queued before a panic, no session (denied), no screenshot yet.
+ let rig_stopped = rig();
+ rig_stopped.ready();
+ let queued_at = rig_stopped.control.epoch();
+ rig_stopped.control.end(Reason::Panic);
+ assert!(is_error(&rig_stopped.helper.call_tool_from(queued_at, "move", &json!({"x": 1, "y": 1})).unwrap()));
+ let rig_denied = rig_with(false, true);
+ assert!(is_error(&rig_denied.call("key", json!({"combo": "tab"}))));
+ let rig_fresh = rig();
+ assert!(is_error(&rig_fresh.call("click", json!({"x": 1, "y": 1}))));
+ for r in [&declined, &rig_blocked, &rig_stopped, &rig_denied, &rig_fresh] {
+ assert!(r.desktop.events().is_empty());
+ assert!(marks(r).is_empty(), "{:?}", marks(r));
+ }
+ }
+
+ #[test]
+ fn screenshots_lists_open_app_and_wait_mark_nothing() {
+ let rig = rig();
+ rig.ready();
+ rig.call("list_apps", json!({}));
+ rig.call("open_app", json!({"name": "Mail"}));
+ rig.call("wait", json!({"ms": 200}));
+ assert!(marks(&rig).is_empty());
+ }
+
+ #[test]
+ fn a_batch_marks_each_step_that_ran_and_none_after_it_stopped() {
+ let rig = rig();
+ rig.ready();
+ *rig.desktop.owner_at.lock().unwrap() = Box::new(|at: Point| {
+ if at.x > 100.0 {
+ Some(app(300, "com.apple.systempreferences", "System Settings"))
+ } else {
+ Some(safari())
+ }
+ });
+ let result = rig.call(
+ "batch",
+ batch(json!([
+ {"tool": "click", "x": 10, "y": 10},
+ {"tool": "wait", "ms": 0},
+ {"tool": "type", "text": "Ana"},
+ {"tool": "click", "x": 200, "y": 10},
+ {"tool": "click", "x": 20, "y": 20},
+ ])),
+ );
+ assert!(is_error(&result));
+ assert_eq!(marks(&rig), vec![(Mark::Click(Point { x: 20.0, y: 20.0 }), 1), (Mark::Key, 2)]);
+ }
+
+ #[test]
+ fn the_glow_changes_no_check_order_or_result() {
+ // The same calls, with and without the glow, give the same results, dialogs, and events.
+ let script = |rig: &Rig| -> Vec<Value> {
+ *rig.desktop.elements.lock().unwrap() = send_button();
+ let mut results = vec![rig.call("screenshot", json!({}))];
+ for (name, args) in [
+ ("click", json!({"x": 10, "y": 10})),
+ ("move", json!({"x": 3, "y": 3})),
+ ("type", json!({"text": "a\nb"})),
+ (
+ "batch",
+ batch(json!([
+ {"tool": "scroll", "x": 1, "y": 1, "dx": 0, "dy": 5},
+ {"tool": "click", "x": 9, "y": 9},
+ ])),
+ ),
+ ("click", json!({"x": 5000, "y": 1})),
+ ] {
+ results.push(rig.call(name, args));
+ }
+ results
+ };
+ for confirm in [true, false] {
+ let (with, without) = (rig_with(true, confirm), rig_without_glow(true, confirm));
+ assert_eq!(script(&with), script(&without), "confirm {confirm}");
+ assert_eq!(with.desktop.events(), without.desktop.events());
+ assert_eq!(*with.dialogs.confirmations.lock().unwrap(), *without.dialogs.confirmations.lock().unwrap());
+ assert_eq!(with.clock.now_ms(), without.clock.now_ms());
+ assert!(without.marks.lock().unwrap().is_empty());
+ }
+ }
}
diff --git a/apps/desktop/src-tauri/src/computer/mod.rs b/apps/desktop/src-tauri/src/computer/mod.rs
index 390c7c7..5e619b5 100644
--- a/apps/desktop/src-tauri/src/computer/mod.rs
+++ b/apps/desktop/src-tauri/src/computer/mod.rs
@@ -10,6 +10,7 @@ pub mod blocklist;
pub mod control;
pub mod coords;
pub mod encode;
+pub mod glow;
#[cfg(test)]
mod fake;
pub mod keys;
diff --git a/apps/desktop/src-tauri/src/computer/windows.rs b/apps/desktop/src-tauri/src/computer/windows.rs
index bf7ecf2..5343386 100644
--- a/apps/desktop/src-tauri/src/computer/windows.rs
+++ b/apps/desktop/src-tauri/src/computer/windows.rs
@@ -1,6 +1,7 @@
//! Which window owns a point (S24.4), decided from the on-screen window list, front to back.
//! The list also holds windows no app owns (the pointer, the menu bar backdrop), which must
-//! never stand in for the app underneath.
+//! never stand in for the app underneath. Neither may Gentle Dot's click-through glow overlay
+//! (S28), skipped by its window number alone: the panel and the Dot still own their points.
use super::coords::Point;
@@ -10,6 +11,8 @@ pub const CURSOR_WINDOW_LEVEL: i64 = 2_147_483_630;
/// One entry of `CGWindowListCopyWindowInfo`, in global points.
#[derive(Debug, Clone, PartialEq)]
pub struct WindowRow {
+ /// `kCGWindowNumber`.
+ pub number: u32,
pub pid: i32,
pub owner_name: String,
pub layer: i64,
@@ -37,8 +40,18 @@ impl WindowRow {
}
/// The frontmost visible window under `at` that belongs to an app, or `None` when there is none.
-pub fn owner_row_at(rows: &[WindowRow], at: Point) -> Option<&WindowRow> {
- rows.iter().find(|row| row.alpha > 0.0 && row.belongs_to_an_app() && row.contains(at))
+/// `overlay` is the glow overlay's window number, once it exists.
+pub fn owner_row_at(rows: &[WindowRow], at: Point, overlay: Option<u32>) -> Option<&WindowRow> {
+ rows.iter().find(|row| {
+ Some(row.number) != overlay && row.alpha > 0.0 && row.belongs_to_an_app() && row.contains(at)
+ })
+}
+
+/// The app to hit-test again when the system-wide Accessibility hit test answered with
+/// Gentle Dot (`hit_pid == own_pid`) while another app owns the window under the point: the
+/// test went through the click-through overlay. `None` keeps the system-wide answer.
+pub fn hit_test_app(own_pid: i32, hit_pid: i32, owner_pid: Option<i32>) -> Option<i32> {
+ owner_pid.filter(|&owner| hit_pid == own_pid && owner != own_pid)
}
#[cfg(test)]
@@ -48,7 +61,15 @@ mod tests {
const AT: Point = Point { x: 500.0, y: 400.0 };
fn row(pid: i32, owner: &str, layer: i64, (x, y, width, height): (f64, f64, f64, f64)) -> WindowRow {
- WindowRow { pid, owner_name: owner.into(), layer, alpha: 1.0, x, y, width, height }
+ WindowRow { number: 0, pid, owner_name: owner.into(), layer, alpha: 1.0, x, y, width, height }
+ }
+
+ const OWN_PID: i32 = 4242;
+ const GLOW_WINDOW: u32 = 9001;
+
+ /// The glow overlay (S28) as a real Mac would list it: Gentle Dot's, over the whole display.
+ fn glow() -> WindowRow {
+ WindowRow { number: GLOW_WINDOW, ..row(OWN_PID, "Gentle Dot", 102, (0.0, 0.0, 1440.0, 900.0)) }
}
/// The pointer window as a real Mac lists it, right under the pointer and first in the list.
@@ -63,14 +84,14 @@ mod tests {
#[test]
fn the_pointer_window_is_skipped_for_the_app_underneath() {
let rows = [cursor(), safari()];
- assert_eq!(owner_row_at(&rows, AT).map(|r| r.pid), Some(200));
+ assert_eq!(owner_row_at(&rows, AT, None).map(|r| r.pid), Some(200));
}
#[test]
fn windows_owned_by_the_window_server_are_skipped() {
let backdrop = row(412, "WindowServer", 0, (0.0, 0.0, 1440.0, 900.0));
let rows = [row(412, "Window Server", 24, (0.0, 0.0, 1440.0, 900.0)), backdrop, safari()];
- assert_eq!(owner_row_at(&rows, AT).map(|r| r.pid), Some(200));
+ assert_eq!(owner_row_at(&rows, AT, None).map(|r| r.pid), Some(200));
}
#[test]
@@ -78,13 +99,13 @@ mod tests {
let clear = WindowRow { alpha: 0.0, ..row(300, "Overlay", 0, (0.0, 0.0, 1440.0, 900.0)) };
let elsewhere = row(301, "Notes", 0, (900.0, 600.0, 200.0, 200.0));
let rows = [clear, elsewhere, safari()];
- assert_eq!(owner_row_at(&rows, AT).map(|r| r.pid), Some(200));
+ assert_eq!(owner_row_at(&rows, AT, None).map(|r| r.pid), Some(200));
}
#[test]
fn rows_without_a_process_are_skipped() {
let rows = [row(0, "", 0, (0.0, 0.0, 1440.0, 900.0)), safari()];
- assert_eq!(owner_row_at(&rows, AT).map(|r| r.pid), Some(200));
+ assert_eq!(owner_row_at(&rows, AT, None).map(|r| r.pid), Some(200));
}
#[test]
@@ -92,12 +113,47 @@ mod tests {
// A lock or authorization prompt sits above every app window but below the pointer.
let prompt = row(150, "loginwindow", 2_147_483_628, (0.0, 0.0, 1440.0, 900.0));
let rows = [cursor(), prompt, safari()];
- assert_eq!(owner_row_at(&rows, AT).map(|r| r.pid), Some(150));
+ assert_eq!(owner_row_at(&rows, AT, None).map(|r| r.pid), Some(150));
}
#[test]
fn only_the_pointer_under_the_point_resolves_to_nothing() {
- assert_eq!(owner_row_at(&[cursor()], AT), None);
- assert_eq!(owner_row_at(&[], AT), None);
+ assert_eq!(owner_row_at(&[cursor()], AT, None), None);
+ assert_eq!(owner_row_at(&[], AT, None), None);
+ }
+
+ #[test]
+ fn the_glow_overlay_is_skipped_by_its_window_number() {
+ let rows = [cursor(), glow(), safari()];
+ assert_eq!(owner_row_at(&rows, AT, Some(GLOW_WINDOW)).map(|r| r.pid), Some(200));
+ }
+
+ #[test]
+ fn the_panel_and_the_dot_under_the_glow_still_resolve_to_gentle_dot() {
+ let panel = WindowRow { number: 9002, ..row(OWN_PID, "Gentle Dot", 3, (400.0, 300.0, 420.0, 640.0)) };
+ let dot = WindowRow { number: 9003, ..row(OWN_PID, "Gentle Dot", 3, (490.0, 390.0, 72.0, 72.0)) };
+ for rows in [[cursor(), glow(), panel.clone(), safari()], [cursor(), glow(), dot, safari()]] {
+ assert_eq!(owner_row_at(&rows, AT, Some(GLOW_WINDOW)).map(|r| r.pid), Some(OWN_PID));
+ }
+ }
+
+ #[test]
+ fn only_the_overlay_number_is_skipped() {
+ // Another window of the same owner and level, or the overlay before its number is known,
+ // still owns the point (and Gentle Dot's windows are refused).
+ let other = WindowRow { number: 9004, ..glow() };
+ assert_eq!(owner_row_at(&[other, safari()], AT, Some(GLOW_WINDOW)).map(|r| r.pid), Some(OWN_PID));
+ assert_eq!(owner_row_at(&[glow(), safari()], AT, None).map(|r| r.pid), Some(OWN_PID));
+ }
+
+ #[test]
+ fn a_hit_test_that_lands_on_the_overlay_is_redone_in_the_app_underneath() {
+ // The Accessibility hit test may land on the overlay; the risk check needs the element
+ // of the app that owns the window under the point.
+ assert_eq!(hit_test_app(OWN_PID, OWN_PID, Some(200)), Some(200));
+ assert_eq!(hit_test_app(OWN_PID, 200, Some(200)), None);
+ // Over the panel or the Dot (or nothing known), the system-wide answer stands.
+ assert_eq!(hit_test_app(OWN_PID, OWN_PID, Some(OWN_PID)), None);
+ assert_eq!(hit_test_app(OWN_PID, OWN_PID, None), None);
}
}
diff --git a/docs/design.md b/docs/design.md
index 316dd2f..036865f 100644
--- a/docs/design.md
+++ b/docs/design.md
@@ -171,6 +171,11 @@ The assistant can see the screen and operate apps through a built-in MCP server
- S24.7 Wiring: on every connection the desktop panel calls `computer_endpoint` and, when it returns `{url, token}`, sends `computer_register {url, token}` (the protocol accepts only `http://127.0.0.1:<port>/mcp` and a header-safe key). The daemon keeps the endpoint in memory, never in `connectors.json`, and renders `computer: {url, headers: {Authorization: "Bearer <key>"}, exposure: "direct"}` into `mcp.json`, then restarts the engine when idle. The guard's policy lists it under `builtin`, so its tools run without in-engine approval cards. It is removed on `computer_unregister` or when the window that registered it disconnects. The key is never logged or sent to a window: `connectors` lists only a built-in "Computer" entry (`builtin: true`) that the connector messages cannot change or remove, no saved or imported connector can take the name `computer`, and the import scan never offers the helper.
- S24.8 Speed: `batch {steps: [{tool, ...args}], intent?}` runs 1 to 20 steps of `click`, `move`, `drag`, `scroll`, `type`, `key`, `open_app`, or `wait` in one call and returns one screenshot. Each step takes the single tool's path (session and epoch, blocklist, risky confirmation, N1/N2 typing rules, coordinates mapped with the screenshot taken before the batch) and counts as one action for the rate limit; a full window delays the step instead of refusing it. Steps are 100 ms apart (1.5 s after `open_app`) so each step's guards read the screen the previous one left. The batch's `intent` applies to every step on top of the step's own. The first refused, declined, or failed step stops the batch; the error says which steps ran and comes with a fresh screenshot while the session lasts (none after Stop or panic). Malformed batches (empty, over 20, a nested `batch`, `screenshot`, `list_apps`, unknown tools, invalid arguments) run nothing. Actions settle 150 ms before their screenshot (`open_app` 1.5 s); screenshots are JPEG quality 65; debug builds optimize `image`, `zune-*`, and this crate (opt-level 1, because the generic encoder is compiled here).
- S24.9 Yolo mode: in memory only, off at launch. `computer_set_yolo {enabled}` and the tray item "Yolo mode" switch it; turning it on asks a native confirmation ("Turn on yolo mode? Gentle Dot will send, pay, delete, and submit without asking you first, for up to 1 hour. Stop and ⌥⇧Esc still work.", Turn On / Cancel). While on, only the risky-action confirmation is skipped; the grant, blocklist, rate limit, Stop, and panic are unchanged, and Stop or panic end the session, not yolo mode. It turns itself off after 1 hour. `computer://state` and `computer_status` add `yolo` and `yoloEndsAt?`; every switch reports the resulting state, so a declined confirmation flips the UI switch back. Connectors → Computer shows a "Yolo mode" switch, the banner shows "Yolo", and the rose carries a yellow mark (`dot-yolo`).
+- S28 The pink glow where the agent acts:
+ - What it shows: a pulse on a click, a trail along a drag (drawn on the display where the drag starts), short markers for moves and scrolls, and an outline around the focused element after `type` and `key`, when that element reports a frame (`AXPosition`, `AXSize`, read right after the keys); without one, nothing shows. The glow uses the accent `#f095c8`. It fades about 1.5 s after the last action (1.2 s lit, then a 300 ms fade) and clears at once when the session ends (Stop, panic, timeout). With "Reduce motion" it stays still: no pulse and no trail, only the drag's end.
+ - Where the marks come from: once an action's input is posted, `perform` hands a `Mark` (global points and a kind) to the helper's glow sink. Each batch step that runs gets its own mark. A refused, declined, stopped, or failed action or step reports nothing. `open_app`, `wait`, `screenshot`, and `list_apps` report nothing either. The sink only queues the mark on a channel, so no check, order, timing, or result changes; a rig test runs the same calls with and without the sink and compares them.
+ - The overlay: a Tauri window `glow` (`index.html?surface=glow`, title `Gentle Dot Glow`). It is transparent, undecorated, not focusable, out of the taskbar, and ignores cursor events. Its NSWindow sits at level 102, above menus, with collection behavior `canJoinAllSpaces | stationary | ignoresCycle | fullScreenAuxiliary`. It is ordered in with `orderFrontRegardless` when a session starts, so it never becomes key or activates the app, and ordered out when the session ends. A thread of its own receives the marks and drops any that arrive after the session ended. That thread looks up the display (`CGDisplayBounds`), moves the overlay over the whole display, and emits `computer://glow` to that window only. The UI also ignores marks that come after a `computer://state` without a session.
+ - Guards and screenshots: the overlay is a Gentle Dot window covering the display, so pointer-owner resolution skips it by its window number (`kCGWindowNumber`) and nothing else. The panel and the Dot still resolve to Gentle Dot and are refused. When the system-wide Accessibility hit test lands on the overlay while another app owns the window under the point, the risk check hit-tests that app's own element instead (`AXUIElementCreateApplication`). Screenshots leave the overlay out with every other window of our process (ScreenCaptureKit `excludingWindows`). The overlay is on screen for the whole session, so it is in the capture's window list before it draws anything.
Why enforcement lives in the app: everything inside the engine, and the daemon's WebSocket, is within the agent's reach (it has a shell, and with the daemon's access key it could answer its own approval cards, L49). The session grant, panic stop, blocklist, rate limit, and risky-action confirmations therefore run in the helper, behind native dialogs the agent cannot answer, and the in-engine guard adds nothing for these tools. macOS also attributes Accessibility and Screen Recording to the app that holds them, Gentle Dot.app. The key in `mcp.json` only reaches the helper, which still asks the user before anything happens.
@@ -260,6 +265,7 @@ The desktop app bundles the built UI (`packages/ui/dist/app`) as its frontend, s
|---|---|---|---|
| `dot` | 72 × 72 pt (the rose in a 66 pt black disc) | `index.html?surface=dot` | title `Gentle Dot`; transparent outside the disc, no decorations, always on top, not resizable, skip taskbar, visible on all workspaces, shadow off; placed in logical points |
| `panel` | 420 × 640 | `index.html?surface=panel` (Linux: `&effects=none`) | title `Gentle Dot Panel`; transparent with macOS vibrancy (`HudWindow`), no decorations, always on top, hidden at start, skip taskbar. With `effects=none` the UI sets `data-effects="none"` on `<html>` and paints the panel opaque (`--bg`, no backdrop blur). |
+| `glow` (macOS) | the display where the agent acts | `index.html?surface=glow` | title `Gentle Dot Glow`; the computer-control glow overlay (S28, §4): transparent, no decorations, not focusable, ignores cursor events, above menus, on every Space, shown only during a control session. Its own capability allows only `core:event:allow-listen` and `allow-unlisten`; it never connects to the daemon. |
When the panel opens, it is placed next to the Dot on the side with more room, and it stays inside the monitor.
@@ -284,6 +290,7 @@ The Dot drags with `getCurrentWindow().startDragging()` (permission `core:window
| `dot://new-conversation` | — | the tray item "New conversation" was chosen; with the conversations list on, the panel starts a new conversation and opens; in the single chat (default) the UI ignores it |
| `dot://panel-shown` | — | the panel became visible; the UI focuses the composer |
| `dot://rose` | `{ "hidden": boolean }` | the rose was hidden or shown (panel header or tray) |
+| `computer://glow` (to `glow` only) | `{ "kind": "click" \| "move" \| "scroll" \| "drag" \| "key", "x", "y", "toX"?, "toY"?, "width"?, "height"? }` in points from the overlay's top-left corner | the agent just acted there (S28); `drag` adds the end point, `key` the focused element's size |
### Menu bar and shortcut
diff --git a/packages/ui/src/App.tsx b/packages/ui/src/App.tsx
index fface04..038e950 100644
--- a/packages/ui/src/App.tsx
+++ b/packages/ui/src/App.tsx
@@ -3,6 +3,7 @@ import { type ConnectionInfo, tokenFromLocation } from "./client.ts";
import { ChatSurface } from "./components/ChatSurface.tsx";
import type { ComputerControls } from "./components/Computer.tsx";
import { DotSurface } from "./components/DotSurface.tsx";
+import { GlowSurface } from "./components/GlowSurface.tsx";
import { computerPermissions, requestComputerPermission, setComputerYolo, stopComputer } from "./computer.ts";
import {
connectionInfo,
@@ -19,9 +20,9 @@ import { usePanelWindow } from "./usePanelWindow.ts";
type Surface = "dot" | "panel" | "web";
-function currentSurface(): Surface {
+function currentSurface(): Surface | "glow" {
const surface = new URLSearchParams(window.location.search).get("surface");
- return surface === "dot" || surface === "panel" ? surface : "web";
+ return surface === "dot" || surface === "panel" || surface === "glow" ? surface : "web";
}
/**
@@ -50,8 +51,13 @@ function isFullscreenShortcut(event: KeyboardEvent): boolean {
);
}
+/** The desktop's glow overlay (S28) is its own surface: it never connects to the assistant. */
export function App() {
const surface = currentSurface();
+ return surface === "glow" ? <GlowSurface /> : <Assistant surface={surface} />;
+}
+
+function Assistant({ surface }: { surface: Surface }) {
const [info, setInfo] = useState<ConnectionInfo | undefined>(undefined);
const [missingToken, setMissingToken] = useState(false);
const [focusKey, setFocusKey] = useState(0);
diff --git a/packages/ui/src/components/GlowSurface.tsx b/packages/ui/src/components/GlowSurface.tsx
new file mode 100644
index 0000000..42ec08b
--- /dev/null
+++ b/packages/ui/src/components/GlowSurface.tsx
@@ -0,0 +1,113 @@
+import { type CSSProperties, useEffect, useRef, useState } from "react";
+import { type ComputerGlow, onComputerGlow, onComputerState } from "../computer.ts";
+
+/** The glow stays lit this long after the last action, then fades for `FADE_MS` (about 1.5 s in all). */
+const HOLD_MS = 1200;
+const FADE_MS = 300;
+/** Older marks drop off so a long run of actions never piles up. */
+const MAX_MARKS = 6;
+
+interface Lit {
+ id: number;
+ glow: ComputerGlow;
+}
+
+function prefersReducedMotion(): boolean {
+ return window.matchMedia?.("(prefers-reduced-motion: reduce)").matches ?? false;
+}
+
+const px = (value: number) => `${value}px`;
+
+function Mark({ glow, still }: { glow: ComputerGlow; still: boolean }) {
+ if (glow.kind === "key") {
+ const frame: CSSProperties = {
+ left: px(glow.x),
+ top: px(glow.y),
+ width: px(glow.width),
+ height: px(glow.height),
+ };
+ return <span className="glow-mark glow-key" style={frame} />;
+ }
+ if (glow.kind === "drag") {
+ const end = <span className="glow-mark glow-drag" style={{ left: px(glow.toX), top: px(glow.toY) }} />;
+ if (still) return end;
+ const length = Math.hypot(glow.toX - glow.x, glow.toY - glow.y);
+ const angle = Math.atan2(glow.toY - glow.y, glow.toX - glow.x);
+ const trail: CSSProperties = {
+ left: px(glow.x),
+ top: px(glow.y),
+ width: px(length),
+ transform: `rotate(${angle}rad)`,
+ };
+ return (
+ <>
+ <span className="glow-trail" style={trail} />
+ {end}
+ </>
+ );
+ }
+ return <span className={`glow-mark glow-${glow.kind}`} style={{ left: px(glow.x), top: px(glow.y) }} />;
+}
+
+/**
+ * The click-through overlay where the agent acts (S28): a pink pulse on a click, a trail along a
+ * drag, short markers for moves and scrolls, and an outline around the focused element for keys.
+ * It shows only while the agent acts, fades out about 1.5 s after the last action, and clears at
+ * once when the session ends. With reduced motion the glow is static: no pulse and no trail.
+ */
+export function GlowSurface() {
+ const [marks, setMarks] = useState<Lit[]>([]);
+ const [fading, setFading] = useState(false);
+ const [still] = useState(prefersReducedMotion);
+ const timers = useRef<ReturnType<typeof setTimeout>[]>([]);
+ const ended = useRef(false);
+ const next = useRef(0);
+
+ useEffect(() => {
+ document.documentElement.dataset.surface = "glow";
+ let live = true;
+ const stopTimers = () => {
+ for (const timer of timers.current) clearTimeout(timer);
+ timers.current = [];
+ };
+ const clear = () => {
+ stopTimers();
+ setFading(false);
+ setMarks([]);
+ };
+ const show = (glow: ComputerGlow) => {
+ // A mark that arrives after Stop, panic, or the timeout stays dark.
+ if (!live || ended.current) return;
+ stopTimers();
+ setFading(false);
+ next.current += 1;
+ const lit = { id: next.current, glow };
+ setMarks((current) => [...current, lit].slice(-MAX_MARKS));
+ timers.current = [
+ setTimeout(() => setFading(true), HOLD_MS),
+ setTimeout(() => setMarks([]), HOLD_MS + FADE_MS),
+ ];
+ };
+ const subscriptions = [
+ onComputerGlow(show),
+ onComputerState((state) => {
+ ended.current = state.session === undefined;
+ if (live && ended.current) clear();
+ }),
+ ];
+ return () => {
+ live = false;
+ stopTimers();
+ for (const s of subscriptions) void s.then((unlisten) => unlisten());
+ };
+ }, []);
+
+ const className = `glow-layer${fading ? " glow-fading" : ""}${still ? " glow-static" : ""}`;
+ return (
+ <div className={className} aria-hidden="true">
+ {marks.map(({ id, glow }) => (
+ <Mark key={id} glow={glow} still={still} />
+ ))}
+ </div>
+ );
+}
diff --git a/packages/ui/src/computer.ts b/packages/ui/src/computer.ts
index 4bb649b..d7fdace 100644
--- a/packages/ui/src/computer.ts
+++ b/packages/ui/src/computer.ts
@@ -81,12 +81,62 @@ export async function computerStatus(): Promise<ComputerState> {
return stateOf(await invoke<ComputerStateEvent>("computer_status"));
}
+/** The event API, imported once: the glow overlay subscribes to two events at the same time. */
+let eventApi: Promise<typeof import("@tauri-apps/api/event")> | undefined;
+function events() {
+ eventApi ??= import("@tauri-apps/api/event");
+ return eventApi;
+}
+
export async function onComputerState(handler: (state: ComputerState) => void) {
if (!isTauri()) return () => {};
- const { listen } = await import("@tauri-apps/api/event");
+ const { listen } = await events();
return listen<ComputerStateEvent>("computer://state", (event) => handler(stateOf(event.payload)));
}
+/**
+ * Where the agent just acted (S28), in points from the top-left corner of the glow overlay. A drag
+ * runs from `x, y` to `toX, toY`; a keyboard action outlines the focused element (`width × height`).
+ */
+export type ComputerGlow =
+ | { kind: "click" | "move" | "scroll"; x: number; y: number }
+ | { kind: "drag"; x: number; y: number; toX: number; toY: number }
+ | { kind: "key"; x: number; y: number; width: number; height: number };
+
+/** The overlay window's `computer://glow` events; anything it cannot place is dropped. */
+export async function onComputerGlow(handler: (glow: ComputerGlow) => void) {
+ if (!isTauri()) return () => {};
+ const { listen } = await events();
+ return listen<unknown>("computer://glow", (event) => {
+ const glow = glowOf(event.payload);
+ if (glow) handler(glow);
+ });
+}
+
+function glowOf(payload: unknown): ComputerGlow | undefined {
+ if (typeof payload !== "object" || payload === null) return undefined;
+ const event = payload as Record<string, unknown>;
+ const finite = (value: unknown): value is number => typeof value === "number" && Number.isFinite(value);
+ const { kind, x, y } = event;
+ if (!finite(x) || !finite(y)) return undefined;
+ switch (kind) {
+ case "click":
+ case "move":
+ case "scroll":
+ return { kind, x, y };
+ case "drag":
+ return finite(event.toX) && finite(event.toY)
+ ? { kind, x, y, toX: event.toX, toY: event.toY }
+ : undefined;
+ case "key":
+ return finite(event.width) && finite(event.height) && event.width > 0 && event.height > 0
+ ? { kind, x, y, width: event.width, height: event.height }
+ : undefined;
+ default:
+ return undefined;
+ }
+}
+
function stateOf(state: ComputerStateEvent | null | undefined): ComputerState {
const session = state?.active === true ? { ...timeOf(state.endsAt) } : undefined;
const yolo = state?.yolo === true ? { ...timeOf(state.yoloEndsAt) } : undefined;
diff --git a/packages/ui/src/styles.css b/packages/ui/src/styles.css
index b4a5e38..0c08ce2 100644
--- a/packages/ui/src/styles.css
+++ b/packages/ui/src/styles.css
@@ -1348,3 +1348,123 @@ button.link:hover:not(:disabled) {
transition: none !important;
}
}
+
+/* The glow where the agent acts (S28), in the desktop's click-through overlay: the accent pink, like
+ the neon rose. A pulse on a click, a trail along a drag, short markers for moves and scrolls, and an
+ outline around the focused element for keys. Shown while the agent acts, faded about 1.5 s later. */
+html[data-surface="glow"] body {
+ background: transparent;
+ overflow: hidden;
+ pointer-events: none;
+}
+
+.glow-layer {
+ position: fixed;
+ inset: 0;
+ pointer-events: none;
+ transition: opacity 300ms var(--ease);
+}
+
+.glow-layer.glow-fading {
+ opacity: 0;
+}
+
+.glow-mark {
+ position: absolute;
+ width: 28px;
+ height: 28px;
+ margin: -14px 0 0 -14px;
+ border-radius: 50%;
+ background: radial-gradient(
+ circle,
+ rgba(255, 255, 255, 0.9) 0 12%,
+ var(--accent) 28%,
+ rgba(240, 149, 200, 0) 70%
+ );
+ box-shadow:
+ 0 0 12px 4px rgba(240, 149, 200, 0.55),
+ 0 0 32px 10px rgba(240, 149, 200, 0.25);
+}
+
+.glow-click::after {
+ content: "";
+ position: absolute;
+ inset: -6px;
+ border: 2px solid var(--accent);
+ border-radius: 50%;
+ animation: glow-pulse 700ms var(--ease) both;
+}
+
+.glow-move,
+.glow-scroll {
+ width: 16px;
+ height: 16px;
+ margin: -8px 0 0 -8px;
+ box-shadow: 0 0 10px 3px rgba(240, 149, 200, 0.45);
+ animation: glow-blip 600ms var(--ease) both;
+}
+
+.glow-trail {
+ position: absolute;
+ height: 6px;
+ margin-top: -3px;
+ border-radius: var(--radius-pill);
+ transform-origin: 0 50%;
+ background: linear-gradient(90deg, rgba(240, 149, 200, 0), var(--accent));
+ box-shadow: 0 0 14px 3px rgba(240, 149, 200, 0.45);
+ animation: glow-trail 450ms var(--ease) both;
+}
+
+.glow-key {
+ margin: -4px 0 0 -4px;
+ padding: 4px;
+ box-sizing: content-box;
+ border: 2px solid var(--accent);
+ border-radius: var(--radius-sm);
+ background: var(--accent-soft);
+ box-shadow:
+ 0 0 14px 2px rgba(240, 149, 200, 0.5),
+ inset 0 0 10px rgba(240, 149, 200, 0.3);
+ animation: glow-blip 600ms var(--ease) both;
+}
+
+/* Reduce motion: the glow stays still (no pulse, no trail; the overlay draws only the drag's end). */
+.glow-static .glow-mark,
+.glow-static .glow-mark::after {
+ animation: none;
+}
+
+.glow-static .glow-click::after {
+ display: none;
+}
+
+@keyframes glow-pulse {
+ from {
+ opacity: 1;
+ transform: scale(0.4);
+ }
+ to {
+ opacity: 0;
+ transform: scale(1.8);
+ }
+}
+
+@keyframes glow-blip {
+ from {
+ opacity: 0;
+ transform: scale(0.6);
+ }
+ to {
+ opacity: 1;
+ transform: scale(1);
+ }
+}
+
+@keyframes glow-trail {
+ from {
+ clip-path: inset(0 100% 0 0);
+ }
+ to {
+ clip-path: inset(0 0 0 0);
+ }
+}
diff --git a/packages/ui/test/glow.test.tsx b/packages/ui/test/glow.test.tsx
new file mode 100644
index 0000000..8622022
--- /dev/null
+++ b/packages/ui/test/glow.test.tsx
@@ -0,0 +1,158 @@
+import { act, render, waitFor } from "@testing-library/react";
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+
+const h = vi.hoisted(() => ({
+ invoke: vi.fn(),
+ listeners: new Map<string, (event: { payload: unknown }) => void>(),
+ reducedMotion: false,
+}));
+
+vi.mock("@tauri-apps/api/core", () => ({
+ isTauri: () => true,
+ invoke: (command: string, args?: unknown) => h.invoke(command, args),
+}));
+
+vi.mock("@tauri-apps/api/event", () => ({
+ listen: async (name: string, handler: (event: { payload: unknown }) => void) => {
+ h.listeners.set(name, handler);
+ return () => h.listeners.delete(name);
+ },
+}));
+
+const { GlowSurface } = await import("../src/components/GlowSurface.tsx");
+const { App } = await import("../src/App.tsx");
+
+function emit(name: string, payload: unknown) {
+ act(() => h.listeners.get(name)?.({ payload }));
+}
+
+const glow = (payload: unknown) => emit("computer://glow", payload);
+const session = (active: boolean) => emit("computer://state", { active, yolo: false });
+
+async function renderGlow() {
+ const view = render(<GlowSurface />);
+ await waitFor(() => expect(h.listeners.has("computer://glow")).toBe(true));
+ await waitFor(() => expect(h.listeners.has("computer://state")).toBe(true));
+ return view.container;
+}
+
+const marks = (root: HTMLElement) => root.querySelectorAll(".glow-mark");
+
+beforeEach(() => {
+ h.invoke.mockReset();
+ h.invoke.mockImplementation(() => new Promise(() => {}));
+ h.listeners.clear();
+ h.reducedMotion = false;
+ window.matchMedia = ((query: string) => ({
+ matches: query.includes("reduced-motion") && h.reducedMotion,
+ media: query,
+ addEventListener: () => {},
+ removeEventListener: () => {},
+ })) as unknown as typeof window.matchMedia;
+});
+
+afterEach(() => {
+ vi.useRealTimers();
+ window.history.replaceState(null, "", "/");
+});
+
+describe("GlowSurface (S28)", () => {
+ it("shows nothing while the agent is idle", async () => {
+ const root = await renderGlow();
+ expect(marks(root)).toHaveLength(0);
+ session(true);
+ expect(marks(root)).toHaveLength(0);
+ });
+
+ it("pulses where the agent clicked, in the overlay's points", async () => {
+ const root = await renderGlow();
+ glow({ kind: "click", x: 60, y: 120 });
+ const click = root.querySelector(".glow-mark.glow-click") as HTMLElement;
+ expect(click).not.toBeNull();
+ expect(click.style.left).toBe("60px");
+ expect(click.style.top).toBe("120px");
+ });
+
+ it("draws a trail along a drag and short markers for move and scroll", async () => {
+ const root = await renderGlow();
+ glow({ kind: "drag", x: 100, y: 100, toX: 400, toY: 500 });
+ const trail = root.querySelector(".glow-trail") as HTMLElement;
+ expect(trail).not.toBeNull();
+ expect(trail.style.width).toBe("500px");
+ expect(root.querySelector(".glow-mark.glow-drag")).not.toBeNull();
+ glow({ kind: "move", x: 1, y: 2 });
+ glow({ kind: "scroll", x: 3, y: 4 });
+ expect(root.querySelector(".glow-mark.glow-move")).not.toBeNull();
+ expect(root.querySelector(".glow-mark.glow-scroll")).not.toBeNull();
+ });
+
+ it("outlines the focused element for keyboard actions", async () => {
+ const root = await renderGlow();
+ glow({ kind: "key", x: 160, y: 100, width: 200, height: 30 });
+ const field = root.querySelector(".glow-mark.glow-key") as HTMLElement;
+ expect(field.style.width).toBe("200px");
+ expect(field.style.height).toBe("30px");
+ });
+
+ it("ignores events it cannot place", async () => {
+ const root = await renderGlow();
+ for (const payload of [
+ null,
+ { kind: "click" },
+ { kind: "explode", x: 1, y: 1 },
+ { kind: "key", x: 1, y: 1 },
+ { kind: "drag", x: 1, y: 1 },
+ { kind: "click", x: Number.NaN, y: 1 },
+ ]) {
+ glow(payload);
+ }
+ expect(marks(root)).toHaveLength(0);
+ });
+
+ it("fades out about 1.5 s after the last action", async () => {
+ const root = await renderGlow();
+ vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout"] });
+ glow({ kind: "click", x: 10, y: 10 });
+ act(() => vi.advanceTimersByTime(1000));
+ // Another action keeps it lit.
+ glow({ kind: "move", x: 20, y: 20 });
+ act(() => vi.advanceTimersByTime(1200));
+ expect(marks(root)).toHaveLength(2);
+ expect(root.querySelector(".glow-layer")).toHaveClass("glow-fading");
+ act(() => vi.advanceTimersByTime(300));
+ expect(marks(root)).toHaveLength(0);
+ });
+
+ it("disappears at once when the session ends and stays dark until the next one", async () => {
+ const root = await renderGlow();
+ session(true);
+ glow({ kind: "click", x: 10, y: 10 });
+ session(false);
+ expect(marks(root)).toHaveLength(0);
+ // A mark that arrives after Stop, panic, or the timeout is dropped.
+ glow({ kind: "click", x: 10, y: 10 });
+ expect(marks(root)).toHaveLength(0);
+ session(true);
+ glow({ kind: "click", x: 10, y: 10 });
+ expect(marks(root)).toHaveLength(1);
+ });
+
+ it("stays still with reduced motion: a static glow, no pulse or trail", async () => {
+ h.reducedMotion = true;
+ const root = await renderGlow();
+ glow({ kind: "drag", x: 100, y: 100, toX: 400, toY: 500 });
+ expect(root.querySelector(".glow-layer")).toHaveClass("glow-static");
+ expect(root.querySelector(".glow-trail")).toBeNull();
+ const end = root.querySelector(".glow-mark.glow-drag") as HTMLElement;
+ expect([end.style.left, end.style.top]).toEqual(["400px", "500px"]);
+ });
+
+ it("is the app's glow window, which never connects to the assistant", async () => {
+ window.history.replaceState(null, "", "/?surface=glow");
+ const view = render(<App />);
+ await waitFor(() => expect(h.listeners.has("computer://glow")).toBe(true));
+ expect(view.container.querySelector(".glow-layer")).not.toBeNull();
+ expect(document.documentElement.dataset.surface).toBe("glow");
+ expect(h.invoke).not.toHaveBeenCalledWith("connection_info", undefined);
+ });
+});