Skip to content

A personal assistant that lives on your desktop, remembers what matters, and can use your Mac for you.

TypeScript

feat(computer): batch actions, a faster helper, and yolo mode

Alan Buscaglia952830ea19a1
diff --git a/apps/desktop/src-tauri/Cargo.toml b/apps/desktop/src-tauri/Cargo.toml
index f04b6e0..19b43e8 100644
--- a/apps/desktop/src-tauri/Cargo.toml
+++ b/apps/desktop/src-tauri/Cargo.toml
@@ -37,3 +37,19 @@ objc2-core-graphics = { version = "0.3", default-features = false, features = ["
objc2-app-kit = { version = "0.3", default-features = false, features = ["std", "libc", "NSRunningApplication", "NSWorkspace"] }
objc2-application-services = { version = "0.3", default-features = false, features = ["std", "libc", "AXError", "AXUIElement", "HIServices"] }
objc2-screen-capture-kit = { version = "0.3", default-features = false, features = ["std", "block2", "libc", "objc2-core-foundation", "objc2-core-graphics", "SCScreenshotManager", "SCShareableContent", "SCStream"] }
+
+# Debug builds encode screenshots fast too (S24.8): `image` holds the JPEG encoder and `zune-*`
+# decode JPEGs (tests). The encoder is generic, so most of its code is compiled in this crate,
+# which gets opt-level 1 as well (measured on a 1280 × 800 encode: 200 ms unoptimized, 126 ms
+# with the dependencies optimized, 40 ms with this crate at 1). Other dependencies are untouched.
+[profile.dev.package.image]
+opt-level = 3
+
+[profile.dev.package.zune-jpeg]
+opt-level = 3
+
+[profile.dev.package.zune-core]
+opt-level = 3
+
+[profile.dev.package.gentle-dot]
+opt-level = 1
diff --git a/apps/desktop/src-tauri/build.rs b/apps/desktop/src-tauri/build.rs
index aba1be9..00e88c5 100644
--- a/apps/desktop/src-tauri/build.rs
+++ b/apps/desktop/src-tauri/build.rs
@@ -12,6 +12,7 @@ const COMMANDS: &[&str] = &[
"computer_request_permission",
"computer_stop",
"computer_status",
+ "computer_set_yolo",
];
fn find_in_path(program: &str, path: &str) -> Option<PathBuf> {
diff --git a/apps/desktop/src-tauri/capabilities/default.json b/apps/desktop/src-tauri/capabilities/default.json
index 4f141f3..d9f5b9e 100644
--- a/apps/desktop/src-tauri/capabilities/default.json
+++ b/apps/desktop/src-tauri/capabilities/default.json
@@ -1,7 +1,7 @@
{
"$schema": "../gen/schemas/desktop-schema.json",
"identifier": "default",
- "description": "What the Dot and panel UI may call: event listening, Dot dragging, opening web pages (sign-in), the four shell commands, and the computer-control commands.",
+ "description": "What the Dot and panel UI may call: event listening, Dot dragging, opening web pages (sign-in), the four shell commands, and the computer-control commands (yolo mode included: its native confirmation decides).",
"windows": ["dot", "panel"],
"permissions": [
"core:event:allow-listen",
@@ -16,6 +16,7 @@
"allow-computer-permissions",
"allow-computer-request-permission",
"allow-computer-stop",
- "allow-computer-status"
+ "allow-computer-status",
+ "allow-computer-set-yolo"
]
}
diff --git a/apps/desktop/src-tauri/permissions/autogenerated/computer_set_yolo.toml b/apps/desktop/src-tauri/permissions/autogenerated/computer_set_yolo.toml
new file mode 100644
index 0000000..6e3497e
--- /dev/null
+++ b/apps/desktop/src-tauri/permissions/autogenerated/computer_set_yolo.toml
@@ -0,0 +1,11 @@
+# Automatically generated - DO NOT EDIT!
+
+[[permission]]
+identifier = "allow-computer-set-yolo"
+description = "Enables the computer_set_yolo command without any pre-configured scope."
+commands.allow = ["computer_set_yolo"]
+
+[[permission]]
+identifier = "deny-computer-set-yolo"
+description = "Denies the computer_set_yolo command without any pre-configured scope."
+commands.deny = ["computer_set_yolo"]
diff --git a/apps/desktop/src-tauri/src/computer/app.rs b/apps/desktop/src-tauri/src/computer/app.rs
index c7855cc..107bee7 100644
--- a/apps/desktop/src-tauri/src/computer/app.rs
+++ b/apps/desktop/src-tauri/src/computer/app.rs
@@ -1,6 +1,7 @@
//! The app side of computer control (L60): the `computer_*` commands, the `computer://state`
//! event, the panic shortcut, the timeout watcher, and the native dialogs. Off macOS the
-//! commands answer "unavailable" and nothing starts.
+//! commands answer "unavailable" and nothing starts. Yolo mode (S24.9) is switched only from
+//! here (`computer_set_yolo` and the tray item), never from the MCP helper or the daemon.
use super::control::Control;
use super::server::Endpoint;
@@ -15,6 +16,12 @@ pub const PANIC_SHORTCUT: &str = "Alt+Shift+Escape";
pub const GRANT_TEXT: &str = "Allow Gentle Dot to see and control this Mac for 30 minutes?";
pub const GRANT_BUTTONS: (&str, &str) = ("Allow", "Deny");
pub const CONFIRM_BUTTONS: (&str, &str) = ("Allow", "Cancel");
+pub const YOLO_TEXT: &str = "Turn on yolo mode? Gentle Dot will send, pay, delete, and submit without asking you first, \
+for up to 1 hour. Stop and ⌥⇧Esc still work.";
+pub const YOLO_BUTTONS: (&str, &str) = ("Turn On", "Cancel");
+
+/// The tray's "Yolo mode" item, kept in step with the state.
+pub struct YoloMenuItem(pub tauri::menu::CheckMenuItem<tauri::Wry>);
pub struct Computer {
control: Arc<Control>,
@@ -30,6 +37,9 @@ pub fn start(app: &AppHandle) -> Computer {
if let Err(error) = events.emit(STATE_EVENT, event) {
eprintln!("gentle-dot: cannot report the computer-control state: {error}");
}
+ if let Some(item) = events.try_state::<YoloMenuItem>() {
+ let _ = item.0.set_checked(event.yolo);
+ }
}));
#[cfg(target_os = "macos")]
{
@@ -110,6 +120,35 @@ pub fn computer_stop(app: AppHandle) {
stop(&app, Reason::Stopped);
}
+/// Turns yolo mode on after the native confirmation, or off. Blocks on the dialog, so it runs
+/// off the main thread; the tray item calls it from a thread of its own.
+pub fn set_yolo(app: &AppHandle, enabled: bool) -> Result<StateEvent, String> {
+ let computer = app.try_state::<Computer>().ok_or("Computer control is not ready.")?;
+ #[cfg(target_os = "macos")]
+ return Ok(computer.control.set_yolo(enabled, SystemClock.now_ms(), &NativeDialogs { app: app.clone() }));
+ #[cfg(not(target_os = "macos"))]
+ {
+ let _ = (computer, enabled);
+ Err("Computer control is only available on macOS.".into())
+ }
+}
+
+/// The tray item: the opposite of the current state.
+pub fn toggle_yolo(app: &AppHandle) {
+ let app = app.clone();
+ std::thread::spawn(move || {
+ let on = app.try_state::<Computer>().is_some_and(|c| c.control.yolo(SystemClock.now_ms()));
+ if let Err(error) = set_yolo(&app, !on) {
+ eprintln!("gentle-dot: cannot switch yolo mode: {error}");
+ }
+ });
+}
+
+#[tauri::command]
+pub async fn computer_set_yolo(app: AppHandle, enabled: bool) -> Result<StateEvent, String> {
+ tauri::async_runtime::spawn_blocking(move || set_yolo(&app, enabled)).await.map_err(|e| e.to_string())?
+}
+
#[tauri::command]
pub fn computer_status(computer: State<'_, Computer>) -> StateEvent {
computer.control.status(SystemClock.now_ms())
@@ -144,6 +183,10 @@ impl super::Dialogs for NativeDialogs {
fn confirm(&self, message: &str) -> bool {
self.ask(message, CONFIRM_BUTTONS)
}
+
+ fn ask_yolo(&self) -> bool {
+ self.ask(YOLO_TEXT, YOLO_BUTTONS)
+ }
}
#[cfg(test)]
@@ -156,6 +199,16 @@ mod tests {
assert_eq!(GRANT_BUTTONS, ("Allow", "Deny"));
}
+ #[test]
+ fn the_yolo_dialog_says_what_it_skips_and_for_how_long() {
+ assert_eq!(
+ YOLO_TEXT,
+ "Turn on yolo mode? Gentle Dot will send, pay, delete, and submit without asking you first, \
+for up to 1 hour. Stop and ⌥⇧Esc still work."
+ );
+ assert_eq!(YOLO_BUTTONS, ("Turn On", "Cancel"));
+ }
+
#[test]
fn the_panic_shortcut_is_option_shift_escape() {
let shortcut: tauri_plugin_global_shortcut::Shortcut = PANIC_SHORTCUT.parse().unwrap();
diff --git a/apps/desktop/src-tauri/src/computer/control.rs b/apps/desktop/src-tauri/src/computer/control.rs
index d0ede33..baf3b54 100644
--- a/apps/desktop/src-tauri/src/computer/control.rs
+++ b/apps/desktop/src-tauri/src/computer/control.rs
@@ -1,7 +1,9 @@
//! The session shared by the MCP server and the app (Stop button, tray item, panic shortcut,
//! timeout watcher). Every transition is reported to `notify` (the `computer://state` event).
+//! Yolo mode (S24.9) is switched only here, from the app's command or tray item, never from MCP.
use super::session::{Reason, Session, StateEvent};
+use super::Dialogs;
use std::sync::Mutex;
type Notify = Box<dyn Fn(&StateEvent) + Send + Sync>;
@@ -52,4 +54,78 @@ impl Control {
let event = self.with(|s| s.expire(now_ms));
self.report(event);
}
+
+ pub fn yolo(&self, now_ms: u64) -> bool {
+ self.with(|s| s.yolo(now_ms))
+ }
+
+ /// Turns yolo mode on after the user confirms it natively, or off without asking. The
+ /// result is always reported, so a switch that flipped before the answer flips back.
+ pub fn set_yolo(&self, enabled: bool, now_ms: u64, dialogs: &dyn Dialogs) -> StateEvent {
+ let on = enabled && dialogs.ask_yolo();
+ let event = self.with(|s| s.set_yolo(on, now_ms));
+ self.report(Some(event.clone()));
+ event
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::super::fake::FakeDialogs;
+ use super::super::session::{StateEvent, YOLO_MS};
+ use super::*;
+ use std::sync::atomic::Ordering;
+ use std::sync::Arc;
+
+ const NOW: u64 = 1_700_000_000_000;
+
+ fn control() -> (Control, Arc<Mutex<Vec<StateEvent>>>) {
+ let states = Arc::new(Mutex::new(Vec::new()));
+ let sink = states.clone();
+ (Control::new(move |event| sink.lock().unwrap().push(event.clone())), states)
+ }
+
+ #[test]
+ fn turning_yolo_on_asks_the_user_and_reports_it() {
+ let (control, states) = control();
+ let dialogs = FakeDialogs::new(true, true);
+ control.set_yolo(true, NOW, &dialogs);
+ assert_eq!(dialogs.yolo_asks.load(Ordering::SeqCst), 1);
+ assert!(control.yolo(NOW));
+ let on = StateEvent { yolo: true, yolo_ends_at: Some(NOW + YOLO_MS), ..StateEvent::default() };
+ assert_eq!(*states.lock().unwrap(), vec![on.clone()]);
+ assert_eq!(control.status(NOW), on);
+ }
+
+ #[test]
+ fn a_declined_yolo_stays_off_and_the_switch_is_told_so() {
+ let (control, states) = control();
+ let dialogs = FakeDialogs::new(true, true);
+ *dialogs.yolo.lock().unwrap() = false;
+ control.set_yolo(true, NOW, &dialogs);
+ assert!(!control.yolo(NOW));
+ assert_eq!(*states.lock().unwrap(), vec![StateEvent::default()]);
+ }
+
+ #[test]
+ fn turning_yolo_off_never_asks() {
+ let (control, states) = control();
+ let dialogs = FakeDialogs::new(true, true);
+ control.set_yolo(true, NOW, &dialogs);
+ control.set_yolo(false, NOW + 1, &dialogs);
+ assert_eq!(dialogs.yolo_asks.load(Ordering::SeqCst), 1);
+ assert!(!control.yolo(NOW + 1));
+ assert_eq!(states.lock().unwrap().last(), Some(&StateEvent::default()));
+ }
+
+ #[test]
+ fn yolo_expiry_turns_it_off_and_reports_it() {
+ let (control, states) = control();
+ control.set_yolo(true, NOW, &FakeDialogs::new(true, true));
+ control.expire(NOW + YOLO_MS - 1);
+ assert_eq!(states.lock().unwrap().len(), 1);
+ control.expire(NOW + YOLO_MS);
+ assert!(!control.yolo(NOW + YOLO_MS));
+ assert_eq!(states.lock().unwrap().last(), Some(&StateEvent::default()));
+ }
}
diff --git a/apps/desktop/src-tauri/src/computer/encode.rs b/apps/desktop/src-tauri/src/computer/encode.rs
index 3e22061..d4c3537 100644
--- a/apps/desktop/src-tauri/src/computer/encode.rs
+++ b/apps/desktop/src-tauri/src/computer/encode.rs
@@ -1,6 +1,6 @@
//! Screenshot encoding: ScreenCaptureKit hands back BGRA rows; the model gets a JPEG.
-pub const JPEG_QUALITY: u8 = 80;
+pub const JPEG_QUALITY: u8 = 65;
/// Encodes a BGRA bitmap whose rows are `bytes_per_row` long (rows may be padded) as JPEG.
pub fn bgra_to_jpeg(
@@ -57,6 +57,63 @@ mod tests {
assert!(b > 240 && r < 20 && g < 20, "{r} {g} {b}");
}
+ /// A 1280 × 800 BGRA mock of an app window: light chrome, a sidebar, rows of 1 px
+ /// "text" strokes, and two colored buttons.
+ fn ui_like_bgra() -> Vec<u8> {
+ let (w, h) = (1280usize, 800usize);
+ let mut bgra = vec![0u8; w * h * 4];
+ for y in 0..h {
+ for x in 0..w {
+ let (r, g, b) = if y < 52 {
+ (236, 236, 236)
+ } else if x < 240 {
+ (246, 242, 248)
+ } else if (600..640).contains(&y) && ((900..1010).contains(&x) || (1030..1140).contains(&x)) {
+ if x < 1010 { (10, 132, 255) } else { (255, 59, 48) }
+ } else {
+ (255, 255, 255)
+ };
+ // Glyph-like strokes: 9 px tall lines of text every 24 px, broken into "words".
+ let line = (y + 12) % 24 < 9 && y > 60 && y < 580;
+ let glyph = (x * 7 + y * 3) % 11 < 4 && x % 64 < 52 && x > 20;
+ let (r, g, b) = if line && glyph { (30, 30, 30) } else { (r, g, b) };
+ let i = (y * w + x) * 4;
+ bgra[i..i + 4].copy_from_slice(&[b, g, r, 255]);
+ }
+ }
+ bgra
+ }
+
+ /// Mean absolute error per channel between the source and the decoded JPEG.
+ fn mean_error(bgra: &[u8], jpeg: &[u8]) -> f64 {
+ let decoded = decode(jpeg);
+ let total: u64 = decoded
+ .pixels()
+ .zip(bgra.chunks(4))
+ .map(|(p, s)| (0..3).map(|c| u64::from(p.0[c].abs_diff(s[2 - c]))).sum::<u64>())
+ .sum();
+ total as f64 / (decoded.pixels().len() * 3) as f64
+ }
+
+ #[test]
+ fn the_screenshot_quality_is_lighter_and_keeps_text_legible() {
+ // S24.8(c): quality 80 → JPEG_QUALITY. Sizes are printed for the record (`--nocapture`).
+ assert!((60..=70).contains(&JPEG_QUALITY), "{JPEG_QUALITY}");
+ let bgra = ui_like_bgra();
+ let before = bgra_to_jpeg(&bgra, 1280, 800, 1280 * 4, 80).unwrap();
+ let started = std::time::Instant::now();
+ let after = bgra_to_jpeg(&bgra, 1280, 800, 1280 * 4, JPEG_QUALITY).unwrap();
+ let took = started.elapsed().as_millis();
+ let (error_before, error_after) = (mean_error(&bgra, &before), mean_error(&bgra, &after));
+ eprintln!(
+ "ui-like 1280x800: quality 80 = {} bytes (error {error_before:.2}), quality {JPEG_QUALITY} = {} bytes (error {error_after:.2}), encoded in {took} ms",
+ before.len(),
+ after.len()
+ );
+ assert!(after.len() * 100 <= before.len() * 85, "{} vs {}", after.len(), before.len());
+ assert!(error_after < 6.0, "{error_after}");
+ }
+
#[test]
fn short_or_empty_bitmaps_are_rejected() {
assert!(bgra_to_jpeg(&[0; 10], 2, 2, 8, 80).is_err());
diff --git a/apps/desktop/src-tauri/src/computer/fake.rs b/apps/desktop/src-tauri/src/computer/fake.rs
index d3abc1e..59d9f67 100644
--- a/apps/desktop/src-tauri/src/computer/fake.rs
+++ b/apps/desktop/src-tauri/src/computer/fake.rs
@@ -114,6 +114,8 @@ pub struct FakeDialogs {
pub confirm: Mutex<bool>,
pub grants: AtomicUsize,
pub confirmations: Mutex<Vec<String>>,
+ pub yolo: Mutex<bool>,
+ pub yolo_asks: AtomicUsize,
/// Runs while a dialog is open (for example, to move the focus to Gentle Dot).
pub during: Mutex<Option<Hook>>,
}
@@ -125,6 +127,8 @@ impl FakeDialogs {
confirm: Mutex::new(confirm),
grants: AtomicUsize::new(0),
confirmations: Mutex::new(Vec::new()),
+ yolo: Mutex::new(true),
+ yolo_asks: AtomicUsize::new(0),
during: Mutex::new(None),
}
}
@@ -148,6 +152,11 @@ impl Dialogs for FakeDialogs {
self.run_hook();
*self.confirm.lock().unwrap()
}
+
+ fn ask_yolo(&self) -> bool {
+ self.yolo_asks.fetch_add(1, Ordering::SeqCst);
+ *self.yolo.lock().unwrap()
+ }
}
/// Starts at a fixed time; `sleep_ms` advances it and runs `on_sleep`.
diff --git a/apps/desktop/src-tauri/src/computer/mcp.rs b/apps/desktop/src-tauri/src/computer/mcp.rs
index ee90234..eba4469 100644
--- a/apps/desktop/src-tauri/src/computer/mcp.rs
+++ b/apps/desktop/src-tauri/src/computer/mcp.rs
@@ -2,6 +2,11 @@
//! `tools/list`, and `tools/call`, behind a per-launch bearer token. Each call passes the
//! guards in order: rate limit, permissions, session grant, blocklist, risky-action
//! confirmation; then it acts and returns a fresh screenshot (S24.2–S24.4).
+//!
+//! `batch` (S24.8) runs up to 20 actions in one call. Each step takes the single tool's path
+//! (`perform`) and counts as one action for the rate limit; a full window delays the step
+//! instead of refusing it. Coordinates refer to the screenshot taken before the batch, and one
+//! screenshot follows it. The first refused, declined, or failed step stops the batch.
use super::blocklist::{blocked_name, blocked_reason, AppIdentity};
use super::control::Control;
@@ -17,8 +22,13 @@ use std::sync::{Arc, Mutex, MutexGuard};
/// Supported MCP revisions, newest first; the newest answers an unknown request.
const PROTOCOL_VERSIONS: &[&str] = &["2025-06-18", "2025-03-26", "2024-11-05"];
/// Time for the screen to settle before the screenshot that follows an action.
-const SETTLE_MS: u64 = 300;
+const SETTLE_MS: u64 = 150;
const OPEN_APP_SETTLE_MS: u64 = 1500;
+const MAX_BATCH_STEPS: usize = 20;
+/// The tools a batch step may use: actions only (a batch returns its own screenshot).
+const BATCH_TOOLS: &[&str] = &["click", "move", "drag", "scroll", "type", "key", "open_app", "wait"];
+/// Between batch steps, so each step's guards read the focus and windows the previous step left.
+const STEP_PAUSE_MS: u64 = 100;
const MAX_WAIT_MS: u64 = 5000;
const WAIT_SLICE_MS: u64 = 100;
/// macOS posts at most 20 UTF-16 units of text per keyboard event.
@@ -33,8 +43,10 @@ const UNKNOWN_TARGET: &str = "Cannot tell which app is under that point, so the
Take a new screenshot and try again.";
const INSTRUCTIONS: &str = "Controls this Mac. Take a screenshot first; x and y are pixels of the latest \
-screenshot. Every action returns a fresh screenshot. The user must allow a session, risky actions ask \
-the user to confirm, and some apps (System Settings, password managers) are off limits.";
+screenshot. Every action returns a fresh screenshot. Use `batch` for a predictable sequence (click a field, \
+type, press tab, type, press return) so it runs in one call; end the batch whenever you must see the screen \
+before deciding the next step. The user must allow a session, risky actions ask the user to confirm, and \
+some apps (System Settings, password managers) are off limits.";
pub struct HttpRequest<'a> {
pub method: &'a str,
@@ -60,6 +72,8 @@ enum Tool {
OpenApp { name: String },
ListApps,
Wait { ms: u64 },
+ /// Steps with their tool names, checked one by one when they run.
+ Batch { steps: Vec<(&'static str, Call)> },
}
struct Call {
@@ -69,12 +83,23 @@ struct Call {
type Content = Vec<Value>;
-fn text(text: impl Into<String>) -> Value {
- json!({"type": "text", "text": text.into()})
+/// A tool error; its content is a message, or what a batch did before it stopped.
+struct Failed(Content);
+
+impl From<String> for Failed {
+ fn from(message: String) -> Self {
+ Failed(vec![text(message)])
+ }
+}
+
+impl From<&str> for Failed {
+ fn from(message: &str) -> Self {
+ Failed(vec![text(message)])
+ }
}
-fn tool_error(message: &str) -> Value {
- json!({"content": [text(message)], "isError": true})
+fn text(text: impl Into<String>) -> Value {
+ json!({"type": "text", "text": text.into()})
}
fn lock<T>(mutex: &Mutex<T>) -> MutexGuard<'_, T> {
@@ -132,7 +157,15 @@ fn tools() -> Value {
{"name": "list_apps", "description": "List the running apps.",
"inputSchema": schema(json!({}), &[])},
{"name": "wait", "description": "Wait, then take a screenshot.",
- "inputSchema": schema(json!({"ms": {"type": "integer", "minimum": 0, "maximum": MAX_WAIT_MS}}), &["ms"])}
+ "inputSchema": schema(json!({"ms": {"type": "integer", "minimum": 0, "maximum": MAX_WAIT_MS}}), &["ms"])},
+ {"name": "batch", "description": "Run 1 to 20 actions in order in one call, then return one screenshot. Use it for \
+predictable sequences, for example type, `tab`, type; or click a field, type, `return`. Coordinates are pixels of \
+the screenshot taken before the batch, so batch only steps whose targets you can already see; when a step's result \
+decides what comes next, end the batch there and look at the screenshot it returns. Each step is checked like a \
+single call; the batch stops at the first refused, declined, or failed step and says which steps ran.",
+ "inputSchema": schema(json!({"steps": {"type": "array", "minItems": 1, "maxItems": MAX_BATCH_STEPS,
+ "items": {"type": "object", "description": "A tool name plus that tool's arguments, for example {\"tool\": \"type\", \"text\": \"hello\"}.",
+ "properties": {"tool": {"type": "string", "enum": BATCH_TOOLS}}, "required": ["tool"]}}}), &["steps"])}
])
}
@@ -206,6 +239,7 @@ fn parse_call(name: &str, args: &Value) -> Option<Result<Call, String>> {
Tool::OpenApp { name }
}
"list_apps" => Tool::ListApps,
+ "batch" => Tool::Batch { steps: batch_steps(args)? },
"wait" => {
let ms = integer(args, "ms")?;
if !(0..=MAX_WAIT_MS as i64).contains(&ms) {
@@ -217,7 +251,7 @@ fn parse_call(name: &str, args: &Value) -> Option<Result<Call, String>> {
})
};
const NAMES: &[&str] =
- &["screenshot", "click", "move", "drag", "scroll", "type", "key", "open_app", "list_apps", "wait"];
+ &["screenshot", "click", "move", "drag", "scroll", "type", "key", "open_app", "list_apps", "wait", "batch"];
if !NAMES.contains(&name) {
return None;
}
@@ -225,6 +259,28 @@ fn parse_call(name: &str, args: &Value) -> Option<Result<Call, String>> {
Some(tool().map(|tool| Call { tool, intent }))
}
+/// The steps of a `batch`, each parsed like a single call. Any invalid step refuses the batch.
+fn batch_steps(args: &Map<String, Value>) -> Result<Vec<(&'static str, Call)>, String> {
+ let steps = args.get("steps").and_then(Value::as_array).ok_or("`steps` must be a list of steps")?;
+ if steps.is_empty() || steps.len() > MAX_BATCH_STEPS {
+ return Err(format!("`steps` must hold 1 to {MAX_BATCH_STEPS} steps"));
+ }
+ let step = |n: usize, step: &Value| -> Result<(&'static str, Call), String> {
+ let mut step = step.as_object().cloned().ok_or(format!("step {n} must be an object"))?;
+ let name = step.remove("tool").and_then(|t| t.as_str().map(str::to_string));
+ let name = name.ok_or(format!("step {n} needs a `tool`"))?;
+ let tool = BATCH_TOOLS.iter().copied().find(|t| *t == name).ok_or_else(|| {
+ format!("step {n}: `{name}` cannot be a batch step; use one of {}", BATCH_TOOLS.join(", "))
+ })?;
+ match parse_call(tool, &Value::Object(step)) {
+ Some(Ok(call)) => Ok((tool, call)),
+ Some(Err(error)) => Err(format!("step {n} ({tool}): {error}")),
+ None => Err(format!("step {n}: unknown tool `{tool}`")),
+ }
+ };
+ steps.iter().enumerate().map(|(i, s)| step(i + 1, s)).collect()
+}
+
/// Splits text into chunks of at most `TEXT_CHUNK_UNITS` UTF-16 units, never inside a character.
fn chunks(text: &str) -> Vec<String> {
let mut out = Vec::new();
@@ -333,16 +389,18 @@ impl Helper {
/// A `tools/call` that arrived while the session was at `epoch`.
fn call_tool_from(&self, epoch: u64, name: &str, args: &Value) -> Option<Value> {
- let outcome = parse_call(name, args)?.and_then(|call| self.run(epoch, call));
+ let outcome = parse_call(name, args)?.map_err(Failed::from).and_then(|call| self.run(epoch, call));
Some(match outcome {
Ok(content) => json!({"content": content}),
- Err(message) => tool_error(&message),
+ Err(Failed(content)) => json!({"content": content, "isError": true}),
})
}
- fn run(&self, epoch: u64, call: Call) -> Result<Content, String> {
- if !lock(&self.limiter).try_acquire(self.clock.now_ms()) {
- return Err(format!("Too many actions: at most {MAX_ACTIONS_PER_SECOND} per second. Wait a moment."));
+ fn run(&self, epoch: u64, call: Call) -> Result<Content, Failed> {
+ // A batch counts each of its steps instead (`pace`).
+ let counted = !matches!(call.tool, Tool::Batch { .. });
+ if counted && !lock(&self.limiter).try_acquire(self.clock.now_ms()) {
+ return Err(format!("Too many actions: at most {MAX_ACTIONS_PER_SECOND} per second. Wait a moment.").into());
}
self.check_permissions(&call.tool)?;
let mut latest = lock(&self.actions);
@@ -362,6 +420,9 @@ impl Helper {
}
fn check_permissions(&self, tool: &Tool) -> Result<(), String> {
+ if let Tool::Batch { steps } = tool {
+ return steps.iter().try_for_each(|(_, step)| self.check_permissions(&step.tool));
+ }
let permissions = self.desktop.permissions();
let sees = !matches!(tool, Tool::ListApps);
let acts = matches!(
@@ -437,6 +498,10 @@ impl Helper {
let Some(reason) = risk else {
return Ok(());
};
+ // Yolo mode (S24.9), switched on by the user in the app, skips only this confirmation.
+ if self.control.yolo(self.clock.now_ms()) {
+ return Ok(());
+ }
let message = format!("{action} in {target}.\n\nThis needs your OK because of {reason}.");
if self.with_dialog(|| self.dialogs.confirm(&message)) {
Ok(())
@@ -471,13 +536,95 @@ impl Helper {
Ok(content)
}
- fn act(&self, epoch: u64, latest: &mut Option<ScreenshotGeometry>, call: Call) -> Result<Content, String> {
- let intent = call.intent.as_deref();
+ fn act(&self, epoch: u64, latest: &mut Option<ScreenshotGeometry>, call: Call) -> Result<Content, Failed> {
+ match call.tool {
+ Tool::Screenshot => Ok(self.screenshot(latest)?),
+ Tool::ListApps => {
+ let apps = self.desktop.list_apps()?;
+ Ok(vec![text(serde_json::to_string(&apps).map_err(|e| e.to_string())?)])
+ }
+ Tool::Batch { steps } => self.batch(epoch, latest, steps, call.intent.as_deref()),
+ tool => {
+ let (done, settle_ms) = self.perform(epoch, latest, tool, call.intent.as_deref())?;
+ Ok(self.after(latest, done, settle_ms)?)
+ }
+ }
+ }
+
+ /// Runs a batch's steps in order, then one screenshot. On the first failure it stops and
+ /// reports the steps that ran, with a fresh screenshot while the session lasts.
+ fn batch(
+ &self,
+ epoch: u64,
+ latest: &mut Option<ScreenshotGeometry>,
+ steps: Vec<(&'static str, Call)>,
+ intent: Option<&str>,
+ ) -> Result<Content, Failed> {
+ let total = steps.len();
+ let mut done: Vec<String> = Vec::new();
+ let mut settle_ms = 0;
+ for (i, (name, step)) in steps.into_iter().enumerate() {
+ if i > 0 {
+ // An app that is opening needs its full wait before anything targets it.
+ self.clock.sleep_ms(if settle_ms == OPEN_APP_SETTLE_MS { settle_ms } else { STEP_PAUSE_MS });
+ }
+ // The batch's intent applies to every step, on top of the step's own.
+ let intents: Vec<&str> = [intent, step.intent.as_deref()].into_iter().flatten().collect();
+ let intents = (!intents.is_empty()).then(|| intents.join("; "));
+ let outcome =
+ self.pace(epoch).and_then(|()| self.perform(epoch, latest, step.tool, intents.as_deref()));
+ match outcome {
+ Ok((message, step_settle_ms)) => {
+ done.push(format!("{}. {message}", i + 1));
+ settle_ms = step_settle_ms;
+ }
+ Err(error) => {
+ let ran = done.len();
+ let at = i + 1;
+ let mut message = format!("Stopped at step {at} of {total} ({name}): {error}\nRan {ran} of {total} steps");
+ message.push_str(if ran == 0 { "." } else { ":\n" });
+ message.push_str(&done.join("\n"));
+ if self.still_running(epoch).is_err() {
+ return Err(Failed(vec![text(message)]));
+ }
+ return Err(Failed(self.after(latest, message, settle_ms.max(SETTLE_MS))?));
+ }
+ }
+ }
+ let summary = format!("Ran {total} of {total} steps:\n{}", done.join("\n"));
+ Ok(self.after(latest, summary, settle_ms)?)
+ }
+
+ /// Counts one batch step against the rate limit, waiting for room instead of refusing.
+ fn pace(&self, epoch: u64) -> Result<(), String> {
+ loop {
+ self.still_running(epoch)?;
+ let wait_ms = {
+ let mut limiter = lock(&self.limiter);
+ let now = self.clock.now_ms();
+ if limiter.try_acquire(now) {
+ return Ok(());
+ }
+ limiter.wait_ms(now)
+ };
+ self.clock.sleep_ms(wait_ms.clamp(1, WAIT_SLICE_MS));
+ }
+ }
+
+ /// One action through every guard, then posted: what was done and how long the screen
+ /// needs to settle before a screenshot. A single call and a batch step both come here.
+ fn perform(
+ &self,
+ epoch: u64,
+ latest: &Option<ScreenshotGeometry>,
+ tool: Tool,
+ intent: Option<&str>,
+ ) -> Result<(String, u64), String> {
let map = |latest: &Option<ScreenshotGeometry>, (x, y): (f64, f64)| -> Result<Point, String> {
latest.ok_or(NEEDS_SCREENSHOT)?.to_point(x, y).map_err(|e| e.to_string())
};
- match call.tool {
- Tool::Screenshot => self.screenshot(latest),
+ match tool {
+ Tool::Screenshot | Tool::ListApps | Tool::Batch { .. } => Err("Not an action.".into()),
Tool::Click { x, y, button, count } => {
let at = map(latest, (x, y))?;
let target = self.pointer_target(at)?;
@@ -490,7 +637,7 @@ impl Helper {
};
self.confirm_if_risky(assess(Act::Click(&chain), intent), &action, &target)?;
self.post(epoch, InputEvent::Click { at, button, count })?;
- self.after(latest, format!("{action} in {target}: done."), SETTLE_MS)
+ Ok((format!("{action} in {target}: done."), SETTLE_MS))
}
Tool::Move { x, y } => {
let at = map(latest, (x, y))?;
@@ -498,7 +645,7 @@ impl Helper {
let action = format!("Move the pointer to ({x}, {y})");
self.confirm_if_risky(assess(Act::Other, intent), &action, &target)?;
self.post(epoch, InputEvent::Move(at))?;
- self.after(latest, format!("{action}: done."), SETTLE_MS)
+ Ok((format!("{action}: done."), SETTLE_MS))
}
Tool::Drag { from, to } => {
let (start, end) = (map(latest, from)?, map(latest, to)?);
@@ -509,7 +656,7 @@ impl Helper {
let (from_chain, to_chain) = (self.desktop.element_at(start), self.desktop.element_at(end));
self.confirm_if_risky(assess(Act::Drag(&from_chain, &to_chain), intent), &action, &target)?;
self.post(epoch, InputEvent::Drag { from: start, to: end })?;
- self.after(latest, format!("{action}: done."), SETTLE_MS)
+ Ok((format!("{action}: done."), SETTLE_MS))
}
Tool::Scroll { x, y, dx, dy } => {
let at = map(latest, (x, y))?;
@@ -517,7 +664,7 @@ impl Helper {
let action = format!("Scroll by ({dx}, {dy}) at ({x}, {y})");
self.confirm_if_risky(assess(Act::Other, intent), &action, &target)?;
self.post(epoch, InputEvent::Scroll { at, dx, dy })?;
- self.after(latest, format!("{action}: done."), SETTLE_MS)
+ Ok((format!("{action}: done."), SETTLE_MS))
}
Tool::Type { text: typed } => {
let target = self.keyboard_target()?;
@@ -534,7 +681,7 @@ impl Helper {
self.post(epoch, InputEvent::Text(chunk))?;
}
}
- self.after(latest, format!("{action} in {target}: done."), SETTLE_MS)
+ Ok((format!("{action} in {target}: done."), SETTLE_MS))
}
Tool::Key { combo } => {
let target = self.keyboard_target()?;
@@ -549,7 +696,7 @@ impl Helper {
let action = format!("Press {}", combo.key);
self.confirm_if_risky(assess(act, intent), &action, &target)?;
self.post(epoch, InputEvent::Key(combo))?;
- self.after(latest, format!("{action} in {target}: done."), SETTLE_MS)
+ Ok((format!("{action} in {target}: done."), SETTLE_MS))
}
Tool::OpenApp { name } => {
if let Some(label) = blocked_name(&name) {
@@ -559,11 +706,7 @@ impl Helper {
self.confirm_if_risky(assess(Act::Other, intent), &action, "macOS")?;
self.still_running(epoch)?;
self.desktop.open_app(&name)?;
- self.after(latest, format!("{action}: done."), OPEN_APP_SETTLE_MS)
- }
- Tool::ListApps => {
- let apps = self.desktop.list_apps()?;
- Ok(vec![text(serde_json::to_string(&apps).map_err(|e| e.to_string())?)])
+ Ok((format!("{action}: done."), OPEN_APP_SETTLE_MS))
}
Tool::Wait { ms } => {
let mut left = ms;
@@ -573,7 +716,7 @@ impl Helper {
left -= slice;
self.still_running(epoch)?;
}
- self.after(latest, format!("Waited {ms} ms."), 0)
+ Ok((format!("Waited {ms} ms."), 0))
}
}
}
@@ -726,7 +869,7 @@ mod tests {
let names: Vec<&str> = tools.iter().map(|t| t["name"].as_str().unwrap()).collect();
assert_eq!(
names,
- ["screenshot", "click", "move", "drag", "scroll", "type", "key", "open_app", "list_apps", "wait"]
+ ["screenshot", "click", "move", "drag", "scroll", "type", "key", "open_app", "list_apps", "wait", "batch"]
);
for tool in tools {
let schema = &tool["inputSchema"];
@@ -771,7 +914,12 @@ mod tests {
assert_eq!(rig.dialogs.grants.load(Ordering::SeqCst), 1);
assert_eq!(
*rig.states.lock().unwrap(),
- vec![StateEvent { active: true, ends_at: Some(NOW + SESSION_MS), reason: Some(Reason::Granted) }]
+ vec![StateEvent {
+ active: true,
+ ends_at: Some(NOW + SESSION_MS),
+ reason: Some(Reason::Granted),
+ ..StateEvent::default()
+ }]
);
}
@@ -1215,6 +1363,285 @@ mod tests {
assert!(chunks.iter().all(|c| c.encode_utf16().count() <= 20), "{chunks:?}");
}
+
+ // --- Speed (S24.8) ---
+
+ #[test]
+ fn an_action_settles_150_ms_and_open_app_still_1500() {
+ let rig = rig();
+ rig.ready();
+ let before = rig.clock.now_ms();
+ rig.call("click", json!({"x": 10, "y": 10}));
+ assert_eq!(rig.clock.now_ms() - before, 150);
+ let before = rig.clock.now_ms();
+ rig.call("open_app", json!({"name": "Mail"}));
+ assert_eq!(rig.clock.now_ms() - before, 1500);
+ }
+
+ #[test]
+ fn a_short_message_with_an_emoji_is_two_events_and_one_settle() {
+ // L66: an 18 s `type` of this text. The helper's own share is two chunks (four
+ // keyboard events, 15 ms apart on the Mac) plus one settle; no confirmation is asked.
+ let rig = rig();
+ rig.ready();
+ let message = "¡Hola, Panda! Te quiero mucho ❤️";
+ let before = rig.clock.now_ms();
+ assert!(!is_error(&rig.call("type", json!({"text": message}))));
+ assert_eq!(rig.clock.now_ms() - before, 150);
+ assert!(rig.dialogs.confirmations.lock().unwrap().is_empty());
+ let events = rig.desktop.events();
+ assert_eq!(events.len(), 2, "{events:?}");
+ let typed: String = events
+ .into_iter()
+ .map(|e| match e {
+ InputEvent::Text(t) => t,
+ other => panic!("{other:?}"),
+ })
+ .collect();
+ assert_eq!(typed, message);
+ }
+
+ fn batch(steps: Value) -> Value {
+ json!({"steps": steps})
+ }
+
+ fn image_count(result: &Value) -> usize {
+ result["content"].as_array().map_or(0, |c| c.iter().filter(|b| b["type"] == "image").count())
+ }
+
+ #[test]
+ fn tools_list_offers_batch_for_predictable_sequences() {
+ let rig = rig();
+ let body = rig.rpc("tools/list", json!({}));
+ let tools = body["result"]["tools"].as_array().unwrap();
+ let batch = tools.iter().find(|t| t["name"] == "batch").unwrap();
+ let steps = &batch["inputSchema"]["properties"]["steps"];
+ assert_eq!((steps["minItems"].as_u64(), steps["maxItems"].as_u64()), (Some(1), Some(20)));
+ assert_eq!(
+ steps["items"]["properties"]["tool"]["enum"],
+ json!(["click", "move", "drag", "scroll", "type", "key", "open_app", "wait"])
+ );
+ assert!(batch["description"].as_str().unwrap().contains("screenshot"));
+ let instructions = rig.rpc("initialize", json!({}))["result"]["instructions"].as_str().unwrap().to_string();
+ assert!(instructions.contains("batch"), "{instructions}");
+ }
+
+ #[test]
+ fn a_batch_runs_its_steps_in_order_and_returns_one_screenshot() {
+ let rig = rig();
+ rig.ready();
+ let result = rig.call(
+ "batch",
+ batch(json!([
+ {"tool": "click", "x": 100, "y": 50},
+ {"tool": "type", "text": "Ana"},
+ {"tool": "key", "combo": "tab"},
+ {"tool": "type", "text": "Hi there"},
+ {"tool": "scroll", "x": 5, "y": 6, "dx": 0, "dy": 120},
+ ])),
+ );
+ assert!(!is_error(&result), "{result}");
+ let p = |x, y| Point { x, y };
+ assert_eq!(
+ rig.desktop.events(),
+ vec![
+ // Mapped with the screenshot taken before the batch (scale 0.5).
+ InputEvent::Click { at: p(200.0, 100.0), button: MouseButton::Left, count: 1 },
+ InputEvent::Text("Ana".into()),
+ InputEvent::Key(parse_combo("tab").unwrap()),
+ InputEvent::Text("Hi there".into()),
+ InputEvent::Scroll { at: p(10.0, 12.0), dx: 0, dy: 120 },
+ ]
+ );
+ assert_eq!(rig.desktop.captures.load(Ordering::SeqCst), 2);
+ assert_eq!(image_count(&result), 1);
+ let summary = first_text(&result);
+ assert!(summary.contains("5 of 5") && summary.contains("Type 3 characters"), "{summary}");
+ }
+
+ #[test]
+ fn a_risky_step_asks_once_and_a_decline_stops_the_batch_after_the_earlier_steps() {
+ let rig = rig_with(true, false);
+ rig.ready();
+ *rig.desktop.elements.lock().unwrap() = send_button();
+ let result = rig.call(
+ "batch",
+ batch(json!([
+ {"tool": "type", "text": "hi"},
+ {"tool": "click", "x": 10, "y": 10},
+ {"tool": "type", "text": "never"},
+ ])),
+ );
+ assert!(is_error(&result));
+ let message = first_text(&result);
+ assert!(message.contains("declined") && message.contains("step 2"), "{message}");
+ assert!(message.contains("1 of 3"), "{message}");
+ assert_eq!(rig.dialogs.confirmations.lock().unwrap().len(), 1);
+ assert_eq!(rig.desktop.events(), vec![InputEvent::Text("hi".into())]);
+ // A fresh screenshot shows where the batch stopped.
+ assert_eq!(image_count(&result), 1);
+ }
+
+ #[test]
+ fn a_blocklisted_step_stops_the_batch() {
+ let rig = rig();
+ rig.ready();
+ *rig.desktop.owner_at.lock().unwrap() = Box::new(|at: Point| {
+ if at.x > 100.0 {
+ Some(app(300, "com.apple.systempreferences", "System Settings"))
+ } else {
+ Some(safari())
+ }
+ });
+ let result = rig.call(
+ "batch",
+ batch(json!([
+ {"tool": "click", "x": 10, "y": 10},
+ {"tool": "click", "x": 200, "y": 10},
+ {"tool": "click", "x": 20, "y": 20},
+ ])),
+ );
+ assert!(is_error(&result));
+ assert!(first_text(&result).contains("System Settings"), "{result}");
+ assert_eq!(rig.desktop.events().len(), 1);
+ }
+
+ #[test]
+ fn a_panic_during_a_batch_drops_the_remaining_steps() {
+ let rig = rig();
+ rig.ready();
+ let control = rig.control.clone();
+ *rig.clock.on_sleep.lock().unwrap() = Some(Box::new(move || control.end(Reason::Panic)));
+ let result = rig.call(
+ "batch",
+ batch(json!([
+ {"tool": "click", "x": 10, "y": 10},
+ {"tool": "click", "x": 20, "y": 20},
+ {"tool": "click", "x": 30, "y": 30},
+ ])),
+ );
+ assert!(is_error(&result));
+ assert!(first_text(&result).contains("stopped"), "{result}");
+ assert_eq!(rig.desktop.events().len(), 1);
+ // No screenshot once the session ended.
+ assert_eq!(rig.desktop.captures.load(Ordering::SeqCst), 1);
+ assert_eq!(image_count(&result), 0);
+ }
+
+ #[test]
+ fn malformed_batches_are_refused_before_anything_runs() {
+ let rig = rig();
+ let wait = json!({"tool": "wait", "ms": 0});
+ let too_many = Value::Array(vec![wait.clone(); 21]);
+ for args in [
+ json!({}),
+ batch(json!("click")),
+ batch(json!([])),
+ batch(too_many),
+ batch(json!(["click"])),
+ batch(json!([{"tool": "batch", "steps": [wait]}])),
+ batch(json!([{"tool": "format_disk"}])),
+ batch(json!([{"tool": "screenshot"}])),
+ batch(json!([{"x": 1, "y": 1}])),
+ batch(json!([{"tool": "wait", "ms": 0}, {"tool": "click", "x": 1}])),
+ ] {
+ let result = rig.call("batch", args.clone());
+ assert!(is_error(&result), "{args}");
+ }
+ assert_eq!(rig.dialogs.grants.load(Ordering::SeqCst), 0);
+ assert!(rig.desktop.events().is_empty());
+ }
+
+ #[test]
+ fn an_intent_in_a_batch_only_adds_checks() {
+ // A harmless step intent cannot hide a risky batch intent or a risky target.
+ let rig = rig_with(true, false);
+ rig.ready();
+ let result = rig.call(
+ "batch",
+ json!({"intent": "buy the ticket", "steps": [{"tool": "scroll", "x": 1, "y": 1, "dx": 0, "dy": 10, "intent": "just scrolling"}]}),
+ );
+ assert!(first_text(&result).contains("declined"), "{result}");
+ *rig.desktop.elements.lock().unwrap() = send_button();
+ let result = rig.call(
+ "batch",
+ json!({"intent": "look around", "steps": [{"tool": "click", "x": 1, "y": 1, "intent": "harmless"}]}),
+ );
+ assert!(first_text(&result).contains("declined"), "{result}");
+ assert_eq!(rig.dialogs.confirmations.lock().unwrap().len(), 2);
+ assert!(rig.desktop.events().is_empty());
+ }
+
+ #[test]
+ fn a_batch_waits_for_the_rate_limit_instead_of_failing() {
+ // Each step counts as one action; a full window delays the batch, it does not refuse it.
+ let rig = rig();
+ rig.ready();
+ for _ in 0..9 {
+ assert!(!is_error(&rig.call("list_apps", json!({}))));
+ }
+ let steps: Vec<Value> = (0..3).map(|i| json!({"tool": "move", "x": i, "y": i})).collect();
+ let result = rig.call("batch", batch(Value::Array(steps)));
+ assert!(!is_error(&result), "{result}");
+ assert_eq!(rig.desktop.events().len(), 3);
+ assert!(rig.clock.now_ms() >= NOW + 1000);
+ }
+
+ #[test]
+ fn a_batch_asks_for_the_session_once() {
+ let rig = rig_with(false, true);
+ let result = rig.call("batch", batch(json!([{"tool": "key", "combo": "tab"}, {"tool": "key", "combo": "tab"}])));
+ assert!(first_text(&result).contains("not allowed"), "{result}");
+ assert_eq!(rig.dialogs.grants.load(Ordering::SeqCst), 1);
+ assert!(rig.desktop.events().is_empty());
+ }
+
+ // --- Yolo mode (S24.9) ---
+
+ fn yolo_on(rig: &Rig) {
+ rig.control.set_yolo(true, rig.clock.now_ms(), rig.dialogs.as_ref());
+ }
+
+ #[test]
+ fn yolo_skips_the_risky_confirmation() {
+ let rig = rig_with(true, false);
+ rig.ready();
+ yolo_on(&rig);
+ *rig.desktop.elements.lock().unwrap() = send_button();
+ let result = rig.call("click", json!({"x": 10, "y": 10}));
+ assert!(!is_error(&result), "{result}");
+ assert!(rig.dialogs.confirmations.lock().unwrap().is_empty());
+ assert_eq!(rig.desktop.events().len(), 1);
+ }
+
+ #[test]
+ fn yolo_keeps_the_grant_the_blocklist_and_the_panic() {
+ let rig = rig_with(false, false);
+ yolo_on(&rig);
+ assert!(first_text(&rig.call("screenshot", json!({}))).contains("not allowed"));
+ *rig.dialogs.allow.lock().unwrap() = true;
+ rig.ready();
+ *rig.desktop.owner_at.lock().unwrap() =
+ Box::new(|_| Some(app(300, "com.apple.systempreferences", "System Settings")));
+ assert!(first_text(&rig.call("click", json!({"x": 10, "y": 10}))).contains("System Settings"));
+ let queued_at = rig.control.epoch();
+ rig.control.end(Reason::Panic);
+ let result = rig.helper.call_tool_from(queued_at, "key", &json!({"combo": "tab"})).unwrap();
+ assert!(first_text(&result).contains("stopped"), "{result}");
+ assert!(rig.desktop.events().is_empty());
+ }
+
+ #[test]
+ fn risky_actions_ask_again_once_yolo_expires() {
+ let rig = rig_with(true, false);
+ rig.ready();
+ yolo_on(&rig);
+ rig.clock.advance(super::super::session::YOLO_MS);
+ rig.ready();
+ *rig.desktop.elements.lock().unwrap() = send_button();
+ assert!(first_text(&rig.call("click", json!({"x": 10, "y": 10}))).contains("declined"));
+ }
+
#[test]
fn list_apps_returns_names_and_bundle_ids() {
let rig = rig();
diff --git a/apps/desktop/src-tauri/src/computer/mod.rs b/apps/desktop/src-tauri/src/computer/mod.rs
index 13bda4c..390c7c7 100644
--- a/apps/desktop/src-tauri/src/computer/mod.rs
+++ b/apps/desktop/src-tauri/src/computer/mod.rs
@@ -113,6 +113,8 @@ pub trait Dialogs: Send + Sync {
fn ask_grant(&self) -> bool;
/// A per-action confirmation showing the action and its target.
fn confirm(&self, message: &str) -> bool;
+ /// "Turn on yolo mode? ..." (Turn On / Cancel), S24.9.
+ fn ask_yolo(&self) -> bool;
}
pub trait Clock: Send + Sync {
diff --git a/apps/desktop/src-tauri/src/computer/rate.rs b/apps/desktop/src-tauri/src/computer/rate.rs
index d43dd9a..3167b8d 100644
--- a/apps/desktop/src-tauri/src/computer/rate.rs
+++ b/apps/desktop/src-tauri/src/computer/rate.rs
@@ -15,12 +15,25 @@ impl RateLimiter {
RateLimiter { max, window_ms, accepted: VecDeque::new() }
}
- /// Counts a call at `now_ms` and returns true, or returns false (not counted) when
- /// `max` calls were already accepted within the last `window_ms`.
- pub fn try_acquire(&mut self, now_ms: u64) -> bool {
+ fn slide(&mut self, now_ms: u64) {
while self.accepted.front().is_some_and(|&t| t + self.window_ms <= now_ms) {
self.accepted.pop_front();
}
+ }
+
+ /// Milliseconds until a call at `now_ms` would be accepted; 0 when it would be now.
+ pub fn wait_ms(&mut self, now_ms: u64) -> u64 {
+ self.slide(now_ms);
+ if self.accepted.len() < self.max {
+ return 0;
+ }
+ self.accepted.front().map_or(0, |&t| t + self.window_ms - now_ms)
+ }
+
+ /// Counts a call at `now_ms` and returns true, or returns false (not counted) when
+ /// `max` calls were already accepted within the last `window_ms`.
+ pub fn try_acquire(&mut self, now_ms: u64) -> bool {
+ self.slide(now_ms);
if self.accepted.len() >= self.max {
return false;
}
@@ -53,6 +66,16 @@ mod tests {
assert!(limiter.try_acquire(1500));
}
+ #[test]
+ fn it_tells_how_long_until_the_next_call_fits() {
+ let mut limiter = RateLimiter::new(2, 1000);
+ assert_eq!(limiter.wait_ms(0), 0);
+ assert!(limiter.try_acquire(0));
+ assert!(limiter.try_acquire(300));
+ assert_eq!(limiter.wait_ms(400), 600);
+ assert_eq!(limiter.wait_ms(1000), 0);
+ }
+
#[test]
fn rejected_calls_do_not_count() {
let mut limiter = RateLimiter::new(1, 1000);
diff --git a/apps/desktop/src-tauri/src/computer/session.rs b/apps/desktop/src-tauri/src/computer/session.rs
index be3952b..f03fd7b 100644
--- a/apps/desktop/src-tauri/src/computer/session.rs
+++ b/apps/desktop/src-tauri/src/computer/session.rs
@@ -1,10 +1,14 @@
//! The computer-control session (S24.3): idle until the user allows it, then granted until
//! `ends_at`, ended by the Stop button, the tray item, the panic shortcut, or the timeout.
//! Every end bumps `epoch`, so actions queued under an earlier epoch are dropped.
+//!
+//! Yolo mode (S24.9) lives here too, in memory only: while it lasts, risky actions skip their
+//! confirmation. It has its own switch and its own hour; Stop and panic end the session, not it.
use serde::Serialize;
pub const SESSION_MS: u64 = 30 * 60 * 1000;
+pub const YOLO_MS: u64 = 60 * 60 * 1000;
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
#[serde(rename_all = "lowercase")]
@@ -15,9 +19,9 @@ pub enum Reason {
Timeout,
}
-/// The `computer://state` event and the `computer_status` reply (L60). `ends_at` is in
-/// Unix epoch milliseconds.
-#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
+/// The `computer://state` event and the `computer_status` reply (L60), plus `yolo` and
+/// `yoloEndsAt` (S24.9). Times are Unix epoch milliseconds.
+#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct StateEvent {
pub active: bool,
@@ -25,12 +29,16 @@ pub struct StateEvent {
pub ends_at: Option<u64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub reason: Option<Reason>,
+ pub yolo: bool,
+ #[serde(skip_serializing_if = "Option::is_none")]
+ pub yolo_ends_at: Option<u64>,
}
#[derive(Debug, Default)]
pub struct Session {
ends_at: Option<u64>,
epoch: u64,
+ yolo_ends_at: Option<u64>,
}
impl Session {
@@ -42,31 +50,64 @@ impl Session {
self.epoch
}
+ /// Whether yolo mode is on at `now_ms`.
+ pub fn yolo(&self, now_ms: u64) -> bool {
+ self.yolo_ends_at.is_some_and(|end| now_ms < end)
+ }
+
pub fn status(&self, now_ms: u64) -> StateEvent {
let active = self.is_active(now_ms);
- StateEvent { active, ends_at: self.ends_at.filter(|_| active), reason: None }
+ let yolo = self.yolo(now_ms);
+ StateEvent {
+ active,
+ ends_at: self.ends_at.filter(|_| active),
+ reason: None,
+ yolo,
+ yolo_ends_at: self.yolo_ends_at.filter(|_| yolo),
+ }
+ }
+
+ /// The status as an event for `reason`, sent at a transition (the clock has not moved).
+ fn event(&self, reason: Option<Reason>) -> StateEvent {
+ StateEvent {
+ active: self.ends_at.is_some(),
+ ends_at: self.ends_at,
+ reason,
+ yolo: self.yolo_ends_at.is_some(),
+ yolo_ends_at: self.yolo_ends_at,
+ }
}
/// Starts (or restarts) a session of `SESSION_MS`.
pub fn grant(&mut self, now_ms: u64) -> StateEvent {
- let ends_at = now_ms + SESSION_MS;
- self.ends_at = Some(ends_at);
- StateEvent { active: true, ends_at: Some(ends_at), reason: Some(Reason::Granted) }
+ self.ends_at = Some(now_ms + SESSION_MS);
+ self.event(Some(Reason::Granted))
}
/// Ends the session for `reason`. `None` when there was none to end.
pub fn end(&mut self, reason: Reason) -> Option<StateEvent> {
self.ends_at.take()?;
self.epoch += 1;
- Some(StateEvent { active: false, ends_at: None, reason: Some(reason) })
+ Some(self.event(Some(reason)))
}
- /// Ends the session with `Timeout` once `now_ms` reaches its end.
+ /// Turns yolo mode on for `YOLO_MS`, or off.
+ pub fn set_yolo(&mut self, on: bool, now_ms: u64) -> StateEvent {
+ self.yolo_ends_at = on.then_some(now_ms + YOLO_MS);
+ self.status(now_ms)
+ }
+
+ /// Ends the session with `Timeout`, and yolo mode, once `now_ms` reaches their end; one
+ /// event for both.
pub fn expire(&mut self, now_ms: u64) -> Option<StateEvent> {
+ let yolo_over = self.yolo_ends_at.is_some_and(|end| now_ms >= end);
+ if yolo_over {
+ self.yolo_ends_at = None;
+ }
if self.ends_at.is_some_and(|end| now_ms >= end) {
self.end(Reason::Timeout)
} else {
- None
+ yolo_over.then(|| self.event(None))
}
}
}
@@ -81,14 +122,14 @@ mod tests {
fn a_new_session_is_idle() {
let session = Session::default();
assert!(!session.is_active(NOW));
- assert_eq!(session.status(NOW), StateEvent { active: false, ends_at: None, reason: None });
+ assert_eq!(session.status(NOW), StateEvent { active: false, ends_at: None, reason: None, ..StateEvent::default() });
}
#[test]
fn granting_starts_thirty_minutes() {
let mut session = Session::default();
let event = session.grant(NOW);
- assert_eq!(event, StateEvent { active: true, ends_at: Some(NOW + SESSION_MS), reason: Some(Reason::Granted) });
+ assert_eq!(event, StateEvent { active: true, ends_at: Some(NOW + SESSION_MS), reason: Some(Reason::Granted), ..StateEvent::default() });
assert!(session.is_active(NOW + SESSION_MS - 1));
assert_eq!(session.status(NOW + 5).ends_at, Some(NOW + SESSION_MS));
}
@@ -99,7 +140,7 @@ mod tests {
let mut session = Session::default();
session.grant(NOW);
let epoch = session.epoch();
- assert_eq!(session.end(reason), Some(StateEvent { active: false, ends_at: None, reason: Some(reason) }));
+ assert_eq!(session.end(reason), Some(StateEvent { active: false, ends_at: None, reason: Some(reason), ..StateEvent::default() }));
assert!(!session.is_active(NOW));
assert_eq!(session.epoch(), epoch + 1);
}
@@ -120,7 +161,7 @@ mod tests {
assert!(!session.is_active(NOW + SESSION_MS));
assert_eq!(
session.expire(NOW + SESSION_MS),
- Some(StateEvent { active: false, ends_at: None, reason: Some(Reason::Timeout) })
+ Some(StateEvent { active: false, ends_at: None, reason: Some(Reason::Timeout), ..StateEvent::default() })
);
assert_eq!(session.epoch(), 1);
assert_eq!(session.expire(NOW + SESSION_MS + 1), None);
@@ -130,9 +171,50 @@ mod tests {
fn the_event_serializes_like_the_l60_contract() {
let mut session = Session::default();
let granted = serde_json::to_value(session.grant(NOW)).unwrap();
- assert_eq!(granted, serde_json::json!({"active": true, "endsAt": NOW + SESSION_MS, "reason": "granted"}));
+ assert_eq!(granted, serde_json::json!({"active": true, "endsAt": NOW + SESSION_MS, "reason": "granted", "yolo": false}));
let panic = serde_json::to_value(session.end(Reason::Panic).unwrap()).unwrap();
- assert_eq!(panic, serde_json::json!({"active": false, "reason": "panic"}));
- assert_eq!(serde_json::to_value(session.status(NOW)).unwrap(), serde_json::json!({"active": false}));
+ assert_eq!(panic, serde_json::json!({"active": false, "reason": "panic", "yolo": false}));
+ assert_eq!(serde_json::to_value(session.status(NOW)).unwrap(), serde_json::json!({"active": false, "yolo": false}));
+ }
+
+ // --- Yolo mode (S24.9) ---
+
+ #[test]
+ fn yolo_is_off_until_turned_on_then_lasts_an_hour() {
+ let mut session = Session::default();
+ assert!(!session.yolo(NOW));
+ let on = session.set_yolo(true, NOW);
+ assert_eq!(on, StateEvent { yolo: true, yolo_ends_at: Some(NOW + YOLO_MS), ..StateEvent::default() });
+ assert!(session.yolo(NOW + YOLO_MS - 1));
+ assert_eq!(
+ serde_json::to_value(session.status(NOW)).unwrap(),
+ serde_json::json!({"active": false, "yolo": true, "yoloEndsAt": NOW + YOLO_MS})
+ );
+ assert_eq!(session.set_yolo(false, NOW + 5), StateEvent::default());
+ assert!(!session.yolo(NOW + 5));
+ }
+
+ #[test]
+ fn yolo_turns_itself_off_after_an_hour_and_reports_it() {
+ let mut session = Session::default();
+ session.set_yolo(true, NOW);
+ assert_eq!(session.expire(NOW + YOLO_MS - 1), None);
+ assert!(!session.yolo(NOW + YOLO_MS));
+ assert_eq!(session.expire(NOW + YOLO_MS), Some(StateEvent::default()));
+ assert_eq!(session.expire(NOW + YOLO_MS + 1), None);
+ // A session timeout and a yolo expiry at the same check are one event.
+ session.set_yolo(true, NOW);
+ session.grant(NOW + YOLO_MS - SESSION_MS);
+ let both = session.expire(NOW + YOLO_MS).unwrap();
+ assert_eq!((both.active, both.reason, both.yolo), (false, Some(Reason::Timeout), false));
+ }
+
+ #[test]
+ fn stop_and_panic_end_the_session_but_leave_yolo_to_its_switch() {
+ let mut session = Session::default();
+ session.set_yolo(true, NOW);
+ session.grant(NOW);
+ let ended = session.end(Reason::Panic).unwrap();
+ assert!(ended.yolo && session.yolo(NOW));
}
}
diff --git a/apps/desktop/src-tauri/src/shell.rs b/apps/desktop/src-tauri/src/shell.rs
index be80bb1..1a64568 100644
--- a/apps/desktop/src-tauri/src/shell.rs
+++ b/apps/desktop/src-tauri/src/shell.rs
@@ -260,7 +260,7 @@ fn conversations_on(value: Option<&str>) -> bool {
/// The tray menu, top to bottom (`-` is a separator). "New conversation" only exists with the
/// conversations list on; in the single continuous chat there is nothing new to start.
-/// "Stop computer control" exists where computer control does (macOS, S24.3).
+/// "Stop computer control" and "Yolo mode" exist where computer control does (macOS, S24.3, S24.9).
fn tray_menu_ids(conversations: bool, computer_control: bool) -> Vec<&'static str> {
let mut ids = vec!["open"];
if conversations {
@@ -268,7 +268,7 @@ fn tray_menu_ids(conversations: bool, computer_control: bool) -> Vec<&'static st
}
ids.push("browser");
if computer_control {
- ids.push("computer-stop");
+ ids.extend(["computer-stop", "computer-yolo"]);
}
ids.extend(["-", "restart", "autostart", "-", "quit"]);
ids
@@ -293,6 +293,12 @@ fn build_tray(app: &AppHandle, config: &DesktopConfig) -> tauri::Result<TrayIcon
true,
None::<&str>,
)?)?,
+ "computer-yolo" => {
+ // Off at launch: yolo mode never outlives the app (S24.9).
+ let item = CheckMenuItem::with_id(app, "computer-yolo", "Yolo mode", true, false, None::<&str>)?;
+ menu.append(&item)?;
+ app.manage(computer::YoloMenuItem(item));
+ }
"restart" => {
menu.append(&MenuItem::with_id(app, "restart", "Restart assistant", true, None::<&str>)?)?;
}
@@ -323,6 +329,10 @@ fn build_tray(app: &AppHandle, config: &DesktopConfig) -> tauri::Result<TrayIcon
computer::stop(app, Reason::Stopped);
Ok(())
}
+ "computer-yolo" => {
+ computer::toggle_yolo(app);
+ Ok(())
+ }
"restart" => {
restart_daemon(app.clone());
Ok(())
@@ -443,7 +453,8 @@ pub fn run() {
computer::computer_permissions,
computer::computer_request_permission,
computer::computer_stop,
- computer::computer_status
+ computer::computer_status,
+ computer::computer_set_yolo
])
.setup(move |app| {
#[cfg(target_os = "macos")]
@@ -518,7 +529,7 @@ mod tests {
fn tray_offers_stop_computer_control_where_it_exists() {
assert_eq!(
tray_menu_ids(false, true),
- vec!["open", "browser", "computer-stop", "-", "restart", "autostart", "-", "quit"]
+ vec!["open", "browser", "computer-stop", "computer-yolo", "-", "restart", "autostart", "-", "quit"]
);
}
diff --git a/docs/design.md b/docs/design.md
index db0ba40..e777c1f 100644
--- a/docs/design.md
+++ b/docs/design.md
@@ -169,6 +169,8 @@ The assistant can see the screen and operate apps through a built-in MCP server
- S24.5 Permissions: the app reports and requests Accessibility and Screen Recording (`computer_permissions`, `computer_request_permission {kind}`). The Connectors screen shows a built-in "Computer" entry with both statuses, Grant buttons, Check again, the note that debug builds lose the grants on every rebuild (macOS ties them to the code signature), and a note when the current model does not accept images (`ConnectorInfo.noImages`, from the model's `input` in the engine's `get_state` and `set_model` answers).
- S24.6 Visible state: the app emits `computer://state {active, endsAt?, reason?}` (`computer_status` gives the current one). While active, the rose shows the "in control" state (`dot-in_control`, mood `control`: accent rim, steady traveling light) and the panel shows "Controlling your Mac · mm:ss left" with Stop (`computer_stop`).
- S24.7 Wiring: on every connection the desktop panel calls `computer_endpoint` and, when it returns `{url, token}`, sends `computer_register {url, token}` (the protocol accepts only `http://127.0.0.1:<port>/mcp` and a header-safe key). The daemon keeps the endpoint in memory, never in `connectors.json`, and renders `computer: {url, headers: {Authorization: "Bearer <key>"}, exposure: "direct"}` into `mcp.json`, then restarts the engine when idle. The guard's policy lists it under `builtin`, so its tools run without in-engine approval cards. It is removed on `computer_unregister` or when the window that registered it disconnects. The key is never logged or sent to a window: `connectors` lists only a built-in "Computer" entry (`builtin: true`) that the connector messages cannot change or remove, no saved or imported connector can take the name `computer`, and the import scan never offers the helper.
+- S24.8 Speed: `batch {steps: [{tool, ...args}], intent?}` runs 1 to 20 steps of `click`, `move`, `drag`, `scroll`, `type`, `key`, `open_app`, or `wait` in one call and returns one screenshot. Each step takes the single tool's path (session and epoch, blocklist, risky confirmation, N1/N2 typing rules, coordinates mapped with the screenshot taken before the batch) and counts as one action for the rate limit; a full window delays the step instead of refusing it. Steps are 100 ms apart (1.5 s after `open_app`) so each step's guards read the screen the previous one left. The batch's `intent` applies to every step on top of the step's own. The first refused, declined, or failed step stops the batch; the error says which steps ran and comes with a fresh screenshot while the session lasts (none after Stop or panic). Malformed batches (empty, over 20, a nested `batch`, `screenshot`, `list_apps`, unknown tools, invalid arguments) run nothing. Actions settle 150 ms before their screenshot (`open_app` 1.5 s); screenshots are JPEG quality 65; debug builds optimize `image`, `zune-*`, and this crate (opt-level 1, because the generic encoder is compiled here).
+- S24.9 Yolo mode: in memory only, off at launch. `computer_set_yolo {enabled}` and the tray item "Yolo mode" switch it; turning it on asks a native confirmation ("Turn on yolo mode? Gentle Dot will send, pay, delete, and submit without asking you first, for up to 1 hour. Stop and ⌥⇧Esc still work.", Turn On / Cancel). While on, only the risky-action confirmation is skipped; the grant, blocklist, rate limit, Stop, and panic are unchanged, and Stop or panic end the session, not yolo mode. It turns itself off after 1 hour. `computer://state` and `computer_status` add `yolo` and `yoloEndsAt?`; every switch reports the resulting state, so a declined confirmation flips the UI switch back. Connectors → Computer shows a "Yolo mode" switch, the banner shows "Yolo", and the rose carries a yellow mark (`dot-yolo`).
Why enforcement lives in the app: everything inside the engine, and the daemon's WebSocket, is within the agent's reach (it has a shell, and with the daemon's access key it could answer its own approval cards, L49). The session grant, panic stop, blocklist, rate limit, and risky-action confirmations therefore run in the helper, behind native dialogs the agent cannot answer, and the in-engine guard adds nothing for these tools. macOS also attributes Accessibility and Screen Recording to the app that holds them, Gentle Dot.app. The key in `mcp.json` only reaches the helper, which still asks the user before anything happens.
diff --git a/packages/ui/src/App.tsx b/packages/ui/src/App.tsx
index 872d6f0..778f947 100644
--- a/packages/ui/src/App.tsx
+++ b/packages/ui/src/App.tsx
@@ -3,7 +3,7 @@ import { type ConnectionInfo, tokenFromLocation } from "./client.ts";
import { ChatSurface } from "./components/ChatSurface.tsx";
import type { ComputerControls } from "./components/Computer.tsx";
import { DotSurface } from "./components/DotSurface.tsx";
-import { computerPermissions, requestComputerPermission, stopComputer } from "./computer.ts";
+import { computerPermissions, requestComputerPermission, setComputerYolo, stopComputer } from "./computer.ts";
import {
connectionInfo,
hidePanel,
@@ -13,7 +13,7 @@ import {
startDragging,
togglePanel,
} from "./desktop.ts";
-import { useComputerRegistration, useComputerSession } from "./useComputer.ts";
+import { useComputerRegistration, useComputerState } from "./useComputer.ts";
import { useDot } from "./useDot.ts";
type Surface = "dot" | "panel" | "web";
@@ -52,7 +52,7 @@ export function App() {
conversations.current = state.features.conversations;
// Computer control lives in the desktop app only (S24.7); the panel registers the helper.
const desktop = inDesktop();
- const computerSession = useComputerSession(desktop && surface !== "web");
+ const { session: computerSession, yolo } = useComputerState(desktop && surface !== "web");
const computerAvailable = useComputerRegistration(
desktop && surface === "panel",
state.connection === "open",
@@ -63,7 +63,9 @@ export function App() {
? {
available: computerAvailable,
...(computerSession ? { session: computerSession } : {}),
+ ...(yolo ? { yolo } : {}),
stop: stopComputer,
+ setYolo: setComputerYolo,
permissions: computerPermissions,
requestPermission: requestComputerPermission,
}
@@ -113,6 +115,7 @@ export function App() {
agentState={state.agentState}
connected={state.connection === "open"}
inControl={computerSession !== undefined}
+ yolo={yolo !== undefined}
toggle={togglePanel}
startDrag={() => void startDragging()}
/>
diff --git a/packages/ui/src/components/ChatSurface.tsx b/packages/ui/src/components/ChatSurface.tsx
index 041dcf6..752f15d 100644
--- a/packages/ui/src/components/ChatSurface.tsx
+++ b/packages/ui/src/components/ChatSurface.tsx
@@ -156,7 +156,9 @@ export function ChatSurface({
) : null}
</header>
- {computer?.session ? <ComputerBanner session={computer.session} stop={computer.stop} /> : null}
+ {computer?.session ? (
+ <ComputerBanner session={computer.session} yolo={computer.yolo !== undefined} stop={computer.stop} />
+ ) : null}
{conversations && showConversations ? (
<nav className="conversations" aria-label="Earlier conversations">
diff --git a/packages/ui/src/components/Computer.tsx b/packages/ui/src/components/Computer.tsx
index 401f53a..bd812a5 100644
--- a/packages/ui/src/components/Computer.tsx
+++ b/packages/ui/src/components/Computer.tsx
@@ -4,6 +4,7 @@ import {
type ComputerPermission,
type ComputerPermissions,
type ComputerSession,
+ type ComputerYolo,
timeLeft,
} from "../computer.ts";
@@ -13,7 +14,11 @@ export interface ComputerControls {
available: boolean;
/** The session the user allowed, while it lasts. */
session?: ComputerSession;
+ /** Yolo mode, while it is on (S24.9). */
+ yolo?: ComputerYolo;
stop: () => void;
+ /** Asks the app; its native confirmation decides, and the state follows. */
+ setYolo: (enabled: boolean) => Promise<unknown>;
permissions: () => Promise<ComputerPermissions>;
requestPermission: (kind: ComputerPermission) => Promise<unknown>;
}
@@ -25,11 +30,19 @@ const PERMISSIONS: [ComputerPermission, string][] = [
export const DEBUG_BUILD_NOTE =
"Debug builds lose these permissions on every rebuild because macOS ties them to the app's signature; grant them again after updating.";
+const YOLO_NOTE =
+ "Yolo mode lets the assistant send, pay, delete, and submit without asking you first. It turns itself off after 1 hour or when the app quits; Stop and ⌥⇧Esc still work.";
const NO_IMAGES_NOTE =
"The current model does not accept images, so the assistant cannot see your screen. Choose a model that does in Profiles.";
+interface ComputerBannerProps {
+ session: ComputerSession;
+ yolo: boolean;
+ stop: () => void;
+}
+
/** "Controlling your Mac · mm:ss left" with a Stop button, while a session is active. */
-export function ComputerBanner({ session, stop }: { session: ComputerSession; stop: () => void }) {
+export function ComputerBanner({ session, yolo, stop }: ComputerBannerProps) {
const [now, setNow] = useState(() => Date.now());
useEffect(() => {
const timer = setInterval(() => setNow(Date.now()), 1000);
@@ -40,6 +53,11 @@ export function ComputerBanner({ session, stop }: { session: ComputerSession; st
<section className="computer-banner" aria-label="Computer control">
<span className="computer-banner-light" aria-hidden="true" />
<span>Controlling your Mac{left}</span>
+ {yolo ? (
+ <span className="chip computer-yolo-chip" title="Risky actions run without asking you first">
+ Yolo
+ </span>
+ ) : null}
<button type="button" className="primary" onClick={stop}>
Stop
</button>
@@ -55,7 +73,7 @@ interface ComputerEntryProps {
/** The built-in Computer connector: the macOS permissions it needs and how to grant them. */
export function ComputerEntry({ info, controls }: ComputerEntryProps) {
- const { permissions, requestPermission } = controls;
+ const { permissions, requestPermission, yolo, setYolo } = controls;
const [granted, setGranted] = useState<ComputerPermissions>();
const check = useCallback(() => {
permissions().then(setGranted, () => setGranted(undefined));
@@ -105,6 +123,22 @@ export function ComputerEntry({ info, controls }: ComputerEntryProps) {
Check again
</button>
</div>
+ <div className="computer-yolo">
+ <span id="computer-yolo-label">Yolo mode</span>
+ <button
+ type="button"
+ role="switch"
+ className="switch"
+ aria-checked={yolo !== undefined}
+ aria-labelledby="computer-yolo-label"
+ onClick={() => {
+ setYolo(yolo === undefined).catch(() => {});
+ }}
+ >
+ <span className="switch-knob" aria-hidden="true" />
+ </button>
+ </div>
+ <p className="connector-note">{YOLO_NOTE}</p>
<p className="connector-note">{DEBUG_BUILD_NOTE}</p>
{info?.noImages ? <p className="connector-note computer-warning">{NO_IMAGES_NOTE}</p> : null}
</li>
diff --git a/packages/ui/src/components/DotSurface.tsx b/packages/ui/src/components/DotSurface.tsx
index de86bb1..1b2f278 100644
--- a/packages/ui/src/components/DotSurface.tsx
+++ b/packages/ui/src/components/DotSurface.tsx
@@ -29,20 +29,30 @@ interface DotSurfaceProps {
connected: boolean;
/** The assistant controls the Mac (S24.6); it shows over every other state. */
inControl?: boolean;
+ /** Yolo mode is on (S24.9): risky actions run without asking, so the rose says so. */
+ yolo?: boolean;
toggle: () => void;
startDrag: () => void;
}
/** The collapsed, always-on-top rose in its black circle. A click opens the panel; a drag moves the window. */
-export function DotSurface({ agentState, connected, inControl = false, toggle, startDrag }: DotSurfaceProps) {
+export function DotSurface({
+ agentState,
+ connected,
+ inControl = false,
+ yolo = false,
+ toggle,
+ startDrag,
+}: DotSurfaceProps) {
const press = useRef<{ x: number; y: number; dragging: boolean } | undefined>(undefined);
const state = connected ? agentState : "restarting";
+ const label = inControl ? "controlling your Mac" : LABELS[state];
return (
<button
type="button"
- className={`dot dot-${state}${inControl ? " dot-in_control" : ""}`}
- aria-label={`Gentle Dot: ${inControl ? "controlling your Mac" : LABELS[state]}`}
+ className={`dot dot-${state}${inControl ? " dot-in_control" : ""}${yolo ? " dot-yolo" : ""}`}
+ aria-label={`Gentle Dot: ${label}${yolo ? ", yolo mode on" : ""}`}
onPointerDown={(event) => {
if (event.button !== 0) return;
press.current = { x: event.clientX, y: event.clientY, dragging: false };
@@ -70,6 +80,7 @@ export function DotSurface({ agentState, connected, inControl = false, toggle, s
<span className="dot-disc" aria-hidden="true">
<Rose mood={inControl ? "control" : MOODS[state]} />
</span>
+ {yolo ? <span className="dot-yolo-mark" aria-hidden="true" /> : null}
{state === "needs_you" ? (
<span className="dot-badge" aria-hidden="true">
!
diff --git a/packages/ui/src/computer.ts b/packages/ui/src/computer.ts
index 3a0d272..4bb649b 100644
--- a/packages/ui/src/computer.ts
+++ b/packages/ui/src/computer.ts
@@ -23,10 +23,23 @@ export interface ComputerSession {
endsAt?: number;
}
+/** Yolo mode (S24.9): risky actions run without asking; `endsAt` in milliseconds since the epoch. */
+export interface ComputerYolo {
+ endsAt?: number;
+}
+
+/** The session while one is active, and yolo mode while it is on; they come and go separately. */
+export interface ComputerState {
+ session?: ComputerSession;
+ yolo?: ComputerYolo;
+}
+
/** What the app reports on `computer://state` and from `computer_status`. */
interface ComputerStateEvent {
active?: unknown;
endsAt?: unknown;
+ yolo?: unknown;
+ yoloEndsAt?: unknown;
}
/** The helper's endpoint, or null when the app has none (off macOS, or the helper did not start). */
@@ -54,21 +67,34 @@ export function stopComputer(): void {
if (isTauri()) void invoke("computer_stop");
}
-export async function computerStatus(): Promise<ComputerSession | undefined> {
- if (!isTauri()) return undefined;
- return sessionOf(await invoke<ComputerStateEvent>("computer_status"));
+/**
+ * Asks the app to turn yolo mode on or off. Turning it on shows the app's own confirmation, and
+ * the answer arrives on `computer://state`; nothing here assumes it.
+ */
+export function setComputerYolo(enabled: boolean): Promise<unknown> {
+ if (!isTauri()) return Promise.resolve();
+ return invoke("computer_set_yolo", { enabled });
+}
+
+export async function computerStatus(): Promise<ComputerState> {
+ if (!isTauri()) return {};
+ return stateOf(await invoke<ComputerStateEvent>("computer_status"));
}
-export async function onComputerState(handler: (session: ComputerSession | undefined) => void) {
+export async function onComputerState(handler: (state: ComputerState) => void) {
if (!isTauri()) return () => {};
const { listen } = await import("@tauri-apps/api/event");
- return listen<ComputerStateEvent>("computer://state", (event) => handler(sessionOf(event.payload)));
+ return listen<ComputerStateEvent>("computer://state", (event) => handler(stateOf(event.payload)));
+}
+
+function stateOf(state: ComputerStateEvent | null | undefined): ComputerState {
+ const session = state?.active === true ? { ...timeOf(state.endsAt) } : undefined;
+ const yolo = state?.yolo === true ? { ...timeOf(state.yoloEndsAt) } : undefined;
+ return { ...(session ? { session } : {}), ...(yolo ? { yolo } : {}) };
}
-/** The session while it is active; an `endsAt` in seconds is read as such. */
-function sessionOf(state: ComputerStateEvent | null | undefined): ComputerSession | undefined {
- if (state?.active !== true) return undefined;
- const endsAt = state.endsAt;
+/** `{endsAt}` in milliseconds when the app sent a time; one in seconds is read as such. */
+function timeOf(endsAt: unknown): { endsAt?: number } {
if (typeof endsAt !== "number" || !Number.isFinite(endsAt)) return {};
return { endsAt: endsAt < 1e12 ? endsAt * 1000 : endsAt };
}
diff --git a/packages/ui/src/styles.css b/packages/ui/src/styles.css
index ed27a9a..6321d7f 100644
--- a/packages/ui/src/styles.css
+++ b/packages/ui/src/styles.css
@@ -449,6 +449,20 @@ textarea:focus {
0 0 4px rgba(245, 165, 36, 0.7);
}
+/* Yolo mode (S24.9): a yellow mark at the bottom right, over every state, while risky actions run unasked. */
+.dot-yolo-mark {
+ position: absolute;
+ right: 5px;
+ bottom: 5px;
+ width: 10px;
+ height: 10px;
+ border-radius: 50%;
+ background: var(--yellow);
+ box-shadow:
+ 0 0 0 1.5px #050506,
+ 0 0 6px var(--yellow);
+}
+
/* Reduce motion: no traveling light (frozen dashes would look broken); the glow stays, stronger when busy.
Glow radii stay small so the glow stays inside the disc. */
@media (prefers-reduced-motion: reduce) {
@@ -692,6 +706,57 @@ html[data-effects="none"] .chat-panel {
animation: pulse 1.6s ease-in-out infinite;
}
+.computer-yolo-chip {
+ color: var(--yellow);
+ border-color: var(--yellow);
+}
+
+.computer-yolo {
+ display: flex;
+ align-items: center;
+ gap: 8px;
+ margin-top: 8px;
+ font-size: 13px;
+}
+
+.computer-yolo > span:first-child {
+ flex: 1;
+}
+
+/* An on/off switch: a pill with a knob, the accent when on. */
+.switch {
+ position: relative;
+ flex: none;
+ width: 36px;
+ height: 20px;
+ padding: 0;
+ border-radius: var(--radius-pill);
+ border: 1px solid var(--line);
+ background: var(--surface);
+}
+
+.switch[aria-checked="true"] {
+ border-color: var(--accent);
+ background: var(--accent);
+ box-shadow: var(--accent-shadow);
+}
+
+.switch-knob {
+ position: absolute;
+ top: 2px;
+ left: 2px;
+ width: 14px;
+ height: 14px;
+ border-radius: 50%;
+ background: var(--muted);
+ transition: transform 150ms var(--ease);
+}
+
+.switch[aria-checked="true"] .switch-knob {
+ transform: translateX(16px);
+ background: var(--accent-ink);
+}
+
.computer-permissions {
display: grid;
gap: 6px;
diff --git a/packages/ui/src/useComputer.ts b/packages/ui/src/useComputer.ts
index 2f9bd9f..df34f73 100644
--- a/packages/ui/src/useComputer.ts
+++ b/packages/ui/src/useComputer.ts
@@ -1,15 +1,15 @@
import type { ClientMessage } from "@gentle-dot/protocol";
import { useEffect, useState } from "react";
-import { type ComputerSession, computerEndpoint, computerStatus, onComputerState } from "./computer.ts";
+import { type ComputerState, computerEndpoint, computerStatus, onComputerState } from "./computer.ts";
-/** The control session the app reports, while one is active (desktop windows only). */
-export function useComputerSession(enabled: boolean): ComputerSession | undefined {
- const [session, setSession] = useState<ComputerSession | undefined>(undefined);
+/** The control session and yolo mode the app reports (desktop windows only). */
+export function useComputerState(enabled: boolean): ComputerState {
+ const [state, setState] = useState<ComputerState>({});
useEffect(() => {
if (!enabled) return;
let live = true;
- const update = (next: ComputerSession | undefined) => {
- if (live) setSession(next);
+ const update = (next: ComputerState) => {
+ if (live) setState(next);
};
computerStatus().then(update, () => {});
const unlisten = onComputerState(update);
@@ -18,7 +18,7 @@ export function useComputerSession(enabled: boolean): ComputerSession | undefine
void unlisten.then((stop) => stop());
};
}, [enabled]);
- return session;
+ return state;
}
/**
diff --git a/packages/ui/test/computer.test.tsx b/packages/ui/test/computer.test.tsx
index cf44c37..36dc4a9 100644
--- a/packages/ui/test/computer.test.tsx
+++ b/packages/ui/test/computer.test.tsx
@@ -268,3 +268,70 @@ describe("while the assistant controls the Mac (S24.6)", () => {
expect(rule(".computer-banner")).toMatch(/border-radius: var\(--radius-md\)/);
});
});
+
+describe("yolo mode (S24.9)", () => {
+ const yoloSwitch = async () => {
+ const entry = await screen.findByRole("listitem", { name: "Computer" });
+ return within(entry).getByRole("switch", { name: "Yolo mode" });
+ };
+
+ it("the switch in Connectors asks the app, and only the app's state moves it", async () => {
+ h.state = connectorsOpen([computerEntry()]);
+ await renderApp("?surface=panel");
+ const toggle = await yoloSwitch();
+ expect(toggle).toHaveAttribute("aria-checked", "false");
+
+ await userEvent.click(toggle);
+ expect(h.invoke).toHaveBeenCalledWith("computer_set_yolo", { enabled: true });
+ // The native confirmation decides; nothing changes until the app reports it.
+ expect(toggle).toHaveAttribute("aria-checked", "false");
+
+ await emitState({ active: false, yolo: true, yoloEndsAt: Date.now() + 3_600_000 });
+ expect(toggle).toHaveAttribute("aria-checked", "true");
+ await userEvent.click(toggle);
+ expect(h.invoke).toHaveBeenCalledWith("computer_set_yolo", { enabled: false });
+
+ await emitState({ active: false, yolo: false });
+ expect(toggle).toHaveAttribute("aria-checked", "false");
+ });
+
+ it("reflects yolo mode that was already on when the panel opened", async () => {
+ h.status = { active: false, yolo: true, yoloEndsAt: Date.now() + 60_000 };
+ h.state = connectorsOpen([computerEntry()]);
+ await renderApp("?surface=panel");
+ const toggle = await yoloSwitch();
+ await waitFor(() => expect(toggle).toHaveAttribute("aria-checked", "true"));
+ });
+
+ it("the session banner says Yolo while it is on", async () => {
+ await renderApp("?surface=panel");
+ await emitState({ active: true, endsAt: Date.now() + 60_000, reason: "granted", yolo: true });
+ const banner = screen.getByRole("region", { name: "Computer control" });
+ expect(within(banner).getByText("Yolo")).toBeInTheDocument();
+ await emitState({ active: true, endsAt: Date.now() + 60_000, yolo: false });
+ expect(within(banner).queryByText("Yolo")).toBeNull();
+ });
+
+ it("the rose carries a yolo marker while it is on", async () => {
+ await renderApp("?surface=dot");
+ const dot = screen.getByRole("button");
+ await emitState({ active: true, endsAt: Date.now() + 60_000, yolo: true });
+ expect(dot).toHaveClass("dot-in_control", "dot-yolo");
+ expect(dot).toHaveAccessibleName("Gentle Dot: controlling your Mac, yolo mode on");
+ expect(dot.querySelector(".dot-yolo-mark")).not.toBeNull();
+
+ await emitState({ active: true, endsAt: Date.now() + 60_000, yolo: false });
+ expect(dot).not.toHaveClass("dot-yolo");
+ expect(dot.querySelector(".dot-yolo-mark")).toBeNull();
+ });
+
+ it("styles the yolo marker and switch with the site's tokens", () => {
+ const stylesheet = "../src/styles.css";
+ const css = readFileSync(fileURLToPath(new URL(stylesheet, import.meta.url)), "utf8");
+ const rule = (selector: string) =>
+ css.match(new RegExp(`\\n${selector.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")} \\{([^}]*)\\}`))?.[1] ?? "";
+ expect(rule(".dot-yolo-mark")).toMatch(/var\(--yellow/);
+ expect(rule(".computer-yolo-chip")).toMatch(/var\(--yellow/);
+ expect(rule('.switch[aria-checked="true"]')).toMatch(/var\(--accent/);
+ });
+});