Make your computer a home for your AI partner. Avibe is the local-first Agent OS: Vibey, its built-in AI partner, works out of the box, and Claude Code, Codex & OpenCode run unchanged beside it — reach them from your browser, phone, or any chat app.
Python agentagent-osaiai-agentsai-partnerchatopsclaudeclaude-codecodexdevtoolsdiscord-botlark-botllmlocal-firstlong-term-memoryopencodeslack-botvibe-codingvibeywechat
feat(desktop): make the Computer Use setup guide actionable and live (#2936)
Maxb83d62de60f2
diff --git a/desktop/README.md b/desktop/README.mdindex 548447a82..4bdabd5b3 100644--- a/desktop/README.md+++ b/desktop/README.md@@ -254,6 +254,7 @@ Packaged installations register `avibe://` for native, single-window navigation:| `avibe://session/<id>` | `/chat/<id>` || `avibe://show/<id>` | `/apps/show/<id>` || `avibe://settings` | `/settings/general` |+| `avibe://settings/computer-use` | `/settings/computer-use` || `avibe://vaults/request/<id>` | `/vaults?request_id=<id>` |IDs use literal ASCII letters, digits, `.`, `_`, and `-`, up to 128 bytes;diff --git a/desktop/runtime-host/src/deep_link.rs b/desktop/runtime-host/src/deep_link.rsindex d07a82e27..9a195c777 100644--- a/desktop/runtime-host/src/deep_link.rs+++ b/desktop/runtime-host/src/deep_link.rs@@ -49,6 +49,7 @@ pub fn parse_deep_link(raw: &str) -> Option<DeepLinkTarget> {format!("{prefix}/{identifier}")}("settings", "") => "/settings/general".to_owned(),+ ("settings", "/computer-use") => "/settings/computer-use".to_owned(),("vaults", path) => {let identifier = path.strip_prefix("/request/")?;if !valid_identifier(identifier) {@@ -163,6 +164,7 @@ mod tests {("avibe://session/ses.A_b-9", "/chat/ses.A_b-9"),("avibe://show/ses.A_b-9", "/apps/show/ses.A_b-9"),("avibe://settings", "/settings/general"),+ ("avibe://settings/computer-use", "/settings/computer-use"),("avibe://vaults/request/req.A_b-9", "/vaults?request_id=req.A_b-9"),];diff --git a/desktop/src-tauri/build.rs b/desktop/src-tauri/build.rsindex 5ff50ddaf..0b467ea46 100644--- a/desktop/src-tauri/build.rs+++ b/desktop/src-tauri/build.rs@@ -5,12 +5,15 @@ fn main() {// generate per-command permissions, so the capability files become the only// thing that can hand them out: `bootstrap.json` gives the bootstrap// commands only to the shell's own local page, `pet.json` gives the pet- // commands only to the pet window, and `main-pet-bind.json` gives the- // Workbench `pet_bind` alone.+ // commands only to the pet window, `main-pet-bind.json` gives the+ // Workbench `pet_bind` alone, and `main-computer-use.json` gives it the+ // two Computer Use guide actions on macOS.let manifest = tauri_build::AppManifest::new().commands(&["bootstrap_status","bootstrap_retry","open_install_docs",+ "computer_use_enable",+ "computer_use_open_permission","pet_ready","pet_set_expanded","pet_hit_regions",diff --git a/desktop/src-tauri/capabilities/main-computer-use.json b/desktop/src-tauri/capabilities/main-computer-use.jsonnew file mode 100644index 000000000..71c40f2c3--- /dev/null+++ b/desktop/src-tauri/capabilities/main-computer-use.json@@ -0,0 +1,12 @@+{+ "$schema": "../gen/schemas/desktop-schema.json",+ "identifier": "main-computer-use",+ "description": "The Workbench's Computer Use guide: turn Computer Use on, and ask for one named privacy permission and open its System Settings pane. Granted to the main window for the loopback Runtime origin only, on macOS only, never to the bundled bootstrap page.",+ "platforms": ["macOS"],+ "local": false,+ "windows": ["main"],+ "remote": {+ "urls": ["http://127.0.0.1:*/*", "http://[\\:\\:1]:*/*"]+ },+ "permissions": ["allow-computer-use-enable", "allow-computer-use-open-permission"]+}diff --git a/desktop/src-tauri/permissions/autogenerated/computer_use_enable.toml b/desktop/src-tauri/permissions/autogenerated/computer_use_enable.tomlnew file mode 100644index 000000000..4db039b0c--- /dev/null+++ b/desktop/src-tauri/permissions/autogenerated/computer_use_enable.toml@@ -0,0 +1,11 @@+# Automatically generated - DO NOT EDIT!++[[permission]]+identifier = "allow-computer-use-enable"+description = "Enables the computer_use_enable command without any pre-configured scope."+commands.allow = ["computer_use_enable"]++[[permission]]+identifier = "deny-computer-use-enable"+description = "Denies the computer_use_enable command without any pre-configured scope."+commands.deny = ["computer_use_enable"]diff --git a/desktop/src-tauri/permissions/autogenerated/computer_use_open_permission.toml b/desktop/src-tauri/permissions/autogenerated/computer_use_open_permission.tomlnew file mode 100644index 000000000..717eff0b6--- /dev/null+++ b/desktop/src-tauri/permissions/autogenerated/computer_use_open_permission.toml@@ -0,0 +1,11 @@+# Automatically generated - DO NOT EDIT!++[[permission]]+identifier = "allow-computer-use-open-permission"+description = "Enables the computer_use_open_permission command without any pre-configured scope."+commands.allow = ["computer_use_open_permission"]++[[permission]]+identifier = "deny-computer-use-open-permission"+description = "Denies the computer_use_open_permission command without any pre-configured scope."+commands.deny = ["computer_use_open_permission"]diff --git a/desktop/src-tauri/src/computer_use.rs b/desktop/src-tauri/src/computer_use.rsindex 6fae98021..aafce895e 100644--- a/desktop/src-tauri/src/computer_use.rs+++ b/desktop/src-tauri/src/computer_use.rs@@ -10,7 +10,7 @@ use std::process::Stdio;use std::ptr;use std::sync::atomic::{AtomicBool, Ordering};use std::sync::{Arc, Mutex};-use std::time::{Duration, Instant};+use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};use avibe_runtime_host::computer_use::{CapabilityCache, CapabilityProbe, CapabilityVerdict, ComputerUseLifecycle, ComputerUsePhase, ComputerUseRecord,@@ -51,6 +51,11 @@ const HEALTH_TIMEOUT: Duration = Duration::from_secs(5);const PERMISSION_REQUEST_TIMEOUT: Duration = Duration::from_secs(5);const PERMISSION_CHILD_TIMEOUT: Duration = Duration::from_secs(10);const START_BACKOFF: Duration = Duration::from_secs(1);+/// Written when the Workbench guide opens Screen Recording, so a macOS+/// "Quit & Reopen" brings the relaunched shell back to the guide.+const SETUP_RESUME_FILE: &str = "permission-setup-resume";+/// How long after that request a launch still counts as its relaunch.+const SETUP_RESUME_WINDOW: Duration = Duration::from_secs(15 * 60);const COMPUTER_USE_SCHEMA: u64 = COMPUTER_USE_SCHEMA_VERSION as u64;const DARWIN_SOCKET_PATH_MAX_BYTES: usize = 103;const SOCKET_DIRECTORY_MODE: u32 = 0o700;@@ -131,8 +136,36 @@ impl Default for MenuView {}}+/// A System Settings privacy pane the Workbench guide may open. The page names+/// one of these; it never supplies a URL.+#[derive(Clone, Copy, Debug, Eq, PartialEq)]+pub(crate) enum PermissionPane {+ Accessibility,+ ScreenRecording,+}++impl PermissionPane {+ pub(crate) fn parse(value: &str) -> Option<Self> {+ match value {+ "accessibility" => Some(Self::Accessibility),+ "screen_recording" => Some(Self::ScreenRecording),+ _ => None,+ }+ }++ /// Checked on macOS 26.2: both land on their Privacy & Security list.+ fn settings_url(self) -> &'static str {+ match self {+ Self::Accessibility => "x-apple.systempreferences:com.apple.preference.security?Privacy_Accessibility",+ Self::ScreenRecording => "x-apple.systempreferences:com.apple.preference.security?Privacy_ScreenCapture",+ }+ }+}+enum Event {Toggle,+ Enable,+ OpenPermission(PermissionPane),RuntimeReady { origin: LoopbackOrigin, adoption: bool },RuntimeLost,RuntimeStopped,@@ -181,6 +214,27 @@ impl Controller {let _ = self.sender.send(Event::Toggle);}+ /// Turns Computer Use on exactly as the tray toggle does, and does nothing+ /// when it is already on: a page acting on a stale status must not turn it+ /// off.+ pub(crate) fn turn_on(&self) -> Result<(), String> {+ self.request(Event::Enable)+ }++ /// Asks macOS for `pane`'s grant where it allows a prompt, then opens the+ /// pane in System Settings.+ pub(crate) fn open_permission(&self, pane: PermissionPane) -> Result<(), String> {+ self.request(Event::OpenPermission(pane))+ }++ /// Hands a page request to the lifecycle. A degraded controller has no+ /// lifecycle task, so the page is told the request went nowhere.+ fn request(&self, event: Event) -> Result<(), String> {+ self.sender+ .send(event)+ .map_err(|_| "Computer Use is unavailable.".to_owned())+ }+pub(crate) fn runtime_ready(&self, origin: LoopbackOrigin, adoption: bool) {let _ = self.sender.send(Event::RuntimeReady { origin, adoption });}@@ -355,6 +409,7 @@ struct RuntimeState {host_bundle_id: String,assets_valid: bool,pending_write: Option<ComputerUseRecord>,+ setup_resume_armed: bool,started_at: Instant,}@@ -394,6 +449,7 @@ impl RuntimeState {host_bundle_id: String::new(),assets_valid,pending_write: None,+ setup_resume_armed: false,started_at: Instant::now(),};runtime.host_bundle_id = runtime.app.config().identifier.clone();@@ -428,6 +484,12 @@ impl RuntimeState {};match event {Event::Toggle => self.toggle().await,+ Event::Enable => {+ if !self.lifecycle.enabled() {+ self.toggle().await;+ }+ }+ Event::OpenPermission(pane) => self.open_permission(pane).await,Event::RuntimeReady { origin, adoption } => {self.runtime_ready(origin, adoption).await;}@@ -484,6 +546,21 @@ impl RuntimeState {self.apply(directive).await;}+ async fn open_permission(&mut self, pane: PermissionPane) {+ match pane {+ PermissionPane::Accessibility => request_accessibility_grant(&self.app, &self.diagnostics).await,+ PermissionPane::ScreenRecording => {+ // The full request also runs the one-pixel capture, which is+ // what lists Avibe in the Screen Recording pane.+ let _ = request_permissions(&self.app, &self.diagnostics).await;+ self.setup_resume_armed = arm_setup_resume(&self.paths.state_dir, SystemTime::now()).is_ok();+ }+ }+ open_permission_settings(&self.app, pane);+ self.activated().await;+ self.settle_setup_resume();+ }+async fn runtime_ready(&mut self, origin: LoopbackOrigin, adoption: bool) {if adoption {self.invalidate_start_retry();@@ -661,12 +738,22 @@ impl RuntimeState {return false;}self.publish_view();+ self.settle_setup_resume();if directive.spawn_permission_probe {self.start_permission_probe().await;}true}+ /// Once this process has seen Screen Recording settle either way, no+ /// relaunch is needed to finish the guide, so the marker goes.+ fn settle_setup_resume(&mut self) {+ if self.setup_resume_armed && !awaiting_screen_recording(self.lifecycle.phase(), self.lifecycle.reason()) {+ let _ = std::fs::remove_file(self.paths.state_dir.join(SETUP_RESUME_FILE));+ self.setup_resume_armed = false;+ }+ }+fn invalidate_start_retry(&mut self) {self.start_retry_id = self.start_retry_id.wrapping_add(1);}@@ -885,6 +972,39 @@ impl RuntimeState {}}+fn awaiting_screen_recording(phase: ComputerUsePhase, reason: Option<&str>) -> bool {+ phase == ComputerUsePhase::NeedsPermission && reason == Some("screen_recording")+}++fn unix_seconds(time: SystemTime) -> u64 {+ time.duration_since(UNIX_EPOCH).map_or(0, |elapsed| elapsed.as_secs())+}++fn arm_setup_resume(state_dir: &Path, now: SystemTime) -> io::Result<()> {+ std::fs::write(state_dir.join(SETUP_RESUME_FILE), unix_seconds(now).to_string())+}++/// Consumes the marker a guide-driven Screen Recording request left, and says+/// whether this launch is that request's relaunch. A marker is read once:+/// stale or not, it never steers a second launch.+fn take_setup_resume_at(state_dir: &Path, now: SystemTime) -> bool {+ let path = state_dir.join(SETUP_RESUME_FILE);+ let Ok(contents) = std::fs::read_to_string(&path) else {+ return false;+ };+ let _ = std::fs::remove_file(&path);+ let Ok(armed_at) = contents.trim().parse::<u64>() else {+ return false;+ };+ let now = unix_seconds(now);+ armed_at <= now && now - armed_at <= SETUP_RESUME_WINDOW.as_secs()+}++/// Whether this launch should reopen the Computer Use guide.+pub(crate) fn take_setup_resume(app: &AppHandle) -> bool {+ state_dir(app).is_ok_and(|dir| take_setup_resume_at(&dir, SystemTime::now()))+}+fn restored_lifecycle(stored: &StoredComputerUseState, assets_valid: bool) -> ComputerUseLifecycle {let mut lifecycle = match stored {StoredComputerUseState::Current(record) => ComputerUseLifecycle::from_record(record),@@ -1772,6 +1892,30 @@ async fn request_permissions(app: &AppHandle, diagnostics: &BootstrapLog) -> Graresult.grants()}+/// The Accessibility half of [`request_permissions`]: the system prompt lists+/// Avibe in the pane, and Screen Recording is left alone.+async fn request_accessibility_grant(app: &AppHandle, diagnostics: &BootstrapLog) {+ let (sender, receiver) = oneshot::channel();+ let dispatched = app.run_on_main_thread(move || {+ let granted = if MainThreadMarker::new().is_some() {+ Some(request_accessibility())+ } else {+ None+ };+ let _ = sender.send(granted);+ });+ let outcome = match dispatched {+ Err(_) => "dispatch_failed".to_owned(),+ Ok(()) => match timeout(PERMISSION_REQUEST_TIMEOUT, receiver).await {+ Ok(Ok(Some(granted))) => granted.to_string(),+ Ok(Ok(None)) => "non_main".to_owned(),+ Ok(Err(_)) => "dropped".to_owned(),+ Err(_) => "timed_out".to_owned(),+ },+ };+ diagnostics.record("computer-use.accessibility-request", &[("accessibility", outcome)]);+}+fn request_accessibility() -> bool {// SAFETY: The dictionary contains two process-lifetime CoreFoundation// constants. The prompting call does not retain it after returning.@@ -2040,16 +2184,17 @@ fn record_permission_request(diagnostics: &BootstrapLog, result: &PermissionRequ}fn open_missing_permission_settings(app: &AppHandle, grants: Grants) {- let screen_recording_needs_guidance = grants.accessibility && !grants.screen_recording;- let url = if !grants.accessibility {- "x-apple.systempreferences:com.apple.preference.security?Privacy_Accessibility"+ let pane = if grants.accessibility {+ PermissionPane::ScreenRecording} else {- "x-apple.systempreferences:com.apple.preference.security?Privacy_ScreenCapture"+ PermissionPane::Accessibility};- if tauri_plugin_opener::open_url(url, None::<&str>).is_err() {- let _ = app;- }- if screen_recording_needs_guidance {+ open_permission_settings(app, pane);+}++fn open_permission_settings(app: &AppHandle, pane: PermissionPane) {+ let _ = tauri_plugin_opener::open_url(pane.settings_url(), None::<&str>);+ if pane == PermissionPane::ScreenRecording {let catalog = crate::native_catalog_for_locales(sys_locale::get_locales()).computer_use;app.dialog().message(catalog.screen_recording_guide)@@ -2744,6 +2889,105 @@ done);}+ #[test]+ fn a_degraded_controller_reports_that_page_requests_went_nowhere() {+ let (sender, receiver) = mpsc::unbounded_channel();+ let controller = Controller {+ sender,+ view: Arc::new(Mutex::new(MenuView::default())),+ shutdown_requested: Arc::new(AtomicBool::new(false)),+ shutdown_started: AtomicBool::new(false),+ };+ assert!(controller.turn_on().is_ok());+ assert!(controller.open_permission(PermissionPane::Accessibility).is_ok());+ // `RuntimeState::new` failing drops the receiver; nothing will run.+ drop(receiver);+ assert!(controller.turn_on().is_err());+ assert!(controller.open_permission(PermissionPane::ScreenRecording).is_err());+ }++ #[test]+ fn the_guide_names_only_the_two_privacy_panes() {+ assert_eq!(+ PermissionPane::parse("accessibility"),+ Some(PermissionPane::Accessibility)+ );+ assert_eq!(+ PermissionPane::parse("screen_recording"),+ Some(PermissionPane::ScreenRecording)+ );+ for rejected in [+ "",+ "Accessibility",+ "screen-recording",+ "camera",+ "x-apple.systempreferences:",+ ] {+ assert_eq!(PermissionPane::parse(rejected), None, "{rejected}");+ }+ assert_eq!(+ PermissionPane::Accessibility.settings_url(),+ "x-apple.systempreferences:com.apple.preference.security?Privacy_Accessibility"+ );+ assert_eq!(+ PermissionPane::ScreenRecording.settings_url(),+ "x-apple.systempreferences:com.apple.preference.security?Privacy_ScreenCapture"+ );+ }++ #[test]+ fn a_setup_resume_marker_steers_one_launch_inside_its_window() {+ let temporary = tempfile::tempdir().expect("temporary directory");+ let dir = temporary.path();+ let armed = UNIX_EPOCH + Duration::from_secs(1_800_000_000);+ assert!(!take_setup_resume_at(dir, armed), "no marker, no resume");++ arm_setup_resume(dir, armed).unwrap();+ assert!(take_setup_resume_at(dir, armed + SETUP_RESUME_WINDOW));+ assert!(+ !take_setup_resume_at(dir, armed + SETUP_RESUME_WINDOW),+ "a marker is consumed by the launch that reads it"+ );++ for (now, label) in [+ (armed + SETUP_RESUME_WINDOW + Duration::from_secs(1), "stale"),+ (armed - Duration::from_secs(1), "from the future"),+ ] {+ arm_setup_resume(dir, armed).unwrap();+ assert!(!take_setup_resume_at(dir, now), "{label}");+ assert!(+ !dir.join(SETUP_RESUME_FILE).exists(),+ "a {label} marker is still removed"+ );+ }++ std::fs::write(dir.join(SETUP_RESUME_FILE), "not a time").unwrap();+ assert!(!take_setup_resume_at(dir, armed));+ assert!(!dir.join(SETUP_RESUME_FILE).exists());+ }++ #[test]+ fn only_a_pending_screen_recording_grant_keeps_the_resume_marker() {+ assert!(awaiting_screen_recording(+ ComputerUsePhase::NeedsPermission,+ Some("screen_recording")+ ));+ assert!(!awaiting_screen_recording(+ ComputerUsePhase::NeedsPermission,+ Some("accessibility")+ ));+ for phase in [+ ComputerUsePhase::Off,+ ComputerUsePhase::Starting,+ ComputerUsePhase::Ready,+ ComputerUsePhase::Error,+ ComputerUsePhase::Stopped,+ ComputerUsePhase::NeedsRuntime,+ ] {+ assert!(!awaiting_screen_recording(phase, Some("screen_recording")), "{phase:?}");+ }+ }+#[test]fn restored_enablement_with_invalid_assets_cannot_spawn() {let record = ComputerUseRecord {diff --git a/desktop/src-tauri/src/lib.rs b/desktop/src-tauri/src/lib.rsindex b125f29fb..01f018a27 100644--- a/desktop/src-tauri/src/lib.rs+++ b/desktop/src-tauri/src/lib.rs@@ -95,6 +95,9 @@ const SETTINGS_DEEP_LINK: &str = "avibe://settings";/// A Workbench that handles it cancels it; an uncancelled one falls back to/// loading the deep link's destination.const OPEN_SETTINGS_EVENT: &str = "avibe:desktop-open-settings";+/// Where a relaunch after a guide-driven Screen Recording grant returns.+#[cfg(target_os = "macos")]+const COMPUTER_USE_SETUP_DEEP_LINK: &str = "avibe://settings/computer-use";#[cfg(feature = "bundled-runtime")]const ACTIVITY_UNINSTALL: u8 = 3;@@ -1168,6 +1171,53 @@ fn open_install_docs(window: WebviewWindow) -> Result<(), String> {.map_err(|_| "Installation help could not be opened.".to_owned())}+/// The Computer Use guide's "Turn on": the tray toggle's on half.+#[tauri::command]+fn computer_use_enable(window: WebviewWindow) -> Result<(), String> {+ #[cfg(target_os = "macos")]+ {+ computer_use_guide_controller(&window)?.turn_on()+ }+ #[cfg(not(target_os = "macos"))]+ {+ let _ = window;+ Err(COMPUTER_USE_UNSUPPORTED.to_owned())+ }+}++/// The Computer Use guide's "Open Settings" for one privacy pane, named by+/// `pane` (`accessibility` or `screen_recording`). The page never supplies a+/// URL, so this cannot become a general System Settings opener.+#[tauri::command]+fn computer_use_open_permission(window: WebviewWindow, pane: String) -> Result<(), String> {+ #[cfg(target_os = "macos")]+ {+ let pane = computer_use::PermissionPane::parse(&pane).ok_or_else(|| "Not a permission pane.".to_owned())?;+ computer_use_guide_controller(&window)?.open_permission(pane)+ }+ #[cfg(not(target_os = "macos"))]+ {+ let _ = (window, pane);+ Err(COMPUTER_USE_UNSUPPORTED.to_owned())+ }+}++#[cfg(not(target_os = "macos"))]+const COMPUTER_USE_UNSUPPORTED: &str = "Computer Use is only available on macOS.";++/// The capability (`main-computer-use.json`) limits these commands to the main+/// window on a loopback page; this also requires the Runtime the shell adopted,+/// so a same-window page on another loopback port cannot drive them.+#[cfg(target_os = "macos")]+fn computer_use_guide_controller(window: &WebviewWindow) -> Result<tauri::State<'_, computer_use::Controller>, String> {+ if !pet::is_workbench(window) {+ return Err("This command is only available to the Avibe Workbench.".to_owned());+ }+ window+ .try_state::<computer_use::Controller>()+ .ok_or_else(|| "Computer Use is unavailable.".to_owned())+}+/// Runs the bootstrap state machine once, unless one is already running.////// The return value is part of the retry contract: the bootstrap page must not@@ -1495,7 +1545,9 @@ fn ensure_main_window(app: &AppHandle) -> Option<WebviewWindow> {.on_new_window(|url, _features| handle_new_window_request(url));// macOS draws the page under a transparent native title bar (`tauri.conf.json`).#[cfg(target_os = "macos")]- let builder = builder.initialization_script(macos_title_bar::inset_script());+ let builder = builder+ .initialization_script(macos_title_bar::inset_script())+ .initialization_script(DESKTOP_COMPUTER_USE_MARKER);let window = builder.build().ok()?;#[cfg(target_os = "macos")]{@@ -1511,6 +1563,13 @@ fn ensure_main_window(app: &AppHandle) -> Option<WebviewWindow> {const DESKTOP_SHELL_MARKER: &str ="if (window.self === window.top) Object.defineProperty(window, '__AVIBE_DESKTOP_SHELL__', { value: true });";+/// Tells the Workbench that this shell answers the Computer Use guide's+/// actions (`computer_use_enable`, `computer_use_open_permission`). Read by+/// `ui/src/lib/desktopShell.ts`; a shell without it leaves the guide read-only.+#[cfg(target_os = "macos")]+const DESKTOP_COMPUTER_USE_MARKER: &str =+ "if (window.self === window.top) Object.defineProperty(window, '__AVIBE_DESKTOP_COMPUTER_USE__', { value: true });";+/// What happens to a browsing context the page asks for.#[derive(Debug, PartialEq, Eq)]enum NewWindowDecision {@@ -2134,6 +2193,8 @@ pub fn run() {bootstrap_status,bootstrap_retry,open_install_docs,+ computer_use_enable,+ computer_use_open_permission,pet::pet_ready,pet::pet_set_expanded,pet::pet_hit_regions,@@ -2181,6 +2242,11 @@ pub fn run() {#[cfg(target_os = "macos")]macos_terminate::install(app.handle());if let Ok(mut links) = app.state::<Mutex<DeepLinks>>().lock() {+ // A link the launch carries still wins: it is received last.+ #[cfg(target_os = "macos")]+ if computer_use::take_setup_resume(app.handle()) {+ links.receive([COMPUTER_USE_SETUP_DEEP_LINK]);+ }links.receive(std::env::args_os().skip(1)@@ -3363,4 +3429,22 @@ mod tests {);}}++ #[cfg(target_os = "macos")]+ #[test]+ fn the_computer_use_guide_marker_and_resume_link_match_their_readers() {+ assert!(DESKTOP_COMPUTER_USE_MARKER.starts_with("if (window.self === window.top)"));+ assert!(DESKTOP_COMPUTER_USE_MARKER.contains("'__AVIBE_DESKTOP_COMPUTER_USE__'"));+ let ui = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../../ui/src");+ let shell = std::fs::read_to_string(ui.join("lib/desktopShell.ts")).unwrap();+ assert!(shell.contains("window.__AVIBE_DESKTOP_COMPUTER_USE__ !== true"));+ for command in ["'computer_use_enable'", "'computer_use_open_permission'"] {+ assert!(shell.contains(command), "the Workbench must invoke {command}");+ }+ let target = avibe_runtime_host::deep_link::parse_deep_link(COMPUTER_USE_SETUP_DEEP_LINK).unwrap();+ assert_eq!(target.path(), "/settings/computer-use");+ assert!(std::fs::read_to_string(ui.join("lib/applicationRoutes.ts"))+ .unwrap()+ .contains("'/settings/computer-use'"));+ }}diff --git a/desktop/src-tauri/src/pet.rs b/desktop/src-tauri/src/pet.rsindex d05aae899..31f5e7449 100644--- a/desktop/src-tauri/src/pet.rs+++ b/desktop/src-tauri/src/pet.rs@@ -584,7 +584,7 @@ fn is_pet_caller(label: &str, built_for: Option<&LoopbackOrigin>, url: &Url) ->}/// The Workbench in `main`, on the Runtime the shell navigated it to.-fn is_workbench(window: &WebviewWindow) -> bool {+pub(crate) fn is_workbench(window: &WebviewWindow) -> bool {let Some(shell) = window.try_state::<Shell>() else {return false;};diff --git a/desktop/src-tauri/tauri.conf.json b/desktop/src-tauri/tauri.conf.jsonindex c85a978f7..ad8541713 100644--- a/desktop/src-tauri/tauri.conf.json+++ b/desktop/src-tauri/tauri.conf.json@@ -33,7 +33,8 @@"bootstrap","window-drag","pet",- "main-pet-bind"+ "main-pet-bind",+ "main-computer-use"],"csp": "default-src 'self'; img-src 'self' data:; style-src 'self' 'unsafe-inline'; connect-src 'self' ipc: http://ipc.localhost"}diff --git a/desktop/src-tauri/tests/shell_boundaries.rs b/desktop/src-tauri/tests/shell_boundaries.rsindex 977d13c8e..90633c89d 100644--- a/desktop/src-tauri/tests/shell_boundaries.rs+++ b/desktop/src-tauri/tests/shell_boundaries.rs@@ -100,10 +100,10 @@ fn the_bootstrap_capability_grants_only_the_three_shell_commands() {}#[test]-fn the_shell_enables_only_its_four_capabilities() {+fn the_shell_enables_only_its_five_capabilities() {assert_eq!(config()["app"]["security"]["capabilities"],- serde_json::json!(["bootstrap", "window-drag", "pet", "main-pet-bind"])+ serde_json::json!(["bootstrap", "window-drag", "pet", "main-pet-bind", "main-computer-use"]));}@@ -400,6 +400,8 @@ fn every_shell_command_is_declared_so_its_permission_exists() {"bootstrap_status","bootstrap_retry","open_install_docs",+ "computer_use_enable",+ "computer_use_open_permission","pet_ready","pet_set_expanded","pet_hit_regions",@@ -763,7 +765,8 @@ fn native_lifecycle_controls_never_add_a_webview_permission() {] {assert!(source.contains(required), "native lifecycle wiring missing {required}");}- assert_eq!(source.matches("#[tauri::command]").count(), 3);+ // The three bootstrap commands and the two Computer Use guide actions.+ assert_eq!(source.matches("#[tauri::command]").count(), 5);for permission in capability()["permissions"].as_array().expect("capability permissions") {let permission = permission.as_str().expect("string permission");assert!(!permission.contains("autostart") && !permission.contains("tray") && !permission.contains("dialog"));@@ -1083,6 +1086,53 @@ fn loopback_patterns(grant: &Value) -> Vec<tauri::utils::acl::RemoteUrlPattern>.collect()}+#[test]+fn the_workbench_gets_the_computer_use_guide_actions_only_on_macos_loopback_pages() {+ let grant = read_json(crate_dir().join("capabilities/main-computer-use.json"));+ assert_eq!(grant["windows"], serde_json::json!([MAIN_WINDOW]));+ assert_eq!(grant["local"], false, "the bundled bootstrap page never gets them");+ assert_eq!(grant["platforms"], serde_json::json!(["macOS"]));+ assert_eq!(+ grant["permissions"],+ serde_json::json!(["allow-computer-use-enable", "allow-computer-use-open-permission"])+ );+ let patterns = loopback_patterns(&grant);+ for allowed in ["http://127.0.0.1:5123/settings/computer-use", "http://[::1]:6174/"] {+ let url = url::Url::parse(allowed).unwrap();+ assert!(patterns.iter().any(|pattern| pattern.test(&url)), "{url}");+ }+ for rejected in [+ "https://example.com/settings/computer-use",+ "http://127.0.0.1.example.com/",+ "http://192.168.1.2:5123/settings/computer-use",+ ] {+ let url = url::Url::parse(rejected).unwrap();+ assert!(!patterns.iter().any(|pattern| pattern.test(&url)), "{url}");+ }+ for entry in std::fs::read_dir(crate_dir().join("capabilities")).unwrap() {+ let path = entry.unwrap().path();+ if path.file_name().is_some_and(|name| name != "main-computer-use.json") {+ assert!(+ !read_to_string(&path).contains("allow-computer-use-"),+ "{} must not grant Computer Use commands",+ path.display()+ );+ }+ }+ // The page names a pane; the shell owns the URLs and re-checks the caller.+ let source = shipping_source("src/lib.rs");+ let command = source+ .split("fn computer_use_open_permission(")+ .nth(1)+ .unwrap()+ .split("\n}\n")+ .next()+ .unwrap();+ assert!(command.contains("PermissionPane::parse(&pane)"));+ assert!(!command.contains("open_url"));+ assert!(source.contains("if !pet::is_workbench(window)"));+}+#[test]fn the_pet_and_the_workbench_get_only_their_pet_commands_on_loopback_pages() {let pet = read_json(crate_dir().join("capabilities/pet.json"));diff --git a/docs/desktop-computer-use.md b/docs/desktop-computer-use.mdindex 3f9d15ceb..582f5beb1 100644--- a/docs/desktop-computer-use.md+++ b/docs/desktop-computer-use.md@@ -25,24 +25,44 @@ Avibe desktop shell is running.### Settings guideIn the desktop app, **Settings → System → Computer Use** walks through the same-setup as four numbered steps: turn on Computer Use, allow Accessibility, allow-Screen & System Audio Recording, then come back and recheck. Each step is-marked done, current, or not yet done from the live Computer Use status, so the-page shows which permission is still missing. The page refreshes every five-seconds, when the window becomes visible or focused, and when you click-**Recheck**. Accessibility comes first because the managed driver checks it-before Screen Recording.--The Settings entry appears only in the Avibe desktop app. A browser tab or phone-that opens `/settings/computer-use` directly sees a short notice that the page-is available in the desktop app, without the steps.--Avibe requests permissions only when you turn the feature on. It does not keep-showing permission prompts in the background. If the menu still reports that a-permission is needed, open **System Settings → Privacy & Security**, confirm-that Avibe is enabled in both permission panes, then use the off/on or-quit/reopen recovery above. Merely bringing the Avibe window to the front does-not start a permission retry.+setup as three numbered steps: turn on Computer Use, allow Accessibility, and+allow Screen & System Audio Recording. Each step is marked done, current, or+not yet done from the live Computer Use status, and the current step carries+its action:++- **Turn on** turns Computer Use on, exactly like the tray or application menu+ switch. It never turns Computer Use off.+- Accessibility's **Open Settings** asks macOS for the Accessibility grant+ (the system prompt lists Avibe in the pane) and opens **Privacy & Security →+ Accessibility**.+- Screen & System Audio Recording's **Open Settings** runs the same request the+ tray switch does, which lists Avibe in the pane where macOS allows it, opens+ **Privacy & Security → Screen & System Audio Recording**, and shows the+ **+** / drag guidance.++While the page is visible it reads the status about every 1.5 seconds, so a+step turns done within seconds of macOS reporting the grant; it stops reading+while the page is hidden and reads again when it returns. **Recheck** remains+as a manual fallback. Accessibility comes first because the managed driver+checks it before Screen Recording.++If macOS asks to **Quit & Reopen** after the Screen Recording grant, choose it.+When the request came from this page, the relaunched app opens the Settings+guide again by itself (within 15 minutes of the request), and the page picks up+the new status. A later launch is not redirected.++The Settings entry appears only in the Avibe desktop app. A browser tab or+phone that opens `/settings/computer-use` directly sees the same live steps+with a notice to complete them in the Avibe desktop app on the Mac; the+actions cannot run from a browser. When the host has no live desktop shell, it+sees a short notice that the page is available in the desktop app.++Avibe requests permissions only when you turn the feature on or press a+step's **Open Settings**. It does not keep showing permission prompts in the+background. If the menu still reports that a permission is needed, open+**System Settings → Privacy & Security**, confirm that Avibe is enabled in both+permission panes, then use the off/on or quit/reopen recovery above. Merely+bringing the Avibe window to the front does not start a permission retry.The native Computer Use control remains available if its optional state orstartup initialization cannot be opened. Avibe reports the failure in thediff --git a/docs/plans/desktop-deep-links.md b/docs/plans/desktop-deep-links.mdindex dc501fe8f..32573c5d5 100644--- a/docs/plans/desktop-deep-links.md+++ b/docs/plans/desktop-deep-links.md@@ -40,6 +40,7 @@ spelling (`avibe:session/...`); one form only.| `avibe://session/<session_id>` | `/chat/<session_id>` || `avibe://show/<session_id>` | `/apps/show/<session_id>` || `avibe://settings` | `/settings/general` |+| `avibe://settings/computer-use` | `/settings/computer-use` || `avibe://vaults/request/<request_id>` | `/vaults?request_id=<request_id>` |Parse as a URL. Accept only when `scheme == avibe`, `host` is one ofdiff --git a/ui/src/components/settings/SettingsComputerUsePage.test.tsx b/ui/src/components/settings/SettingsComputerUsePage.test.tsxindex 0770daf49..90bea5b2a 100644--- a/ui/src/components/settings/SettingsComputerUsePage.test.tsx+++ b/ui/src/components/settings/SettingsComputerUsePage.test.tsx@@ -1,6 +1,6 @@/* @vitest-environment jsdom */-import { cleanup, render, screen, waitFor } from '@testing-library/react';+import { act, cleanup, fireEvent, render, screen, waitFor } from '@testing-library/react';import userEvent from '@testing-library/user-event';import { afterEach, describe, expect, it, vi } from 'vitest';@@ -24,6 +24,25 @@ const statusResponse = (body: Record<string, unknown>) => ({ ok: true, json: asyconst enterDesktopShell = () =>Object.defineProperty(window, '__AVIBE_DESKTOP_SHELL__', { value: true, configurable: true });+// A shell that answers the guide's actions, recording every native call.+const enterActionableShell = (invoke = vi.fn(async (_command: string, _args?: Record<string, unknown>) => undefined)) => {+ enterDesktopShell();+ Object.defineProperty(window, '__AVIBE_DESKTOP_COMPUTER_USE__', { value: true, configurable: true });+ Object.defineProperty(window, '__TAURI_INTERNALS__', { value: { invoke }, configurable: true });+ return invoke;+};++const setVisibility = (state: DocumentVisibilityState) => {+ Object.defineProperty(document, 'visibilityState', { value: state, configurable: true });+ document.dispatchEvent(new Event('visibilitychange'));+};++// Answers each status read with the next body, then keeps the last one.+const statusSequence = (...bodies: Record<string, unknown>[]) => {+ let index = 0;+ apiFetch.mockImplementation(async () => statusResponse(bodies[Math.min(index++, bodies.length - 1)]));+};+const stepStates = () => Object.fromEntries(screen.getAllByRole('listitem').map((item) => [item.getAttribute('data-step'),@@ -33,8 +52,12 @@ const stepStates = () => Object.fromEntries(afterEach(() => {cleanup();+ vi.useRealTimers();apiFetch.mockReset();Reflect.deleteProperty(window, '__AVIBE_DESKTOP_SHELL__');+ Reflect.deleteProperty(window, '__AVIBE_DESKTOP_COMPUTER_USE__');+ Reflect.deleteProperty(window, '__TAURI_INTERNALS__');+ Reflect.deleteProperty(document, 'visibilityState');});describe('computerUseGuideProgress', () => {@@ -42,23 +65,21 @@ describe('computerUseGuideProgress', () => {COMPUTER_USE_GUIDE_STEPS.map((step) => computerUseGuideProgress({ status, reason })[step]);it('advances the steps only as far as the status proves', () => {- expect(progress('off', 'never_enabled')).toEqual(['current', 'pending', 'pending', 'pending']);- expect(progress('needs_permission', 'accessibility'))- .toEqual(['complete', 'current', 'pending', 'pending']);+ expect(progress('off', 'never_enabled')).toEqual(['current', 'pending', 'pending']);+ expect(progress('needs_permission', 'accessibility')).toEqual(['complete', 'current', 'pending']);// The driver checks Accessibility first, so a Screen Recording gap proves it passed.- expect(progress('needs_permission', 'screen_recording'))- .toEqual(['complete', 'complete', 'current', 'pending']);- expect(progress('needs_permission', null)).toEqual(['complete', 'current', 'pending', 'pending']);- expect(progress('ready')).toEqual(['complete', 'complete', 'complete', 'complete']);+ expect(progress('needs_permission', 'screen_recording')).toEqual(['complete', 'complete', 'current']);+ expect(progress('needs_permission', null)).toEqual(['complete', 'current', 'pending']);+ expect(progress('ready')).toEqual(['complete', 'complete', 'complete']);});it('claims no permission progress for states that happen before or beside the permission checks', () => {for (const status of ['starting', 'needs_runtime', 'error']) {- expect(progress(status, 'any')).toEqual(['complete', 'pending', 'pending', 'pending']);+ expect(progress(status, 'any')).toEqual(['complete', 'pending', 'pending']);}- expect(progress('unavailable', 'shell_not_running')).toEqual(['pending', 'pending', 'pending', 'pending']);+ expect(progress('unavailable', 'shell_not_running')).toEqual(['pending', 'pending', 'pending']);// An unknown state reads as off, matching the status copy's fallback.- expect(progress('future_state')).toEqual(['current', 'pending', 'pending', 'pending']);+ expect(progress('future_state')).toEqual(['current', 'pending', 'pending']);});});@@ -83,7 +104,6 @@ describe('SettingsComputerUsePage', () => {enable: 'complete',accessibility: 'complete',screenRecording: 'current',- recheck: 'pending',});expect(screen.getAllByRole('listitem').find((item) => item.getAttribute('aria-current') === 'step')?.getAttribute('data-step')).toBe('screenRecording');@@ -106,19 +126,196 @@ describe('SettingsComputerUsePage', () => {apiFetch.mockResolvedValue(statusResponse({ supported: true, status: 'ready', reason: null }));await user.click(screen.getByRole('button', { name: /settings\.computerUse\.recheck/ }));- await waitFor(() => expect(stepStates().recheck).toBe('complete'));+ await waitFor(() => expect(stepStates().screenRecording).toBe('complete'));expect(apiFetch.mock.calls.length).toBeGreaterThan(callsBefore);- expect(Object.values(stepStates())).toEqual(['complete', 'complete', 'complete', 'complete']);+ expect(Object.values(stepStates())).toEqual(['complete', 'complete', 'complete']);});- it('shows only the desktop-app notice without the shell, and never reads the status', () => {+ it('gives each current step its native action', async () => {+ const invoke = enterActionableShell();+ statusSequence({ supported: true, status: 'off', reason: 'never_enabled' });+ const user = userEvent.setup();++ const { unmount } = render(<SettingsComputerUsePage />);+ await user.click(await screen.findByTestId('computer-use-action-enable'));+ expect(invoke).toHaveBeenLastCalledWith('computer_use_enable');+ // Only the current step acts; later steps wait for the status to reach them.+ expect(screen.queryByTestId('computer-use-action-accessibility')).toBeNull();+ unmount();++ statusSequence({ supported: true, status: 'needs_permission', reason: 'accessibility' });+ const second = render(<SettingsComputerUsePage />);+ await user.click(await screen.findByTestId('computer-use-action-accessibility'));+ expect(invoke).toHaveBeenLastCalledWith('computer_use_open_permission', { pane: 'accessibility' });+ expect(screen.queryByTestId('computer-use-action-enable')).toBeNull();+ second.unmount();++ statusSequence({ supported: true, status: 'needs_permission', reason: 'screen_recording' });render(<SettingsComputerUsePage />);+ await user.click(await screen.findByTestId('computer-use-action-screenRecording'));+ expect(invoke).toHaveBeenLastCalledWith('computer_use_open_permission', { pane: 'screen_recording' });+ });- expect(screen.getByTestId('computer-use-unsupported')).toBeTruthy();- expect(screen.getByText('settings.computerUse.unsupportedTitle')).toBeTruthy();- expect(screen.queryByTestId('computer-use-steps')).toBeNull();- expect(screen.queryByRole('button', { name: /settings\.computerUse\.recheck/ })).toBeNull();- expect(apiFetch).not.toHaveBeenCalled();+ it('flips each step to done as the polled status advances, without a recheck', async () => {+ enterActionableShell();+ statusSequence(+ { supported: true, status: 'off', reason: 'never_enabled' },+ { supported: true, status: 'needs_permission', reason: 'accessibility' },+ { supported: true, status: 'needs_permission', reason: 'screen_recording' },+ { supported: true, status: 'ready', reason: null },+ );+ vi.useFakeTimers();++ render(<SettingsComputerUsePage />);+ await act(async () => { await vi.advanceTimersByTimeAsync(0); });+ expect(stepStates()).toEqual({ enable: 'current', accessibility: 'pending', screenRecording: 'pending' });++ const expected = [+ { enable: 'complete', accessibility: 'current', screenRecording: 'pending' },+ { enable: 'complete', accessibility: 'complete', screenRecording: 'current' },+ { enable: 'complete', accessibility: 'complete', screenRecording: 'complete' },+ ];+ for (const states of expected) {+ await act(async () => { await vi.advanceTimersByTimeAsync(2_000); });+ expect(stepStates()).toEqual(states);+ }+ });++ it('converges when every read is slower than the refresh interval, with one read at a time', async () => {+ enterActionableShell();+ const bodies = [+ { supported: true, status: 'off', reason: 'never_enabled' },+ { supported: true, status: 'needs_permission', reason: 'accessibility' },+ { supported: true, status: 'needs_permission', reason: 'screen_recording' },+ { supported: true, status: 'ready', reason: null },+ ];+ let index = 0;+ let open = 0;+ let maxOpen = 0;+ // `ready` can wait up to 2 s on the Controller: make every read take 2 s.+ apiFetch.mockImplementation(() => {+ open += 1;+ maxOpen = Math.max(maxOpen, open);+ const body = bodies[Math.min(index++, bodies.length - 1)];+ return new Promise((resolve) => {+ window.setTimeout(() => {+ open -= 1;+ resolve(statusResponse(body));+ }, 2_000);+ });+ });+ vi.useFakeTimers();++ render(<SettingsComputerUsePage />);+ await act(async () => { await vi.advanceTimersByTimeAsync(2_000); });+ expect(stepStates().enable).toBe('current');+ // Rechecking and focusing mid-read join the read already in flight.+ await act(async () => { await vi.advanceTimersByTimeAsync(2_500); });+ act(() => {+ fireEvent.click(screen.getByRole('button', { name: /settings\.computerUse\.recheck/ }));+ window.dispatchEvent(new Event('focus'));+ });++ await act(async () => { await vi.advanceTimersByTimeAsync(20_000); });+ expect(Object.values(stepStates())).toEqual(['complete', 'complete', 'complete']);+ expect(screen.getByTestId('computer-use-guide-status').getAttribute('data-computer-use-status')).toBe('ready');+ expect(maxOpen).toBe(1);+ });++ it('stops polling while the page is hidden and reads again when it returns', async () => {+ enterActionableShell();+ statusSequence({ supported: true, status: 'needs_permission', reason: 'accessibility' });+ vi.useFakeTimers();++ render(<SettingsComputerUsePage />);+ await act(async () => { await vi.advanceTimersByTimeAsync(3_000); });+ expect(apiFetch.mock.calls.length).toBeGreaterThanOrEqual(2);++ act(() => setVisibility('hidden'));+ const hiddenAt = apiFetch.mock.calls.length;+ await act(async () => { await vi.advanceTimersByTimeAsync(30_000); });+ expect(apiFetch.mock.calls.length).toBe(hiddenAt);++ act(() => setVisibility('visible'));+ await act(async () => { await vi.advanceTimersByTimeAsync(0); });+ expect(apiFetch.mock.calls.length).toBe(hiddenAt + 1);+ });++ it('explains the Quit & Reopen relaunch once Screen Recording is requested, until it is granted', async () => {+ enterActionableShell();+ statusSequence({ supported: true, status: 'needs_permission', reason: 'screen_recording' });+ const user = userEvent.setup();++ render(<SettingsComputerUsePage />);+ const action = await screen.findByTestId('computer-use-action-screenRecording');+ expect(screen.queryByTestId('computer-use-relaunch-notice')).toBeNull();++ await user.click(action);+ expect((await screen.findByTestId('computer-use-relaunch-notice')).textContent)+ .toContain('settings.computerUse.relaunchNotice');++ statusSequence({ supported: true, status: 'ready', reason: null });+ await user.click(screen.getByRole('button', { name: /settings\.computerUse\.recheck/ }));+ await waitFor(() => expect(stepStates().screenRecording).toBe('complete'));+ expect(screen.queryByTestId('computer-use-relaunch-notice')).toBeNull();+ });++ it('reports a failed native action without leaving the step', async () => {+ enterActionableShell(vi.fn(async () => { throw new Error('denied'); }));+ statusSequence({ supported: true, status: 'needs_permission', reason: 'accessibility' });+ const user = userEvent.setup();++ render(<SettingsComputerUsePage />);+ await user.click(await screen.findByTestId('computer-use-action-accessibility'));++ expect(await screen.findByTestId('computer-use-action-failed')).toBeTruthy();+ expect(stepStates().accessibility).toBe('current');+ });++ it('keeps a failed action on its own step once the guide moves on', async () => {+ enterActionableShell(vi.fn(async () => { throw new Error('denied'); }));+ statusSequence({ supported: true, status: 'needs_permission', reason: 'accessibility' });+ const user = userEvent.setup();++ render(<SettingsComputerUsePage />);+ await user.click(await screen.findByTestId('computer-use-action-accessibility'));+ expect(await screen.findByTestId('computer-use-action-failed')).toBeTruthy();++ // The user grants Accessibility another way; the next step never failed.+ statusSequence({ supported: true, status: 'needs_permission', reason: 'screen_recording' });+ await user.click(screen.getByRole('button', { name: /settings\.computerUse\.recheck/ }));+ await waitFor(() => expect(stepStates().screenRecording).toBe('current'));+ expect(screen.getByTestId('computer-use-action-screenRecording')).toBeTruthy();+ expect(screen.queryByTestId('computer-use-action-failed')).toBeNull();+ });++ it('follows the live status read-only in a browser, pointing to the desktop app on the Mac', async () => {+ statusSequence(+ { supported: true, status: 'needs_permission', reason: 'accessibility' },+ { supported: true, status: 'needs_permission', reason: 'screen_recording' },+ );+ const user = userEvent.setup();++ render(<SettingsComputerUsePage />);++ expect(screen.getByTestId('computer-use-remote-notice').textContent)+ .toContain('settings.computerUse.remoteNotice');+ await waitFor(() => expect(stepStates().accessibility).toBe('current'));+ expect(screen.queryByTestId(/computer-use-action-/)).toBeNull();+ await user.click(screen.getByRole('button', { name: /settings\.computerUse\.recheck/ }));+ await waitFor(() => expect(stepStates().screenRecording).toBe('current'));+ expect(screen.queryByTestId(/computer-use-action-/)).toBeNull();+ });++ it('keeps a shell without the guide actions read-only instead of offering dead buttons', async () => {+ enterDesktopShell();+ statusSequence({ supported: true, status: 'off', reason: 'never_enabled' });++ render(<SettingsComputerUsePage />);++ await waitFor(() => expect(stepStates().enable).toBe('current'));+ expect(screen.queryByTestId(/computer-use-action-/)).toBeNull();+ expect(screen.queryByTestId('computer-use-remote-notice')).toBeNull();});it('shows the same notice when the host reports Computer Use unsupported', async () => {@@ -135,6 +332,15 @@ describe('SettingsComputerUsePage', () => {expect(screen.queryByTestId('computer-use-steps')).toBeNull();});+ it('shows the desktop-app notice in a browser when the host has no desktop shell', async () => {+ statusSequence({ supported: false, status: 'unavailable', reason: 'unsupported_host' });++ render(<SettingsComputerUsePage />);++ expect(await screen.findByTestId('computer-use-unsupported')).toBeTruthy();+ expect(screen.queryByTestId('computer-use-steps')).toBeNull();+ });+it('ships the page copy in both locales', () => {expect(en.settings.sections.computerUse).toBe('Computer Use');expect(zh.settings.sections.computerUse).toBe('电脑操作');@@ -148,5 +354,14 @@ describe('SettingsComputerUsePage', () => {}expect(zh.settings.computerUse.steps.screenRecording.detail).toContain('屏幕与系统音频录制');expect(en.settings.computerUse.steps.screenRecording.detail).toContain('Screen & System Audio Recording');+ for (const locale of [en, zh]) {+ const copy = locale.settings.computerUse;+ for (const text of [copy.remoteNotice, copy.relaunchNotice, copy.actionFailed, copy.actions.enable, copy.actions.openSettings]) {+ expect(text).toBeTruthy();+ }+ }+ expect(zh.settings.computerUse.remoteNotice).toContain('请在这台 Mac 上的 Avibe 桌面应用中操作');+ expect(zh.settings.computerUse.relaunchNotice).toContain('Avibe 将重新打开,完成后本页会自动继续');+ expect(zh.settings.computerUse.actions).toEqual({ enable: '开启', openSettings: '去设置' });});});diff --git a/ui/src/components/settings/SettingsComputerUsePage.tsx b/ui/src/components/settings/SettingsComputerUsePage.tsxindex 9bdf69afb..007bc6a4a 100644--- a/ui/src/components/settings/SettingsComputerUsePage.tsx+++ b/ui/src/components/settings/SettingsComputerUsePage.tsx@@ -1,16 +1,17 @@-import { useCallback, useEffect, useRef, useState } from 'react';-import type { FC } from 'react';-import { Check, LoaderCircle, Monitor, RotateCw } from 'lucide-react';+import { useCallback, useEffect, useMemo, useRef, useState } from 'react';+import type { FC, ReactNode } from 'react';+import { Check, ExternalLink, Info, LoaderCircle, Monitor, Power, RotateCw } from 'lucide-react';import { useTranslation } from 'react-i18next';import clsx from 'clsx';+import { Badge } from '@/components/ui/badge';import { Button } from '@/components/ui/button';-import { isDesktopShell } from '@/lib/desktopShell';+import { computerUseShellActions, isDesktopShell, type ComputerUsePermissionPane } from '@/lib/desktopShell';import type { TranslationKey } from '@/i18n/types';import { ComputerUseStatusIcon } from '../workbench/ComputerUseStatusLine';import {+ COMPUTER_USE_GUIDE_REFRESH_INTERVAL_MS,COMPUTER_USE_GUIDE_STEPS,- COMPUTER_USE_REFRESH_INTERVAL_MS,computerUseGuideProgress,computerUseToneClass,readComputerUseStatus,@@ -35,10 +36,12 @@ const STEP_COPY: Record<ComputerUseGuideStep, { title: TranslationKey; detail: Ttitle: 'settings.computerUse.steps.screenRecording.title',detail: 'settings.computerUse.steps.screenRecording.detail',},- recheck: {- title: 'settings.computerUse.steps.recheck.title',- detail: 'settings.computerUse.steps.recheck.detail',- },+};++/** The privacy pane each permission step opens; `enable` turns the feature on instead. */+const STEP_PANE: Partial<Record<ComputerUseGuideStep, ComputerUsePermissionPane>> = {+ accessibility: 'accessibility',+ screenRecording: 'screen_recording',};const STEP_STATE_LABEL: Record<ComputerUseGuideStepState, TranslationKey> = {@@ -61,68 +64,128 @@ const StepMarker: FC<{ index: number; state: ComputerUseGuideStepState }> = ({ i</span>);+const GuideNote: FC<{ testId: string; tone?: 'muted' | 'destructive'; children: ReactNode }> = ({+ testId,+ tone = 'muted',+ children,+}) => (+ <p+ data-testid={testId}+ className={clsx(+ 'mt-1.5 flex max-w-2xl items-start gap-1.5 text-[12px] leading-relaxed',+ tone === 'destructive' ? 'text-destructive-ink' : 'text-foreground',+ )}+ >+ <Info className="mt-0.5 size-3.5 shrink-0" aria-hidden="true" />+ <span>{children}</span>+ </p>+);+/*** Settings → Computer Use: the macOS permission setup as one numbered guide- * whose progress is read from the desktop status endpoint. Only the desktop- * shell lists this page; any other host reaching it gets a short notice.+ * whose progress is read from the desktop status endpoint while the page is+ * visible. Inside the desktop shell each current step carries its action; any+ * other host follows the same live status read-only.*/export const SettingsComputerUsePage: FC = () => {const { t } = useTranslation();const desktopShell = isDesktopShell();+ const actions = useMemo(() => computerUseShellActions(), []);const [read, setRead] = useState<ComputerUseStatusRead | null>(null);const [checking, setChecking] = useState(false);- // A poll that started before a Recheck must not overwrite the newer answer.- const latestRequest = useRef(0);-- // Resolves to the read, or `undefined` once a newer read has started.- const readLatest = useCallback(async (): Promise<ComputerUseStatusRead | null | undefined> => {- const request = ++latestRequest.current;- const next = await readComputerUseStatus();- return request === latestRequest.current ? next : undefined;- }, []);+ const [runningStep, setRunningStep] = useState<ComputerUseGuideStep | null>(null);+ const [failedStep, setFailedStep] = useState<ComputerUseGuideStep | null>(null);+ const [screenRecordingRequested, setScreenRecordingRequested] = useState(false);+ // At most one status read is in flight. Every caller shares it rather than+ // starting a newer one: a read is never invalidated, so even reads slower+ // than the refresh interval (`ready` can wait on the Controller) land.+ const inFlight = useRef<Promise<void> | null>(null);+ const mounted = useRef(true);// A transient failure (`null`) keeps the last observed state.- const apply = (next: ComputerUseStatusRead | null | undefined) => {- if (next) setRead(next);- };+ const readStatus = useCallback((): Promise<void> => {+ if (!inFlight.current) {+ inFlight.current = readComputerUseStatus()+ .then((next) => {+ if (mounted.current && next) setRead(next);+ })+ .finally(() => {+ inFlight.current = null;+ });+ }+ return inFlight.current;+ }, []);const recheck = () => {setChecking(true);- void readLatest().then((next) => {- setChecking(false);- apply(next);+ void readStatus().then(() => {+ if (mounted.current) setChecking(false);});};useEffect(() => {- if (!desktopShell) return undefined;- const poll = () => {- void readLatest().then((next) => {- if (next) setRead(next);+ mounted.current = true;+ // One polling chain while the page is visible: the next read is scheduled+ // only after the current one settles. `chained` is true while the chain+ // owns either a pending timer or a read it is waiting for.+ let active = true;+ let chained = false;+ let timer: number | undefined;+ const visible = () => document.visibilityState !== 'hidden';+ const step = () => {+ timer = undefined;+ chained = true;+ void readStatus().then(() => {+ if (!active || !visible()) {+ chained = false;+ return;+ }+ timer = window.setTimeout(step, COMPUTER_USE_GUIDE_REFRESH_INTERVAL_MS);});};- poll();- const interval = window.setInterval(poll, COMPUTER_USE_REFRESH_INTERVAL_MS);- // Granting a permission happens in System Settings; returning to the- // window is the moment the answer is most likely to have changed.const onVisibilityChange = () => {- if (document.visibilityState === 'visible') poll();+ if (visible()) {+ // Returning reads at once, since granting happens in System Settings.+ if (!chained) step();+ } else if (timer !== undefined) {+ window.clearTimeout(timer);+ timer = undefined;+ chained = false;+ }};- window.addEventListener('focus', poll);+ const onFocus = () => void readStatus();+ if (visible()) step();+ else void readStatus();+ window.addEventListener('focus', onFocus);document.addEventListener('visibilitychange', onVisibilityChange);return () => {- // Invalidate a read still in flight so it lands nowhere.- latestRequest.current += 1;- window.clearInterval(interval);- window.removeEventListener('focus', poll);+ active = false;+ mounted.current = false;+ window.clearTimeout(timer);+ window.removeEventListener('focus', onFocus);document.removeEventListener('visibilitychange', onVisibilityChange);};- }, [desktopShell, readLatest]);+ }, [readStatus]);- const unsupported = !desktopShell || read?.kind === 'unsupported';- const status = read?.kind === 'status' ? read.status : null;+ const runStep = (step: ComputerUseGuideStep) => {+ if (!actions) return;+ const pane = STEP_PANE[step];+ setRunningStep(step);+ setFailedStep(null);+ void (pane ? actions.openPermission(pane) : actions.enable())+ .then(+ () => {+ if (step === 'screenRecording') setScreenRecordingRequested(true);+ },+ () => setFailedStep(step),+ )+ .finally(() => {+ setRunningStep(null);+ void readStatus();+ });+ };- if (unsupported) {+ if (read?.kind === 'unsupported') {return (<SettingsPageShellactiveTab="computer-use"@@ -146,6 +209,7 @@ export const SettingsComputerUsePage: FC = () => {);}+ const status = read?.kind === 'status' ? read.status : null;const copy = status ? computerUseStatusCopy(status, (key: ComputerUseCopyKey) => t(key)) : null;const progress = status ? computerUseGuideProgress(status) : null;@@ -157,7 +221,7 @@ export const SettingsComputerUsePage: FC = () => {actions={<Buttontype="button"- variant="secondary"+ variant="ghost"size="xs"disabled={checking}onClick={recheck}@@ -167,6 +231,15 @@ export const SettingsComputerUsePage: FC = () => {</Button>}>+ {!desktopShell && (+ <SettingsPanel variant="preference">+ <div className="flex items-start gap-3" data-testid="computer-use-remote-notice">+ <Monitor className="mt-0.5 size-5 shrink-0 text-muted" aria-hidden="true" />+ <p className="text-[13px] font-medium text-foreground">{t('settings.computerUse.remoteNotice')}</p>+ </div>+ </SettingsPanel>+ )}+<divrole="status"aria-live="polite"@@ -190,6 +263,8 @@ export const SettingsComputerUsePage: FC = () => {<ol className="flex flex-col" data-testid="computer-use-steps">{COMPUTER_USE_GUIDE_STEPS.map((step, index) => {const state = progress?.[step] ?? 'pending';+ const actionable = actions !== null && state === 'current';+ const running = runningStep === step;return (<likey={step}@@ -202,20 +277,53 @@ export const SettingsComputerUsePage: FC = () => {)}><StepMarker index={index} state={state} />- <div className="flex min-w-0 flex-col gap-0.5">+ <div className="flex min-w-0 flex-1 flex-col gap-0.5"><pclassName={clsx(- 'text-[13px] font-medium',+ 'flex items-center gap-2 text-[13px] font-medium',state === 'pending' ? 'text-muted' : 'text-foreground',)}>{t(STEP_COPY[step].title)}+ {state === 'complete' && (+ <Badge variant="success" aria-hidden="true">{t(STEP_STATE_LABEL.complete)}</Badge>+ )}<span className="sr-only">{` — ${t(STEP_STATE_LABEL[state])}`}</span></p><p className="max-w-2xl text-[12px] leading-relaxed text-muted">{t(STEP_COPY[step].detail)}</p>+ {actionable && failedStep === step && (+ <GuideNote testId="computer-use-action-failed" tone="destructive">+ {t('settings.computerUse.actionFailed')}+ </GuideNote>+ )}+ {step === 'screenRecording' && screenRecordingRequested && state !== 'complete' && (+ <GuideNote testId="computer-use-relaunch-notice">+ {t('settings.computerUse.relaunchNotice')}+ </GuideNote>+ )}</div>+ {actionable && (+ <Button+ type="button"+ variant={step === 'enable' ? 'default' : 'secondary'}+ size="xs"+ className="shrink-0"+ disabled={runningStep !== null}+ onClick={() => runStep(step)}+ data-testid={`computer-use-action-${step}`}+ >+ {running+ ? <LoaderCircle className="animate-spin" aria-hidden="true" />+ : step === 'enable'+ ? <Power aria-hidden="true" />+ : <ExternalLink aria-hidden="true" />}+ {step === 'enable'+ ? t('settings.computerUse.actions.enable')+ : t('settings.computerUse.actions.openSettings')}+ </Button>+ )}</li>);})}diff --git a/ui/src/components/workbench/computerUseStatus.ts b/ui/src/components/workbench/computerUseStatus.tsindex 4ab63f042..1a019aef3 100644--- a/ui/src/components/workbench/computerUseStatus.ts+++ b/ui/src/components/workbench/computerUseStatus.ts@@ -4,6 +4,8 @@ import type { ComputerUseStatus, StatusTone } from './computerUseStatusCopy';export const COMPUTER_USE_STATUS_PATH = '/api/desktop/computer-use/status';export const COMPUTER_USE_SETTINGS_PATH = '/settings/computer-use';export const COMPUTER_USE_REFRESH_INTERVAL_MS = 5_000;+/** The setup guide polls faster: a grant should show up as soon as macOS records it. */+export const COMPUTER_USE_GUIDE_REFRESH_INTERVAL_MS = 1_500;/*** One read of the desktop status endpoint.@@ -46,7 +48,6 @@ export const COMPUTER_USE_GUIDE_STEPS = ['enable','accessibility','screenRecording',- 'recheck',] as const;export type ComputerUseGuideStep = (typeof COMPUTER_USE_GUIDE_STEPS)[number];diff --git a/ui/src/i18n/en.json b/ui/src/i18n/en.jsonindex 7496ceb52..e9ae7cefb 100644--- a/ui/src/i18n/en.json+++ b/ui/src/i18n/en.json@@ -520,13 +520,20 @@},"computerUse": {"title": "Computer Use",- "subtitle": "Let Avibe agents see and operate this Mac. Follow the steps in order; this page updates as each one is done.",+ "subtitle": "Let Avibe agents see and operate this Mac. Follow the steps in order; each one is marked done as soon as macOS reports it.","recheck": "Recheck","checking": "Checking…","loading": "Checking Computer Use status…","stepsTitle": "Setup steps","unsupportedTitle": "Available in the Avibe desktop app","unsupportedDetail": "Computer Use is set up from Avibe Desktop on a Mac. Open the desktop app to continue.",+ "remoteNotice": "Complete these steps in the Avibe desktop app on the Mac. This page shows their progress.",+ "actionFailed": "The desktop app did not respond. Try again, or use the Avibe menu bar menu.",+ "relaunchNotice": "If macOS asks you to Quit & Reopen, choose it. Avibe will reopen and continue this page automatically.",+ "actions": {+ "enable": "Turn on",+ "openSettings": "Open Settings"+ },"stepState": {"complete": "done","current": "current step",@@ -535,19 +542,15 @@"steps": {"enable": {"title": "Turn on Computer Use",- "detail": "In the Avibe Desktop menu bar (tray) menu, turn on Computer Use."+ "detail": "Same as the Computer Use switch in the Avibe menu bar menu."},"accessibility": {"title": "Allow Accessibility",- "detail": "In System Settings → Privacy & Security → Accessibility, turn on Avibe. Avibe usually opens this pane for you; if Avibe is not listed, click + and add it."+ "detail": "In System Settings → Privacy & Security → Accessibility, turn on Avibe. If Avibe is not listed, click + and add it."},"screenRecording": {"title": "Allow Screen & System Audio Recording","detail": "In System Settings → Privacy & Security → Screen & System Audio Recording, authorize the running Avibe Desktop app: click +, or drag Avibe into the list, then turn it on."- },- "recheck": {- "title": "Come back and recheck",- "detail": "After you grant Screen Recording, macOS may quit and reopen Avibe. When it reopens, come back here and click Recheck."}}},@@ -5254,8 +5257,8 @@"errorKind": "Failure kind","blockers": "Sources that were unavailable","upstreamNote": "These come from the upstream provider's API. Avibe keeps the status, error code, and the provider's error message with credentials removed, up to 400 characters.",- "upstreamDetail": "Upstream said",- "connectionDetail": "Connection error",+ "upstreamDetail": "Upstream said",+ "connectionDetail": "Connection error","backendDetail": "Agent backend reported","copyRecord": "Copy record","reason": {diff --git a/ui/src/i18n/zh.json b/ui/src/i18n/zh.jsonindex 6c811effd..332320377 100644--- a/ui/src/i18n/zh.json+++ b/ui/src/i18n/zh.json@@ -520,13 +520,20 @@},"computerUse": {"title": "电脑操作",- "subtitle": "让 Avibe Agent 查看并操作这台 Mac。请按顺序完成以下步骤,每完成一步本页都会自动更新。",+ "subtitle": "让 Avibe Agent 查看并操作这台 Mac。请按顺序完成以下步骤,macOS 一确认,对应步骤就会自动标记为完成。","recheck": "重新检查","checking": "检查中…","loading": "正在检查电脑操作状态…","stepsTitle": "设置步骤","unsupportedTitle": "仅桌面端可用","unsupportedDetail": "电脑操作需要在 Mac 上的 Avibe 桌面应用中设置。请打开桌面应用继续。",+ "remoteNotice": "请在这台 Mac 上的 Avibe 桌面应用中操作。本页会同步显示进度。",+ "actionFailed": "桌面应用没有响应。请重试,或使用菜单栏中的 Avibe 菜单。",+ "relaunchNotice": "如果 macOS 提示“退出并重新打开”,请选择它。Avibe 将重新打开,完成后本页会自动继续。",+ "actions": {+ "enable": "开启",+ "openSettings": "去设置"+ },"stepState": {"complete": "已完成","current": "当前步骤",@@ -535,19 +542,15 @@"steps": {"enable": {"title": "开启电脑操作",- "detail": "在菜单栏的 Avibe 桌面应用菜单(托盘)中开启“电脑操作”。"+ "detail": "与菜单栏 Avibe 菜单中的“电脑操作”开关相同。"},"accessibility": {"title": "允许辅助功能",- "detail": "在“系统设置 → 隐私与安全性 → 辅助功能”中开启 Avibe。Avibe 通常会自动打开这个页面;如果列表里没有 Avibe,请点击 + 添加。"+ "detail": "在“系统设置 → 隐私与安全性 → 辅助功能”中开启 Avibe。如果列表里没有 Avibe,请点击 + 添加。"},"screenRecording": {"title": "允许屏幕与系统音频录制","detail": "在“系统设置 → 隐私与安全性 → 屏幕与系统音频录制”中授权正在运行的 Avibe 桌面应用:点击 +,或将 Avibe 拖入列表,然后启用它。"- },- "recheck": {- "title": "返回这里重新检查",- "detail": "授权“屏幕录制”后,macOS 可能会退出并重新打开 Avibe。应用重新打开后,请回到本页点击“重新检查”。"}}},@@ -5254,8 +5257,8 @@"errorKind": "失败类型","blockers": "当时不可用的供应商","upstreamNote": "以上内容均由上游供应商的 API 返回。Avibe 会保留状态码、错误码,以及去除凭据后的上游报错信息(最多 400 字)。",- "upstreamDetail": "上游返回",- "connectionDetail": "连接错误",+ "upstreamDetail": "上游返回",+ "connectionDetail": "连接错误","backendDetail": "Agent 后端报告","copyRecord": "复制记录","reason": {diff --git a/ui/src/lib/desktopShell.ts b/ui/src/lib/desktopShell.tsindex a20c533a4..463c7c1f2 100644--- a/ui/src/lib/desktopShell.ts+++ b/ui/src/lib/desktopShell.ts@@ -7,6 +7,8 @@ declare global {readonly __AVIBE_DESKTOP_DRAG__?: true;/** The shell supports the desktop pet IPC. Conversation selection is main-only. */readonly __AVIBE_DESKTOP_PET__?: true;+ /** The macOS shell answers the Computer Use guide's actions. */+ readonly __AVIBE_DESKTOP_COMPUTER_USE__?: true;}}@@ -44,3 +46,31 @@ export function desktopDragRegion(): '' | undefined {export function canShowInDesktopPet(): boolean {return isDesktopShell() && window.__AVIBE_DESKTOP_PET__ === true;}++type ShellInvoke = (command: string, args?: Record<string, unknown>) => Promise<unknown>;++const computerUseInvoke = (): ShellInvoke | null => {+ if (!isDesktopShell() || window.__AVIBE_DESKTOP_COMPUTER_USE__ !== true) return null;+ const internals = (window as unknown as { __TAURI_INTERNALS__?: { invoke?: ShellInvoke } }).__TAURI_INTERNALS__;+ return internals?.invoke ? internals.invoke.bind(internals) : null;+};++/** A System Settings privacy pane the Computer Use guide can open. */+export type ComputerUsePermissionPane = 'accessibility' | 'screen_recording';++/**+ * The native side of the Computer Use guide, or `null` when this page cannot+ * drive it: a browser tab, a remote browser, or an older shell. The shell+ * re-checks that the caller is its own Workbench.+ */+export function computerUseShellActions(): {+ enable: () => Promise<void>;+ openPermission: (pane: ComputerUsePermissionPane) => Promise<void>;+} | null {+ const invoke = computerUseInvoke();+ if (!invoke) return null;+ return {+ enable: async () => { await invoke('computer_use_enable'); },+ openPermission: async (pane) => { await invoke('computer_use_open_permission', { pane }); },+ };+}