YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
feat(app): per-monitor pause in tray recording menu (#4729)
Divanshueee362e49ac7
diff --git a/TESTING.md b/TESTING.mdindex e1450817e..ac5fbe0c2 100644--- a/TESTING.md+++ b/TESTING.md@@ -140,6 +140,7 @@ commits: `28e5c247`- [ ] **dead System Audio auto-reconnect** — Simulate a dead system audio stream. Verify it auto-reconnects and resumes capture. (`0f287761d`)- [ ] **SCK System Audio dead-display recovery (clamshell)** — Mid-call, close the MacBook lid with an external display attached → remote-participant (System Audio) audio resumes within ~1 min via auto re-anchor, mic unaffected. AND: leave the machine idle (nothing playing) for 10 min with stable displays → NO System Audio rebuild/churn (guards the reverted output recv-timeout `0f287761d` / `357e4dfcc`). Watchdog: `core::sck_output_watchdog` (#3901).- [ ] **per-device audio toggle** — In the tray menu, verify you can toggle recording for individual audio devices. (`3ee3defcb`)+- [ ] **per-monitor tray toggle** — With 2+ displays, open the tray recording section and uncheck one monitor. Verify only that display stops screen capture; re-check resumes it. Pause one display individually, then global pause/resume — the individually paused display stays off. (`4685`)- [ ] **stable audio device order** — Verify that audio devices listed in the tray menu maintain a stable order across refreshes. (`4577ac8a6`)- [ ] **Mic disconnect false-positives on sleep/wake** — Put the computer to sleep and wake it up. Verify that no false-positive mic disconnect notifications or logs are generated. (`796baa619`)diff --git a/apps/screenpipe-app-tauri/src-tauri/src/capture_session.rs b/apps/screenpipe-app-tauri/src-tauri/src/capture_session.rsindex 305a35fde..93fcfed6c 100644--- a/apps/screenpipe-app-tauri/src-tauri/src/capture_session.rs+++ b/apps/screenpipe-app-tauri/src-tauri/src/capture_session.rs@@ -45,6 +45,10 @@ pub struct CaptureSession {ui_recorder_handle: Option<screenpipe_engine::UiRecorderHandle>,audio_manager: Arc<screenpipe_audio::audio_manager::AudioManager>,audio_disabled: bool,+ /// Cleared on stop so `/vision/device/*` stops pointing at a shut-down manager.+ vision_manager_handle: Option<+ Arc<arc_swap::ArcSwap<Option<Arc<screenpipe_engine::vision_manager::VisionManager>>>>,+ >,}impl CaptureSession {@@ -74,6 +78,7 @@ impl CaptureSession {// --- Frame-linker sender (set by VisionManager, consumed by UI recorder + capture loops) ---let mut linker_tx: Option<screenpipe_engine::frame_linker_actor::LinkerSender> = None;let mut vision_task = None;+ let mut vision_manager_handle = None;// --- Vision ---// Gate on screen recording permission before calling any ScreenCaptureKit API.@@ -120,6 +125,14 @@ impl CaptureSession {.with_high_fps_controller(server.high_fps_controller.clone()),);+ // Wire the live manager into AppState so tray/popover `/vision/device/*`+ // calls hit the instance that is actually capturing (mirrors how+ // high_fps_controller is shared across HTTP and capture).+ server+ .vision_manager_handle+ .store(Arc::new(Some(vision_manager.clone())));+ vision_manager_handle = Some(server.vision_manager_handle.clone());+capture_trigger_tx = Some(vision_manager.trigger_sender());linker_tx = Some(vision_manager.linker_sender());@@ -275,6 +288,7 @@ impl CaptureSession {ui_recorder_handle,audio_manager: server.audio_manager.clone(),audio_disabled: config.disable_audio,+ vision_manager_handle,})}@@ -318,6 +332,10 @@ impl CaptureSession {}}+ if let Some(handle) = self.vision_manager_handle.take() {+ handle.store(Arc::new(None));+ }+invalidate_macos_screen_streams("capture session stop").await;// Wait for UI recorder tasks to finishdiff --git a/apps/screenpipe-app-tauri/src-tauri/src/health.rs b/apps/screenpipe-app-tauri/src-tauri/src/health.rsindex 637100fc5..6af627b3b 100644--- a/apps/screenpipe-app-tauri/src-tauri/src/health.rs+++ b/apps/screenpipe-app-tauri/src-tauri/src/health.rs@@ -206,6 +206,9 @@ pub struct DeviceInfo {pub kind: DeviceKind,pub active: bool,pub last_seen_secs_ago: u64,+ /// Numeric monitor id from `/vision/device/status`. Present only when the+ /// sidecar exposes per-display pause; absent on older engines (display-only).+ pub monitor_id: Option<u32>,}/// Full recording info including per-device status@@ -261,6 +264,32 @@ pub fn set_audio_device_status(devices: Vec<AudioDeviceEntry>) {*guard = devices;}+/// Cached vision/monitor device status from `/vision/device/status`.+/// Updated by the health polling loop so the tray can toggle displays without blocking.+#[derive(Clone, Debug)]+pub struct VisionDeviceEntry {+ pub id: u32,+ pub name: String,+ pub user_disabled: bool,+}++static VISION_DEVICE_STATUS: Lazy<RwLock<Vec<VisionDeviceEntry>>> =+ Lazy::new(|| RwLock::new(Vec::new()));++pub fn get_vision_device_status() -> Vec<VisionDeviceEntry> {+ VISION_DEVICE_STATUS+ .read()+ .unwrap_or_else(|e| e.into_inner())+ .clone()+}++pub fn set_vision_device_status(devices: Vec<VisionDeviceEntry>) {+ let mut guard = VISION_DEVICE_STATUS+ .write()+ .unwrap_or_else(|e| e.into_inner());+ *guard = devices;+}+/// Mirror of `routes::capture::HighFpsState` — what the tray needs to/// render the "HD recording" menu item label and checked state./// Updated by the health poll so the tray reads from a local cache@@ -666,6 +695,7 @@ fn parse_devices_from_health(health_result: &Result<HealthCheckResponse>) -> Veckind: DeviceKind::Monitor,active: health.frame_status.as_deref() == Some("ok"),last_seen_secs_ago: 0,+ monitor_id: None,});}}@@ -711,6 +741,7 @@ fn parse_devices_from_health(health_result: &Result<HealthCheckResponse>) -> Veckind,active,last_seen_secs_ago: last_seen,+ monitor_id: None,});}}@@ -873,39 +904,10 @@ pub async fn start_health_check(app: tauri::AppHandle) -> Result<()> {// returns stale results. The app now subscribes to `permission_lost`// / `permission_restored` events via /ws/events (see engine_events.rs).- // Parse device info from health response, filtered by monitor settings+ // Parse device info from health response; monitor allowlist applied+ // after vision status replaces monitor rows (below).let mut devices = parse_devices_from_health(&health_result);- // Filter monitors to only show actively recording ones- if let Ok(Some(store)) = crate::store::SettingsStore::get(&app) {- if !store.recording.use_all_monitors- && !store.recording.monitor_ids.is_empty()- && store.recording.monitor_ids != vec!["default".to_string()]- {- devices.retain(|d| {- if d.kind != DeviceKind::Monitor {- return true;- }- store.recording.monitor_ids.iter().any(|allowed| {- // Stable ID format: "Display 3_1920x1080_0,0"- // Extract name prefix before last '_' (position coords)- let allowed_name = allowed.rsplitn(2, '_').last().unwrap_or(allowed);- // Health monitor format: "Display 3 (1920x1080)"- // Extract just the display name- let health_name = d.name.split(" (").next().unwrap_or(&d.name);- let allowed_short =- allowed_name.split('_').next().unwrap_or(allowed_name);- // Also match numeric monitor IDs from CLI -m flag- // e.g. allowed="3" should match health_name="Display 3"- let numeric_match = health_name- .strip_prefix("Display ")- .map_or(false, |id| id == *allowed);- health_name == allowed_short || numeric_match- })- });- }- }-// Fetch all audio devices (including user-disabled) for tray displaylet api = local_api_context_from_app(&app);if let Ok(res) = api@@ -954,6 +956,7 @@ pub async fn start_health_check(app: tauri::AppHandle) -> Result<()> {kind,active: false,last_seen_secs_ago: 0,+ monitor_id: None,});}}@@ -963,6 +966,81 @@ pub async fn start_health_check(app: tauri::AppHandle) -> Result<()> {}}+ // Per-monitor vision status — replaces health-derived monitor rows when+ // available so the tray can toggle individual displays.+ match api+ .apply_auth(+ reqwest::Client::new().get(api.url("/vision/device/status")),+ )+ .send()+ .await+ {+ Ok(res) if res.status().is_success() => {+ if let Ok(devs) = res.json::<Vec<serde_json::Value>>().await {+ if devs.is_empty() {+ set_vision_device_status(Vec::new());+ } else {+ devices.retain(|d| d.kind != DeviceKind::Monitor);+ let mut vision_entries = Vec::new();+ for d in &devs {+ let id = d["id"].as_u64().unwrap_or(0) as u32;+ let name = d["name"].as_str().unwrap_or("").to_string();+ let user_disabled =+ d["user_disabled"].as_bool().unwrap_or(false);+ vision_entries.push(VisionDeviceEntry {+ id,+ name: name.clone(),+ user_disabled,+ });+ devices.push(DeviceInfo {+ name,+ kind: DeviceKind::Monitor,+ active: !user_disabled,+ last_seen_secs_ago: 0,+ monitor_id: Some(id),+ });+ }+ set_vision_device_status(vision_entries);+ }+ } else {+ set_vision_device_status(Vec::new());+ }+ }+ _ => {+ set_vision_device_status(Vec::new());+ }+ }++ // Filter monitors to only show those selected in recording settings.+ if let Ok(Some(store)) = crate::store::SettingsStore::get(&app) {+ if !store.recording.use_all_monitors+ && !store.recording.monitor_ids.is_empty()+ && store.recording.monitor_ids != vec!["default".to_string()]+ {+ devices.retain(|d| {+ if d.kind != DeviceKind::Monitor {+ return true;+ }+ store.recording.monitor_ids.iter().any(|allowed| {+ // Stable ID format: "Display 3_1920x1080_0,0"+ // Extract name prefix before last '_' (position coords)+ let allowed_name = allowed.rsplitn(2, '_').last().unwrap_or(allowed);+ // Health monitor format: "Display 3 (1920x1080)"+ // Extract just the display name+ let health_name = d.name.split(" (").next().unwrap_or(&d.name);+ let allowed_short =+ allowed_name.split('_').next().unwrap_or(allowed_name);+ // Also match numeric monitor IDs from CLI -m flag+ // e.g. allowed="3" should match health_name="Display 3"+ let numeric_match = health_name+ .strip_prefix("Display ")+ .map_or(false, |id| id == *allowed);+ health_name == allowed_short || numeric_match+ })+ });+ }+ }+// Poll the HD controller state for the tray's menu item.// 503 is normal (vision disabled / older engine) — keep last known.if let Ok(res) = apidiff --git a/apps/screenpipe-app-tauri/src-tauri/src/server_core.rs b/apps/screenpipe-app-tauri/src-tauri/src/server_core.rsindex ecef871ad..a60b87ac1 100644--- a/apps/screenpipe-app-tauri/src-tauri/src/server_core.rs+++ b/apps/screenpipe-app-tauri/src-tauri/src/server_core.rs@@ -41,6 +41,11 @@ pub struct ServerCore {/// toggles. Handed to both the HTTP server and the VisionManager so HTTP/// toggles and the capture loop see the same session state.pub high_fps_controller: Arc<screenpipe_engine::high_fps_controller::HighFpsController>,+ /// Runtime handle to the active VisionManager. CaptureSession registers+ /// its instance on start and clears on stop so `/vision/device/*` routes+ /// hit the manager that is actually capturing.+ pub vision_manager_handle:+ Arc<arc_swap::ArcSwap<Option<Arc<screenpipe_engine::vision_manager::VisionManager>>>>,pub data_dir: PathBuf,pub data_path: PathBuf,pub port: u16,@@ -664,6 +669,8 @@ impl ServerCore {info!("HTTP server bound to port {}", config.port);+ let vision_manager_handle = server.vision_manager.clone();+// Start serving in backgroundtokio::spawn(async move {if let Err(e) = server.start_with_listener(listener).await {@@ -983,6 +990,7 @@ impl ServerCore {pipe_manager: shared_pipe_manager,manual_meeting,high_fps_controller,+ vision_manager_handle,data_dir: local_data_dir,data_path,port: config.port,diff --git a/apps/screenpipe-app-tauri/src-tauri/src/tray.rs b/apps/screenpipe-app-tauri/src-tauri/src/tray.rsindex 6e7ee55e1..4bf50bfb3 100644--- a/apps/screenpipe-app-tauri/src-tauri/src/tray.rs+++ b/apps/screenpipe-app-tauri/src-tauri/src/tray.rs@@ -6,7 +6,7 @@ use crate::commands::{hide_main_window, show_main_window};use crate::enterprise_policy::{is_app_ui_hidden, is_tray_item_hidden};use crate::health::{get_audio_device_status, get_high_fps_status, get_recording_info, get_recording_status,- set_high_fps_status, DeviceKind, HighFpsCacheEntry, RecordingStatus,+ get_vision_device_status, set_high_fps_status, DeviceKind, HighFpsCacheEntry, RecordingStatus,};use crate::process_exit;use crate::recording::{local_api_context_from_app, RecordingState};@@ -799,19 +799,39 @@ fn create_dynamic_menu({let info = get_recording_info();- // Show monitors (non-clickable)- for device in info+ // Monitors: CheckMenuItem when the sidecar reports a numeric id (per-display+ // pause via /vision/device/*). Older sidecars stay display-only.+ let vision_status = get_vision_device_status();+ let mut monitors: Vec<_> = info.devices.iter().filter(|d| d.kind == DeviceKind::Monitor)- {- let dot = if device.active { "●" } else { "○" };- let label = format!(" {} ▣ {}", dot, device.name);- menu_builder = menu_builder.item(- &MenuItemBuilder::with_id(format!("monitor_{}", device.name), label)- .enabled(false)- .build(app)?,- );+ .collect();+ monitors.sort_by(|a, b| a.name.cmp(&b.name));+ for device in monitors {+ let label = format!(" ▣ {}", device.name);+ if let Some(monitor_id) = device.monitor_id {+ let is_active = vision_status+ .iter()+ .find(|d| d.id == monitor_id)+ .map(|d| !d.user_disabled)+ .unwrap_or(device.active);+ let toggle = CheckMenuItemBuilder::with_id(+ format!("toggle_vision_device_{}", monitor_id),+ label,+ )+ .checked(is_active)+ .build(app)?;+ menu_builder = menu_builder.item(&toggle);+ } else {+ let dot = if device.active { "●" } else { "○" };+ let fallback_label = format!(" {} ▣ {}", dot, device.name);+ menu_builder = menu_builder.item(+ &MenuItemBuilder::with_id(format!("monitor_{}", device.name), fallback_label)+ .enabled(false)+ .build(app)?,+ );+ }}// Show only the audio devices from get_recording_info (the ones@@ -1304,6 +1324,36 @@ fn handle_menu_event(app_handle: &AppHandle, event: tauri::menu::MenuEvent) {}});}+ id if id.starts_with("toggle_vision_device_") => {+ let monitor_id: u32 = id+ .strip_prefix("toggle_vision_device_")+ .unwrap()+ .parse()+ .unwrap_or(0);++ let cached = get_vision_device_status();+ let is_active = cached+ .iter()+ .find(|d| d.id == monitor_id)+ .map(|d| !d.user_disabled)+ .unwrap_or(true);++ let api = local_api_context_from_app(&app_handle);+ let endpoint = if is_active {+ api.url("/vision/device/stop")+ } else {+ api.url("/vision/device/start")+ };+ tauri::async_runtime::spawn(async move {+ let client = reqwest::Client::new();+ let _ = api+ .apply_auth(client.post(endpoint))+ .header("Content-Type", "application/json")+ .body(serde_json::json!({"monitor_id": monitor_id}).to_string())+ .send()+ .await;+ });+ }id if id.starts_with("toggle_audio_device_") => {let device_name = id.strip_prefix("toggle_audio_device_").unwrap().to_string();diff --git a/crates/screenpipe-engine/src/bin/screenpipe-engine.rs b/crates/screenpipe-engine/src/bin/screenpipe-engine.rsindex da065714b..7be31c455 100644--- a/crates/screenpipe-engine/src/bin/screenpipe-engine.rs+++ b/crates/screenpipe-engine/src/bin/screenpipe-engine.rs@@ -1282,7 +1282,9 @@ async fn main() -> anyhow::Result<()> {server.timeline_disabled = config.disable_timeline;server.power_manager = Some(power_manager);// Share the VisionManager so /vision/device/* can pause/resume monitors.- server.vision_manager = vision_manager_for_server;+ if let Some(vm) = vision_manager_for_server {+ server.vision_manager.store(Arc::new(Some(vm)));+ }server.manual_meeting = Some(manual_meeting.clone());server.api_auth = config.api_auth;server.api_auth_key = config.api_auth_key.clone();diff --git a/crates/screenpipe-engine/src/routes/vision.rs b/crates/screenpipe-engine/src/routes/vision.rsindex b26edb650..6953c693d 100644--- a/crates/screenpipe-engine/src/routes/vision.rs+++ b/crates/screenpipe-engine/src/routes/vision.rs@@ -49,10 +49,14 @@ pub(crate) struct VisionDeviceStatusEntry {/// Resolve the shared VisionManager or return a 409 when vision capture isn't/// running (vision disabled in settings, or a headless config with no manager).+fn load_vision_manager(state: &Arc<AppState>) -> Option<Arc<crate::vision_manager::VisionManager>> {+ state.vision_manager.load().as_ref().clone()+}+fn require_vision_manager(state: &Arc<AppState>,-) -> Result<&Arc<crate::vision_manager::VisionManager>, (StatusCode, JsonResponse<Value>)> {- state.vision_manager.as_ref().ok_or_else(|| {+) -> Result<Arc<crate::vision_manager::VisionManager>, (StatusCode, JsonResponse<Value>)> {+ load_vision_manager(state).ok_or_else(|| {(StatusCode::CONFLICT,JsonResponse(json!({@@ -71,7 +75,7 @@ pub(crate) async fn start_vision_device(let vision_manager = require_vision_manager(&state)?;let monitor_id = payload.monitor_id;- if let Err(e) = vision_manager.resume_monitor(monitor_id).await {+ if let Err(e) = vision_manager.clone().resume_monitor(monitor_id).await {return Err((StatusCode::INTERNAL_SERVER_ERROR,JsonResponse(json!({@@ -95,7 +99,7 @@ pub(crate) async fn stop_vision_device(let vision_manager = require_vision_manager(&state)?;let monitor_id = payload.monitor_id;- if let Err(e) = vision_manager.pause_monitor(monitor_id).await {+ if let Err(e) = vision_manager.clone().pause_monitor(monitor_id).await {return Err((StatusCode::INTERNAL_SERVER_ERROR,JsonResponse(json!({@@ -115,8 +119,8 @@ pub(crate) async fn stop_vision_device(pub(crate) async fn vision_device_status(State(state): State<Arc<AppState>>,) -> Result<JsonResponse<Vec<VisionDeviceStatusEntry>>, (StatusCode, JsonResponse<Value>)> {- // Vision disabled or no manager wired (headless): nothing to report.- let Some(vision_manager) = state.vision_manager.as_ref() else {+ // Vision disabled or no manager wired (capture stopped): nothing to report.+ let Some(vision_manager) = load_vision_manager(&state) else {return Ok(JsonResponse(Vec::new()));};diff --git a/crates/screenpipe-engine/src/server.rs b/crates/screenpipe-engine/src/server.rsindex 5b6b4a763..8890941ab 100644--- a/crates/screenpipe-engine/src/server.rs+++ b/crates/screenpipe-engine/src/server.rs@@ -211,9 +211,9 @@ pub struct AppState {/// `--disable-vision`).pub high_fps_controller: Option<Arc<crate::high_fps_controller::HighFpsController>>,/// Shared VisionManager so the `/vision/device/*` routes can pause/resume- /// individual monitors from the recording popover. `None` when vision is- /// disabled or on headless configs that run no capture.- pub vision_manager: Option<Arc<crate::vision_manager::VisionManager>>,+ /// individual monitors. Updated at runtime when the desktop app's+ /// CaptureSession starts/stops; set once at boot for the CLI engine.+ pub vision_manager: Arc<ArcSwap<Option<Arc<crate::vision_manager::VisionManager>>>>,}pub struct SCServer {@@ -266,10 +266,10 @@ pub struct SCServer {/// Shared high-FPS controller. Set before `start()` so AppState and/// the per-monitor capture loops point at the same instance.pub high_fps_controller: Option<Arc<crate::high_fps_controller::HighFpsController>>,- /// Shared VisionManager. Set before `start()` so the `/vision/device/*`- /// routes can pause/resume individual monitors. `None` on vision-disabled- /// or headless configs that run no capture.- pub vision_manager: Option<Arc<crate::vision_manager::VisionManager>>,+ /// Handle to the active VisionManager. CaptureSession registers its+ /// instance here on start and clears on stop so `/vision/device/*` hits+ /// the manager that is actually capturing.+ pub vision_manager: Arc<ArcSwap<Option<Arc<crate::vision_manager::VisionManager>>>>,/// When true, the timeline / rewind feature is disabled. The server skips/// warming the hot frame cache from the DB at startup (the cache is only/// read by the timeline streaming endpoint). Set before `start()`.@@ -361,7 +361,7 @@ impl SCServer {oauth_refresher: None,external_memory_sync: None,high_fps_controller: None,- vision_manager: None,+ vision_manager: Arc::new(ArcSwap::from_pointee(None)),timeline_disabled: false,advertise_mdns: should_advertise_mdns(addr),}diff --git a/crates/screenpipe-engine/src/vision_manager/manager.rs b/crates/screenpipe-engine/src/vision_manager/manager.rsindex 4bb84aa93..6993f2ef5 100644--- a/crates/screenpipe-engine/src/vision_manager/manager.rs+++ b/crates/screenpipe-engine/src/vision_manager/manager.rs@@ -681,10 +681,16 @@ impl VisionManager {}/// Resume recording on a monitor the user previously paused. Clears the- /// paused flag first so `start_monitor`'s guard lets it through.+ /// paused flag first so `start_monitor`'s guard lets it through. When the+ /// manager isn't running (global capture paused), records resume intent only.pub async fn resume_monitor(&self, monitor_id: u32) -> Result<()> {self.user_disabled.remove(&monitor_id);info!("user resumed vision recording for monitor {}", monitor_id);++ if self.status().await != VisionManagerStatus::Running {+ return Ok(());+ }+self.start_monitor(monitor_id).await}@@ -818,11 +824,32 @@ mod tests {// Resuming clears the flag (the actual start then fails only because the// fake id has no monitor — the flag clear is what we assert here).- vm.user_disabled.remove(&id);+ vm.resume_monitor(id).await.expect("resume clears intent");assert!(!vm.is_monitor_user_disabled(id));assert!(vm.user_disabled_monitors().is_empty());}+ /// Resuming while the manager is stopped clears user pause intent but does+ /// not spawn capture — mirrors audio `resume_device` when capture is off.+ #[tokio::test]+ async fn resume_while_stopped_clears_intent_without_starting() {+ let vm = make_vm_with_monitor_ids(vec!["default".to_string()]).await;+ let id = 4242;++ vm.pause_monitor(id).await.expect("pause records intent");+ assert!(vm.is_monitor_user_disabled(id));+ assert_eq!(vm.status().await, VisionManagerStatus::Stopped);++ vm.resume_monitor(id)+ .await+ .expect("resume while stopped is Ok");+ assert!(!vm.is_monitor_user_disabled(id));+ assert!(+ !vm.recording_tasks.contains_key(&id),+ "stopped manager must not start capture on resume"+ );+ }+/// Verify that stop_monitor completes promptly when the task finishes normally.#[tokio::test]async fn test_stop_monitor_normal_task() {