YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
feat(app): mic capture-health notifications + meeting piggyback settings toggle
Ezra Ellette2667fffc14fb
diff --git a/apps/screenpipe-app-tauri/components/settings/recording-settings.tsx b/apps/screenpipe-app-tauri/components/settings/recording-settings.tsxindex 5258c15a9..943794d09 100644--- a/apps/screenpipe-app-tauri/components/settings/recording-settings.tsx+++ b/apps/screenpipe-app-tauri/components/settings/recording-settings.tsx@@ -26,6 +26,7 @@ export const searchIndex: SettingsField[] = [// conditional: platform/OS-gated (Windows-only / macOS CoreAudio tap).{ label: "Echo cancellation mode", keywords: ["echo", "aec", "voiceprocessingio", "wasapi"], conditional: true },{ label: "CoreAudio system audio capture", keywords: ["coreaudio", "system audio"], conditional: true },+ { label: "meeting piggyback (experimental)", keywords: ["meeting", "piggyback", "per-process", "meeting audio"], conditional: true },{ label: "Screen context capture", keywords: ["screen", "video", "accessibility"] },{ label: "Screenshot images", keywords: ["screenshot", "pixels", "ocr", "jpeg"] },{ label: "Use all monitors", keywords: ["monitor", "display"], conditional: true },@@ -3549,6 +3550,35 @@ Your screen is a pipe. Everything you see, hear, and type flows through it. Scre</Card>)}+ {/* Meeting piggyback (experimental): during meetings, capture only the+ meeting app's audio and the mic it actually uses. Windows always has+ a piggyback path (WASAPI session capture); macOS needs the CoreAudio+ Process Tap. */}+ {!settings.disableAudio && (isWindows || coreaudioTapAvailable) && (+ <Card className="border-border bg-card">+ <CardContent className="px-3 py-2.5">+ <div className="flex items-center justify-between">+ <div className="flex items-center space-x-2.5">+ <Mic className="h-4 w-4 text-muted-foreground shrink-0" />+ <div>+ <h3 className="text-sm font-medium text-foreground">+ meeting piggyback (experimental)+ </h3>+ <p className="text-xs text-muted-foreground">+ during meetings, capture only the meeting app's audio and the microphone it actually uses. falls back to standard capture automatically if unavailable.+ </p>+ </div>+ </div>+ <Switch+ id="experimentalMeetingPiggyback"+ checked={Boolean(settings.experimentalMeetingPiggyback ?? false)}+ onCheckedChange={(checked) => handleSettingsChange({ experimentalMeetingPiggyback: checked }, true)}+ />+ </div>+ </CardContent>+ </Card>+ )}+{/* Per-app exclusion list for the CoreAudio Process Tap. Onlymeaningful when the tap is the active backend. */}{!settings.disableAudio && coreaudioTapAvailable && settings.experimentalCoreaudioSystemAudio && (diff --git a/apps/screenpipe-app-tauri/lib/hooks/use-settings.tsx b/apps/screenpipe-app-tauri/lib/hooks/use-settings.tsxindex beff1fb15..97cbdedf0 100644--- a/apps/screenpipe-app-tauri/lib/hooks/use-settings.tsx+++ b/apps/screenpipe-app-tauri/lib/hooks/use-settings.tsx@@ -297,6 +297,10 @@ export type Settings = SettingsStore & {/** Experimental: capture System Audio via CoreAudio Process Tap (macOS 14.4+) instead of ScreenCaptureKit.* Off by default. Ignored on macOS <14.4 and non-macOS — falls back to SCK. */experimentalCoreaudioSystemAudio?: boolean;+ /** Experimental: during meetings, capture only the meeting app's audio and the microphone it+ * actually uses (per-process piggyback). Off by default. Falls back to standard capture+ * automatically if unavailable. */+ experimentalMeetingPiggyback?: boolean;/** Experimental: request Windows WASAPI microphone AEC when supported. */windowsInputAecEnabled?: boolean;/** Experimental: request Apple VoiceProcessingIO AEC on the default macOS microphone. */@@ -673,6 +677,7 @@ let DEFAULT_SETTINGS: Settings = {disableClickCapture: false,keepComputerAwake: false,experimentalCoreaudioSystemAudio: false,+ experimentalMeetingPiggyback: false,windowsInputAecEnabled: false,macosInputVpioEnabled: false,screenpipeAecEnabled: false,diff --git a/apps/screenpipe-app-tauri/lib/utils/tauri.ts b/apps/screenpipe-app-tauri/lib/utils/tauri.tsindex a7748734e..6fd227e2c 100644--- a/apps/screenpipe-app-tauri/lib/utils/tauri.ts+++ b/apps/screenpipe-app-tauri/lib/utils/tauri.ts@@ -2718,6 +2718,16 @@ useSystemDefaultAudio: boolean;* falls back to the SCK path automatically. Ignored on non-macOS platforms.*/experimentalCoreaudioSystemAudio?: boolean;+/**+ * Experimental: during detected meetings, capture the meeting app's own+ * audio via a per-process tap plus the microphone that app actually has+ * open (instead of the global mix + assumed-default mic). Default `false`.+ * Requires macOS 14.4+ or Windows; when the platform can't do it or the+ * tap fails at runtime, capture automatically falls back to the stable+ * path (default mic + global system audio) — never less capture than+ * with the flag off.+ */+experimentalMeetingPiggyback?: boolean;/*** Experimental: request Windows WASAPI microphone Acoustic Echo Cancellation.* Ignored on non-Windows platforms and fail-open when unsupported by device/driver.diff --git a/apps/screenpipe-app-tauri/src-tauri/src/engine_events.rs b/apps/screenpipe-app-tauri/src-tauri/src/engine_events.rsindex fb50fb149..6687df588 100644--- a/apps/screenpipe-app-tauri/src-tauri/src/engine_events.rs+++ b/apps/screenpipe-app-tauri/src-tauri/src/engine_events.rs@@ -133,9 +133,11 @@ fn dispatch(app: &AppHandle, text: &str) {| "audio_device_fallback_cleared"| "audio_device_fallback_unavailable"| "audio_device_status_changed" => audio_device::handle(app, name, &data),- "audio_capture_health_speaker_silent" | "audio_capture_health_recovered" => {- audio_health::handle(app, name, &data)- }+ "audio_capture_health_speaker_silent"+ | "audio_capture_health_recovered"+ | "audio_capture_health_mic_silent"+ | "audio_capture_health_mic_recovered"+ | "audio_capture_health_mic_capture_failed" => audio_health::handle(app, name, &data),"power_profile_changed" => power::handle(app, name, &data),_ => { /* unrelated event — ignore */ }}diff --git a/apps/screenpipe-app-tauri/src-tauri/src/engine_events/audio_health.rs b/apps/screenpipe-app-tauri/src-tauri/src/engine_events/audio_health.rsindex 6d2765eb5..06213312a 100644--- a/apps/screenpipe-app-tauri/src-tauri/src/engine_events/audio_health.rs+++ b/apps/screenpipe-app-tauri/src-tauri/src/engine_events/audio_health.rs@@ -13,6 +13,14 @@//! output capture, so a notification (with a RESTART action) is the right//! escalation: the user can fix audio routing while it still matters.//!+//! The mic-side twins (`audio_capture_health_mic_silent`,+//! `audio_capture_health_mic_recovered`, `audio_capture_health_mic_capture_failed`)+//! cover the meeting-piggyback per-process mic tap: silent means the tap is+//! open but no audio has arrived for 2+ minutes (RESTART action, same as+//! speaker-silent); capture-failed means the tap never opened at all+//! (no action — capture already fell back to the default mic); recovered is+//! emit-only, no notification, matching `audio_capture_health_recovered`.+//!//! See `crates/screenpipe-audio/src/audio_manager/windows_output_follow.rs`//! for the watchdog, and//! `crates/screenpipe-events/src/custom_events/audio_health.rs` for the@@ -27,6 +35,9 @@ pub(super) fn handle(app: &AppHandle, name: &str, data: &Value) {let tauri_event = match name {"audio_capture_health_speaker_silent" => "audio-capture-health-speaker-silent","audio_capture_health_recovered" => "audio-capture-health-recovered",+ "audio_capture_health_mic_silent" => "audio-capture-health-mic-silent",+ "audio_capture_health_mic_recovered" => "audio-capture-health-mic-recovered",+ "audio_capture_health_mic_capture_failed" => "audio-capture-health-mic-capture-failed",_ => {debug!("audio_health::handle called with unexpected name: {}", name);return;@@ -37,8 +48,18 @@ pub(super) fn handle(app: &AppHandle, name: &str, data: &Value) {warn!("failed to emit {}: {}", tauri_event, e);}- if name == "audio_capture_health_speaker_silent" {- show_speaker_silent_notification(app.clone(), data.clone());+ match name {+ "audio_capture_health_speaker_silent" => {+ show_speaker_silent_notification(app.clone(), data.clone());+ }+ "audio_capture_health_mic_silent" => {+ show_mic_silent_notification(app.clone(), data.clone());+ }+ "audio_capture_health_mic_capture_failed" => {+ show_mic_capture_failed_notification(app.clone(), data.clone());+ }+ // mic_recovered: tauri event emit only (parity with speaker Recovered) — no notification.+ _ => {}}}@@ -88,3 +109,77 @@ fn show_speaker_silent_notification(app: AppHandle, data: Value) {}});}++/// Show the in-app notification panel telling the user their meeting's+/// per-process mic capture has gone silent. Mirrors [`show_speaker_silent_notification`]+/// (same RESTART action and panel type) — the mic-side twin of the speaker+/// stall alert.+fn show_mic_silent_notification(app: AppHandle, data: Value) {+ let capturing_inputs = data+ .get("capturing_inputs")+ .and_then(|v| v.as_array())+ .map(|a| {+ a.iter()+ .filter_map(|v| v.as_str())+ .collect::<Vec<_>>()+ .join(", ")+ })+ .unwrap_or_default();++ let body = if capturing_inputs.is_empty() {+ "your meeting app is recording, but screenpipe's mic capture has been \+ silent for 2+ minutes."+ .to_string()+ } else {+ format!(+ "your meeting app is recording from {}, but screenpipe's capture \+ has been silent for 2+ minutes.",+ capturing_inputs+ )+ };++ let payload = serde_json::json!({+ "id": "audio_capture_health_mic_silent",+ "type": "capture_stall",+ "title": "screenpipe may not be hearing your mic",+ "body": body,+ "actions": [+ { "label": "RESTART", "action": "restart_recording", "primary": true }+ ],+ "autoDismissMs": 30000+ });++ tauri::async_runtime::spawn(async move {+ if let Err(e) = crate::commands::show_notification_panel(app, payload.to_string()).await {+ warn!("failed to show mic-silent notification: {}", e);+ }+ });+}++/// Show the in-app notification panel telling the user screenpipe couldn't+/// open the meeting app's microphone (per-process capture failed to start),+/// but recording continues on the default mic. No action button — there's+/// nothing to restart; the device is unavailable, not stalled.+fn show_mic_capture_failed_notification(app: AppHandle, data: Value) {+ let reason = data+ .get("reason")+ .and_then(|v| v.as_str())+ .unwrap_or("the device may be in exclusive use");++ let body = format!("{} — recording continues on your default microphone.", reason);++ let payload = serde_json::json!({+ "id": "audio_capture_health_mic_capture_failed",+ "type": "capture_stall",+ "title": "couldn't open your meeting microphone",+ "body": body,+ "actions": [],+ "autoDismissMs": 30000+ });++ tauri::async_runtime::spawn(async move {+ if let Err(e) = crate::commands::show_notification_panel(app, payload.to_string()).await {+ warn!("failed to show mic-capture-failed notification: {}", e);+ }+ });+}