YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
feat(audio): session-scoped devices + Meeting Tap virtual device (never persisted to enabled_devices)
Ezra Ellette81017d564839
diff --git a/crates/screenpipe-audio/src/audio_manager/device_monitor.rs b/crates/screenpipe-audio/src/audio_manager/device_monitor.rsindex 52022db50..6e3fa0e49 100644--- a/crates/screenpipe-audio/src/audio_manager/device_monitor.rs+++ b/crates/screenpipe-audio/src/audio_manager/device_monitor.rs@@ -846,10 +846,22 @@ pub async fn start_device_monitor(// Only stop the old inputs once the new one is live.if new_started {+ // Never stop meeting-session or piggyback-suspended+ // devices during a default swap: they are not user+ // intent and the sweep owns their lifecycle. (They+ // are already absent from enabled_devices; this is a+ // belt-and-suspenders skip that also documents it.)+ let session = audio_manager.session_devices();+ let suspended = audio_manager.suspended_devices();for device_name in enabled_devices.iter() {if *device_name == new_default_input {continue; // don't stop the one we just started}+ if session.contains(device_name)+ || suspended.contains(device_name)+ {+ continue;+ }if let Ok(device) = parse_audio_device(device_name) {if device.device_type == DeviceType::Input {let _ = audio_manager.stop_device(device_name).await;@@ -923,6 +935,13 @@ pub async fn start_device_monitor(// Don't stop the communications output device during swap#[cfg(target_os = "windows")]let comm_name = default_tracker.last_communications_output.clone();+ // Never stop the Meeting Tap or other meeting-session /+ // piggyback-suspended outputs during a default swap: they+ // are meeting-lifetime, not user intent, and the sweep owns+ // their teardown. Misattributing/stopping the Meeting Tap+ // here would drop far-end call audio mid-meeting.+ let session = audio_manager.session_devices();+ let suspended = audio_manager.suspended_devices();for device_name in audio_manager.enabled_devices().await.iter() {if *device_name == new_default_output {continue; // don't stop the one we just started@@ -931,6 +950,11 @@ pub async fn start_device_monitor(if comm_name.as_deref() == Some(device_name.as_str()) {continue; // don't stop the communications device}+ if session.contains(device_name)+ || suspended.contains(device_name)+ {+ continue;+ }if let Ok(device) = parse_audio_device(device_name) {if device.device_type == DeviceType::Output {info!(diff --git a/crates/screenpipe-audio/src/audio_manager/manager.rs b/crates/screenpipe-audio/src/audio_manager/manager.rsindex b20507693..37c10b1df 100644--- a/crates/screenpipe-audio/src/audio_manager/manager.rs+++ b/crates/screenpipe-audio/src/audio_manager/manager.rs@@ -157,6 +157,19 @@ pub struct AudioManager {/// The device monitor must never auto-start devices in this set./// Cleared on global start/stop but preserved across reconnects.user_disabled_devices: Arc<RwLock<HashSet<String>>>,+ /// Meeting-lifetime session streams (e.g. the Meeting Tap, a piggyback+ /// mic). Registered by `start_session_device`, removed by+ /// `stop_session_device`. NEVER persisted to `enabled_devices` (those are+ /// user intent). Read inside the blocking audio-receiver loop to bypass the+ /// meetings-only drop gate, so this is a `std::sync::RwLock` (not tokio):+ /// the guard is taken, the set membership checked, and the guard dropped —+ /// never held across an `.await`.+ session_devices: Arc<std::sync::RwLock<HashSet<String>>>,+ /// Devices the piggyback sweep has suspended for the duration of a meeting+ /// (a session stream stands in for them). Nothing else may start a device+ /// in this set; the sweep resumes it on meeting end / fallback. Same+ /// `std::sync::RwLock` discipline as `session_devices`.+ suspended_devices: Arc<std::sync::RwLock<HashSet<String>>>,}/// Result of checking / restarting the two central handler tasks.@@ -243,6 +256,8 @@ impl AudioManager {reconciliation_handle: Arc::new(RwLock::new(None)),drm_stopped_devices: Arc::new(RwLock::new(Vec::new())),user_disabled_devices: Arc::new(RwLock::new(HashSet::new())),+ session_devices: Arc::new(std::sync::RwLock::new(HashSet::new())),+ suspended_devices: Arc::new(std::sync::RwLock::new(HashSet::new())),};Ok(manager)@@ -653,6 +668,21 @@ impl AudioManager {return Ok(());}+ // Suspended by the piggyback sweep for the duration of a meeting (a+ // session stream replaces this device). The sweep resumes it on+ // meeting end / fallback; nothing else may start it meanwhile. This is+ // the single choke point so every monitor path that could restart a+ // suspended device is blocked at once.+ if self+ .suspended_devices+ .read()+ .unwrap()+ .contains(&device.to_string())+ {+ debug!("skipping start of piggyback-suspended device: {}", device);+ return Ok(());+ }+if let Err(e) = self.device_manager.start_device(device).await {let err_str = e.to_string();@@ -683,6 +713,103 @@ impl AudioManager {Ok(())}+ /// Start a meeting-session stream. Mirrors `start_device` but: never+ /// touches `enabled_devices` (session streams are meeting-lifetime, not+ /// user intent), registers in `session_devices` (drop-gate bypass ++ /// sweep bookkeeping), and can carry tap pids for the Meeting Tap device.+ pub async fn start_session_device(+ &self,+ device: &AudioDevice,+ tap_pids: Option<Vec<i32>>,+ ) -> Result<()> {+ if self.options.read().await.is_disabled {+ return Ok(());+ }+ // Insert BEFORE starting: the audio-receiver drop-gate bypass must see+ // this device from the very first chunk. Rolled back on failure below.+ self.session_devices+ .write()+ .unwrap()+ .insert(device.to_string());+ let start_result = match tap_pids {+ Some(pids) => {+ self.device_manager+ .start_device_with_tap_pids(device, pids)+ .await+ }+ None => self.device_manager.start_device(device).await,+ };+ if let Err(e) = start_result {+ let err_str = e.to_string();+ if !err_str.contains("already running") {+ self.session_devices+ .write()+ .unwrap()+ .remove(&device.to_string());+ return Err(e);+ }+ }+ if !self.recording_handles.contains_key(device) {+ if let Some(is_running) = self.device_manager.is_running_mut(device) {+ is_running.store(true, Ordering::Relaxed);+ }+ match self.record_device(device).await {+ Ok(handle) => {+ self.recording_handles+ .insert(device.clone(), Arc::new(Mutex::new(handle)));+ }+ Err(e) => {+ self.session_devices+ .write()+ .unwrap()+ .remove(&device.to_string());+ return Err(e);+ }+ }+ }+ Ok(())+ }++ /// Tear down a meeting-session stream. Never touches `enabled_devices`.+ pub async fn stop_session_device(&self, device: &AudioDevice) -> Result<()> {+ self.session_devices+ .write()+ .unwrap()+ .remove(&device.to_string());+ self.stop_device_recording(device).await+ }++ /// Snapshot of the currently-registered meeting-session device names.+ pub fn session_devices(&self) -> HashSet<String> {+ self.session_devices.read().unwrap().clone()+ }++ /// Suspend a device for the duration of a meeting: nothing may start it+ /// until `unsuspend_device`. Used by the piggyback sweep when a session+ /// stream stands in for a normally-captured device.+ ///+ /// NOTE: the resume counterpart is `unsuspend_device`, NOT `resume_device`.+ /// `resume_device` already exists as the user-facing un-pause flow (it+ /// operates on `user_disabled_devices` and calls `start_device`), a+ /// different concept from meeting suspension; reusing that name would be a+ /// duplicate-method compile error.+ pub fn suspend_device(&self, name: &str) {+ self.suspended_devices+ .write()+ .unwrap()+ .insert(name.to_string());+ }++ /// Lift a suspension set by `suspend_device`.+ pub fn unsuspend_device(&self, name: &str) {+ self.suspended_devices.write().unwrap().remove(name);+ }++ /// Snapshot of the currently-suspended device names.+ pub fn suspended_devices(&self) -> HashSet<String> {+ self.suspended_devices.read().unwrap().clone()+ }+pub async fn use_all_devices(&self) -> bool {self.options.read().await.use_all_devices}@@ -801,6 +928,9 @@ impl AudioManager {let db = self.db.clone();let shared_engine = self.engine.clone();let on_insert_session = self.on_transcription_insert.clone();+ // Session streams (Meeting Tap, piggyback mic) bypass the meetings-only+ // drop gate below — they exist only during a meeting by construction.+ let session_devices = self.session_devices.clone();// Build unified transcription engine — only loads the needed modellet engine = TranscriptionEngine::new(@@ -873,11 +1003,18 @@ impl AudioManager {// cannot tell whether we're in a meeting, so we keep capturing rather// than silently dropping everything.if audio_capture_mode == AudioCaptureMode::MeetingsOnly {+ // Session streams exist only during a meeting by+ // construction — never drop them, even if the detector+ // flaps mid-meeting.+ let is_session_stream = session_devices+ .read()+ .unwrap()+ .contains(&audio.device.to_string());let in_session = meeting_detector.as_ref().map(|m| m.is_in_audio_session()).unwrap_or(true);- if !in_session {+ if !is_session_stream && !in_session {debug!("meetings-only capture: no active meeting, dropping audio chunk from {:?}",audio.device.name@@ -1504,11 +1641,21 @@ impl AudioManager {/// Returns device names that need to be cleaned up and restartedpub async fn check_stale_recording_handles(&self) -> Vec<String> {let mut stale_devices = Vec::new();+ let session_devices = self.session_devices.read().unwrap().clone();for pair in self.recording_handles.iter() {let device = pair.key();let handle = pair.value();+ // Session devices are virtual / meeting-lifetime and absent from+ // both enabled_devices and the available-device list. If the monitor+ // culled a stale session handle it would then try to restart it via+ // start_device (losing the tap pids AND leaking the name into+ // enabled_devices). Only stop_session_device may cull them.+ if session_devices.contains(&device.to_string()) {+ continue;+ }+// Check if the JoinHandle has finished (task completed/crashed)if handle.lock().await.is_finished() {stale_devices.push(device.to_string());@@ -1603,6 +1750,21 @@ impl AudioManager {/// Clean up a stale device that had its recording handle crash/timeout/// This prepares the device for restart by the device monitorpub async fn cleanup_stale_device(&self, device_name: &str) -> anyhow::Result<()> {+ // Defensive: never clean up a session device here. `check_stale_recording_handles`+ // already omits them from the stale list (the only caller), but guard the+ // public entry point too — only `stop_session_device` may tear these down.+ if self+ .session_devices+ .read()+ .unwrap()+ .contains(device_name)+ {+ debug!(+ "cleanup_stale_device({device_name}): session device, leaving intact (only stop_session_device may cull it)"+ );+ return Ok(());+ }+let device = match parse_audio_device(device_name) {Ok(device) => device,Err(_) => return Err(anyhow!("Device {} not found", device_name)),diff --git a/crates/screenpipe-audio/src/core/device.rs b/crates/screenpipe-audio/src/core/device.rsindex 1b09ae211..8e734f827 100644--- a/crates/screenpipe-audio/src/core/device.rs+++ b/crates/screenpipe-audio/src/core/device.rs@@ -23,6 +23,11 @@ use tokio::sync::Mutex as AsyncMutex;#[cfg(target_os = "macos")]pub const MACOS_OUTPUT_AUDIO_DEVICE_NAME: &str = "System Audio";+/// Virtual output device representing the per-process meeting tap. Session-+/// scoped: started by the piggyback sweep during meetings, never advertised in+/// device lists, never persisted to enabled_devices.+pub const MEETING_TAP_DEVICE_NAME: &str = "Meeting Tap";+#[derive(OaSchema, Clone, Debug)]pub struct DeviceControl {pub is_running: bool,@@ -1195,6 +1200,23 @@ mod resolve_audio_tests {}}+#[cfg(test)]+mod meeting_tap_device_tests {+ use super::{parse_audio_device, AudioDevice, DeviceType, MEETING_TAP_DEVICE_NAME};++ /// The Meeting Tap virtual output device name must round-trip through+ /// `parse_audio_device` in its display form `"Meeting Tap (output)"`, so a+ /// session device started by the piggyback sweep can be parsed back from its+ /// string form everywhere the pipeline stringifies devices.+ #[test]+ fn meeting_tap_display_name_round_trips() {+ assert_eq!(+ parse_audio_device("Meeting Tap (output)").unwrap(),+ AudioDevice::new(MEETING_TAP_DEVICE_NAME.to_string(), DeviceType::Output)+ );+ }+}+#[cfg(all(test, not(all(target_os = "linux", feature = "pulseaudio"))))]mod stream_config_tests {use super::ensure_usable_stream_config;diff --git a/crates/screenpipe-audio/src/core/stream.rs b/crates/screenpipe-audio/src/core/stream.rsindex 6301befb1..deb4b359a 100644--- a/crates/screenpipe-audio/src/core/stream.rs+++ b/crates/screenpipe-audio/src/core/stream.rs@@ -145,6 +145,11 @@ impl AudioStream {#[cfg_attr(not(target_os = "macos"), allow(unused_variables))] use_coreaudio_tap: bool,#[cfg_attr(not(target_os = "windows"), allow(unused_variables))] windows_input_aec: bool,#[cfg_attr(not(target_os = "macos"), allow(unused_variables))] macos_input_vpio: bool,+ #[cfg_attr(+ not(any(target_os = "macos", target_os = "windows")),+ allow(unused_variables)+ )]+ tap_pids: Option<Vec<i32>>,) -> Result<Self> {let (tx, _) = broadcast::channel::<Vec<f32>>(1000);let tx_clone = tx.clone();@@ -184,7 +189,37 @@ impl AudioStream {#[cfg(not(target_os = "macos"))]let use_process_tap = false;- if use_process_tap {+ // Per-process meeting tap: virtual device backed by+ // spawn_process_tap_capture_for_pids. No inline fallback here — the+ // piggyback sweep owns the fallback decision (it starts the stable+ // global path instead when this errors), so a failure must surface+ // as Err, not silently degrade into misattributed audio.+ #[cfg(any(target_os = "macos", target_os = "windows"))]+ let meeting_tap_pids = tap_pids.filter(|_| {+ device.device_type == super::device::DeviceType::Output+ && device.name == super::device::MEETING_TAP_DEVICE_NAME+ });+ #[cfg(not(any(target_os = "macos", target_os = "windows")))]+ let meeting_tap_pids: Option<Vec<i32>> = None;++ if let Some(pids) = meeting_tap_pids {+ #[cfg(any(target_os = "macos", target_os = "windows"))]+ {+ let (config, thread) = super::process_tap::spawn_process_tap_capture_for_pids(+ pids,+ tx.clone(),+ is_running.clone(),+ is_disconnected.clone(),+ )?;+ drop(stream_control_rx);+ // Process Tap backend → not SCK-backed.+ (config, thread, false)+ }+ #[cfg(not(any(target_os = "macos", target_os = "windows")))]+ {+ unreachable!()+ }+ } else if use_process_tap {#[cfg(target_os = "macos")]{match super::process_tap::spawn_process_tap_capture(@@ -1040,6 +1075,42 @@ mod from_wav_tests {}}+#[cfg(all(test, any(target_os = "macos", target_os = "windows")))]+mod meeting_tap_stream_tests {+ use super::*;+ use crate::core::device::DeviceType;++ /// Live: the Meeting Tap virtual output device builds a working stream when+ /// pointed at a live pid (our own process). Self-skips when the process tap+ /// is unavailable (macOS < 14.4, or Windows below the loopback build floor),+ /// mirroring the process_tap live tests. Proves the `tap_pids` arm in+ /// `from_device` wires through to `spawn_process_tap_capture_for_pids`.+ #[tokio::test]+ async fn meeting_tap_stream_builds_for_own_pid() {+ if !crate::core::process_tap::is_process_tap_available() {+ eprintln!("skipping: process tap unavailable");+ return;+ }+ let device = Arc::new(AudioDevice::new(+ crate::core::device::MEETING_TAP_DEVICE_NAME.to_string(),+ DeviceType::Output,+ ));+ let stream = AudioStream::from_device(+ device,+ Arc::new(AtomicBool::new(true)),+ false,+ false,+ false,+ Some(vec![std::process::id() as i32]),+ )+ .await+ .expect("meeting tap stream should build for a live pid");+ // `AudioStream::stop` takes no argument (see `from_wav_emits_chunks`); it+ // flips is_disconnected and joins/aborts the tap thread.+ stream.stop().await.expect("stop meeting tap stream");+ }+}+#[cfg(all(test, not(all(target_os = "linux", feature = "pulseaudio"))))]mod wasapi_format_tests {use super::is_wasapi_unsupported_format;diff --git a/crates/screenpipe-audio/src/device/device_manager.rs b/crates/screenpipe-audio/src/device/device_manager.rsindex 91a173f98..19e56a76c 100644--- a/crates/screenpipe-audio/src/device/device_manager.rs+++ b/crates/screenpipe-audio/src/device/device_manager.rs@@ -107,7 +107,31 @@ impl DeviceManager {}pub async fn start_device(&self, device: &AudioDevice) -> Result<()> {- if !self.devices().await.contains(device) {+ self.start_device_inner(device, None).await+ }++ /// Start a device whose stream is backed by the per-process meeting tap+ /// (`spawn_process_tap_capture_for_pids`) for `tap_pids`. Only meaningful+ /// for the Meeting Tap virtual output device; other devices ignore the pids+ /// in `AudioStream::from_device`.+ pub async fn start_device_with_tap_pids(+ &self,+ device: &AudioDevice,+ tap_pids: Vec<i32>,+ ) -> Result<()> {+ self.start_device_inner(device, Some(tap_pids)).await+ }++ async fn start_device_inner(+ &self,+ device: &AudioDevice,+ tap_pids: Option<Vec<i32>>,+ ) -> Result<()> {+ // The meeting tap is a virtual device synthesized from pids+ // (spawn_process_tap_capture_for_pids); it is deliberately never in+ // list_audio_devices(), so skip the availability check on the tap path.+ // Every other device must still be a real, currently-listed device.+ if tap_pids.is_none() && !self.devices().await.contains(device) {return Err(anyhow!("device {device} not found"));}@@ -122,6 +146,7 @@ impl DeviceManager {self.use_coreaudio_tap.load(Ordering::Relaxed),self.windows_input_aec.load(Ordering::Relaxed),self.effective_macos_input_vpio(device),+ tap_pids,).await{diff --git a/crates/screenpipe-audio/tests/core_tests.rs b/crates/screenpipe-audio/tests/core_tests.rsindex 98f3c2dcc..b97af2513 100644--- a/crates/screenpipe-audio/tests/core_tests.rs+++ b/crates/screenpipe-audio/tests/core_tests.rs@@ -66,7 +66,7 @@ mod tests {let is_running_clone = Arc::clone(&is_running);let audio_stream =- AudioStream::from_device(device_spec, is_running_clone, false, false, false)+ AudioStream::from_device(device_spec, is_running_clone, false, false, false, None).await.unwrap();@@ -127,10 +127,16 @@ mod tests {let is_running = Arc::new(AtomicBool::new(true));let is_running_clone = Arc::clone(&is_running);- let audio_stream =- AudioStream::from_device(device_spec, is_running_clone.clone(), false, false, false)- .await- .unwrap();+ let audio_stream = AudioStream::from_device(+ device_spec,+ is_running_clone.clone(),+ false,+ false,+ false,+ None,+ )+ .await+ .unwrap();// interrupt in 10 secondstokio::spawn(async move {@@ -406,10 +412,16 @@ mod tests {let is_running = Arc::new(AtomicBool::new(true));// Create an audio stream- let stream =- AudioStream::from_device(Arc::new(device), is_running.clone(), false, false, false)- .await- .expect("should create audio stream");+ let stream = AudioStream::from_device(+ Arc::new(device),+ is_running.clone(),+ false,+ false,+ false,+ None,+ )+ .await+ .expect("should create audio stream");// Let it run for a bit to ensure the audio thread is activetokio::time::sleep(Duration::from_millis(100)).await;