YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
perf(meeting-detector): gate the depth-25 AX walk on recent audio (#3957)
Louis Beaumont33b5e184a5ad
diff --git a/crates/screenpipe-audio/src/meeting_detector.rs b/crates/screenpipe-audio/src/meeting_detector.rsindex 036759229..0c6f58c77 100644--- a/crates/screenpipe-audio/src/meeting_detector.rs+++ b/crates/screenpipe-audio/src/meeting_detector.rs@@ -11,7 +11,23 @@//! The detection loop calls `set_v2_in_meeting(true/false)` and both//! `is_in_meeting()` and `is_in_audio_session()` simply return that flag.-use std::sync::atomic::{AtomicBool, Ordering};+use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};+use std::time::{SystemTime, UNIX_EPOCH};++use tokio::sync::Notify;++/// How recent audio activity must be (ms) for [`MeetingDetector::on_audio_activity`]+/// to treat a fresh active chunk as a genuine quiet->active transition rather+/// than an ongoing session. Mirrors the engine detector's `AUDIO_GATE_WINDOW`.+const AUDIO_RECENCY_WINDOW_MS: u64 = 45_000;++/// Wall-clock milliseconds since the Unix epoch (0 if the clock predates it).+fn now_ms() -> u64 {+ SystemTime::now()+ .duration_since(UNIX_EPOCH)+ .map(|d| d.as_millis() as u64)+ .unwrap_or(0)+}/// Lightweight meeting state holder for the audio pipeline.///@@ -22,6 +38,14 @@ use std::sync::atomic::{AtomicBool, Ordering};pub struct MeetingDetector {/// Override flag set by the v2 meeting detection system.v2_override: AtomicBool,+ /// Wall-clock ms of the most recent audio activity (input or output),+ /// stamped by [`Self::on_audio_activity`]. 0 = none observed yet. The+ /// engine's detection loop reads this to gate its expensive AX scan.+ last_audio_activity_ms: AtomicU64,+ /// Fired on the rising edge of audio activity (quiet -> active) so the+ /// detection loop can wake immediately and scan instead of waiting out a+ /// slow idle interval.+ audio_onset: Notify,}impl Default for MeetingDetector {@@ -34,6 +58,8 @@ impl MeetingDetector {pub fn new() -> Self {Self {v2_override: AtomicBool::new(false),+ last_audio_activity_ms: AtomicU64::new(0),+ audio_onset: Notify::new(),}}@@ -59,13 +85,45 @@ impl MeetingDetector {self.v2_override.load(Ordering::Relaxed)}- /// No-op kept for audio pipeline compatibility.+ /// Record audio activity reported by the capture pipeline (input or output).+ ///+ /// The pipeline already computes `has_activity` per chunk via an RMS+ /// threshold (see `audio_manager`); we stamp the time of the latest active+ /// chunk so the engine's detection loop can gate its expensive AX scan on+ /// "was there any audio recently?". On a quiet->active transition we also+ /// wake any onset waiter so a call that just started is scanned promptly.+ ///+ /// Inactive chunks are ignored — they must not reset recency.pub fn on_audio_activity(&self,_device_type: &crate::core::device::DeviceType,- _has_activity: bool,+ has_activity: bool,) {- // v2 detection is UI-based; audio activity is not used for meeting detection.+ if !has_activity {+ return;+ }+ let now = now_ms();+ let prev = self.last_audio_activity_ms.swap(now, Ordering::Relaxed);+ // Fire the onset wake only on a genuine quiet->active edge. Notifying on+ // every active chunk would defeat the loop's slow idle cadence.+ if prev == 0 || now.saturating_sub(prev) >= AUDIO_RECENCY_WINDOW_MS {+ self.audio_onset.notify_waiters();+ }+ }++ /// True if audio activity was observed within the last `window_ms`. The+ /// detection loop uses this to keep scanning at the fast idle rate only+ /// while audio is flowing.+ pub fn audio_active_within(&self, window_ms: u64) -> bool {+ let last = self.last_audio_activity_ms.load(Ordering::Relaxed);+ last != 0 && now_ms().saturating_sub(last) < window_ms+ }++ /// Resolves on the next quiet->active audio transition. The detection loop+ /// selects on this so a call that just started is picked up immediately,+ /// without waiting out a slow idle interval.+ pub async fn wait_for_audio_onset(&self) {+ self.audio_onset.notified().await;}/// No-op kept for audio pipeline compatibility.@@ -98,4 +156,41 @@ mod tests {assert!(!detector.is_in_meeting());assert!(!detector.is_in_audio_session());}++ #[test]+ fn audio_recency_starts_inactive() {+ let detector = MeetingDetector::new();+ // Nothing observed yet -> never recent, regardless of window.+ assert!(!detector.audio_active_within(45_000));+ assert!(!detector.audio_active_within(u64::MAX));+ }++ #[test]+ fn audio_activity_marks_recent() {+ let detector = MeetingDetector::new();+ detector.on_audio_activity(&crate::core::device::DeviceType::Output, true);+ // Just stamped -> recent within a normal window.+ assert!(detector.audio_active_within(45_000));+ // ...but a zero-width window is never "within".+ assert!(!detector.audio_active_within(0));+ }++ #[test]+ fn inactive_chunks_do_not_mark_recent() {+ let detector = MeetingDetector::new();+ detector.on_audio_activity(&crate::core::device::DeviceType::Input, false);+ assert!(!detector.audio_active_within(45_000));+ }++ #[test]+ fn audio_activity_is_independent_of_v2_flag() {+ // The audio-recency signal must NOT be derived from the v2 override+ // (that would re-introduce the circular dependency this replaces).+ let detector = MeetingDetector::new();+ detector.set_v2_in_meeting(true);+ assert!(!detector.audio_active_within(45_000));+ detector.on_audio_activity(&crate::core::device::DeviceType::Input, true);+ detector.set_v2_in_meeting(false);+ assert!(detector.audio_active_within(45_000));+ }}diff --git a/crates/screenpipe-engine/src/meeting_detector.rs b/crates/screenpipe-engine/src/meeting_detector.rsindex 453ef93b0..e5c36112a 100644--- a/crates/screenpipe-engine/src/meeting_detector.rs+++ b/crates/screenpipe-engine/src/meeting_detector.rs@@ -2452,6 +2452,36 @@ const IDLE_APPS_SCAN_INTERVAL: Duration = Duration::from_secs(10);/// Scan interval when idle and no meeting apps are running at all.const IDLE_NO_APPS_SCAN_INTERVAL: Duration = Duration::from_secs(30);+/// Scan interval when idle with meeting apps open but NO recent audio. A+/// meeting is implausible without audio (you hear people / your mic is live),+/// so we poll slowly to avoid the costly AX walk. Audio onset re-wakes the loop+/// instantly, so a call that starts is still detected without added latency.+const IDLE_QUIET_SCAN_INTERVAL: Duration = Duration::from_secs(30);++/// How recent audio activity must be to keep scanning at the fast idle rate.+/// Kept in sync with `screenpipe_audio`'s `AUDIO_RECENCY_WINDOW_MS`.+const AUDIO_GATE_WINDOW: Duration = Duration::from_secs(45);++/// Scan interval to use when meeting apps are present, gated on audio activity.+///+/// Pure (no I/O) so the gating policy is unit-tested directly. Only the Idle+/// state is gated: with apps open but no recent audio a meeting is implausible,+/// so we drop from the fast idle rate to the quiet rate (audio onset re-wakes+/// the loop instantly). Confirming/Active/Ending always scan at `base` so an+/// in-progress meeting is tracked at full fidelity — the audio gate can never+/// slow down or end a meeting that the state machine is already tracking.+fn apps_present_scan_interval(is_idle: bool, audio_recent: bool, base: Duration) -> Duration {+ if is_idle {+ if audio_recent {+ IDLE_APPS_SCAN_INTERVAL+ } else {+ IDLE_QUIET_SCAN_INTERVAL+ }+ } else {+ base+ }+}+/// Run the meeting detection loop.////// This is the main entry point for the v2 meeting detection system.@@ -2623,8 +2653,21 @@ pub async fn run_meeting_detection_loop();loop {+ // Audio onset wakes us immediately so a call that just started is+ // detected without waiting out a slow idle interval. With no detector+ // (tests / detector disabled) this future never resolves, so the cadence+ // is pure-sleep and byte-identical to the prior behaviour.+ let audio_onset = async {+ match detector.as_ref() {+ Some(d) => d.wait_for_audio_onset().await,+ None => std::future::pending::<()>().await,+ }+ };tokio::select! {_ = tokio::time::sleep(current_interval) => {}+ _ = audio_onset => {+ debug!("meeting v2: woken early by audio onset");+ }_ = shutdown_rx.recv() => {info!("meeting v2: shutdown received, exiting detection loop");// If we're in an active meeting, end it cleanly@@ -2953,12 +2996,17 @@ pub async fn run_meeting_detection_loop(let (new_state, action) = advance_state(state, &scan_results, keep_alive);state = new_state;- // Adaptive interval based on state+ // Adaptive interval based on state, gated on recent audio when Idle.+ // With no detector, `audio_recent` is true => unchanged fast idle rate.idle_scan_count = 0; // reset idle counter when apps are present- current_interval = match &state {- MeetingState::Idle => IDLE_APPS_SCAN_INTERVAL, // apps open but no call- _ => base_interval, // Confirming/Active/Ending — scan fast- };+ let audio_recent = detector.as_ref().map_or(true, |d| {+ d.audio_active_within(AUDIO_GATE_WINDOW.as_millis() as u64)+ });+ current_interval = apps_present_scan_interval(+ matches!(state, MeetingState::Idle),+ audio_recent,+ base_interval,+ );// 4. Handle actionsif let Some(action) = action {@@ -3411,6 +3459,48 @@ async fn insert_new_meeting(mod tests {use super::*;+ // ── audio-gated scan cadence tests ─────────────────────────────────+ // These pin the CPU optimisation: with apps open but no recent audio the+ // Idle scan rate drops; a tracked meeting (any non-Idle state) is never+ // slowed, so detection accuracy can't regress.++ #[test]+ fn idle_with_apps_and_audio_scans_at_fast_idle_rate() {+ assert_eq!(+ apps_present_scan_interval(true, true, ACTIVE_SCAN_INTERVAL),+ IDLE_APPS_SCAN_INTERVAL+ );+ }++ #[test]+ fn idle_with_apps_but_no_audio_scans_slowly() {+ assert_eq!(+ apps_present_scan_interval(true, false, ACTIVE_SCAN_INTERVAL),+ IDLE_QUIET_SCAN_INTERVAL+ );+ }++ #[test]+ fn tracked_meeting_ignores_audio_gate() {+ // Non-Idle (Confirming/Active/Ending) -> always `base`, with or without+ // audio. The gate can never slow down or end a tracked meeting.+ assert_eq!(+ apps_present_scan_interval(false, false, ACTIVE_SCAN_INTERVAL),+ ACTIVE_SCAN_INTERVAL+ );+ assert_eq!(+ apps_present_scan_interval(false, true, ACTIVE_SCAN_INTERVAL),+ ACTIVE_SCAN_INTERVAL+ );+ }++ #[test]+ fn quiet_idle_rate_is_strictly_slower_than_active_idle_rate() {+ // The whole point: quiet polling is slower than the audio-present rate,+ // so the AX walk runs less often when no meeting is plausible.+ assert!(IDLE_QUIET_SCAN_INTERVAL > IDLE_APPS_SCAN_INTERVAL);+ }+// ── ignoredMeetingApps filter tests ────────────────────────────────fn zoom_test_profile() -> MeetingDetectionProfile {