YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
fix(audio): keep draining capped transcription backlogs (#5406)
Louis Beaumont852925264d4b
diff --git a/crates/screenpipe-audio/src/audio_manager/manager.rs b/crates/screenpipe-audio/src/audio_manager/manager.rsindex 473585675..cba456ed4 100644--- a/crates/screenpipe-audio/src/audio_manager/manager.rs+++ b/crates/screenpipe-audio/src/audio_manager/manager.rs@@ -16,7 +16,7 @@ use std::{time::{Duration, SystemTime, UNIX_EPOCH},};use tokio::{- sync::{broadcast, oneshot, Mutex, RwLock},+ sync::{broadcast, oneshot, Mutex, Notify, RwLock},task::JoinHandle,};use tracing::{debug, error, info, warn};@@ -121,6 +121,8 @@ struct MeetingEventData {type RecordingHandlesMap = DashMap<AudioDevice, Arc<Mutex<JoinHandle<Result<()>>>>>;const MEETING_AUDIO_FRAME_BUFFER: usize = 512;+const RECONCILIATION_IDLE_INTERVAL: Duration = Duration::from_secs(120);+const MAX_IMMEDIATE_RECONCILIATION_SWEEPS: usize = 4;/// Wall-clock milliseconds since the Unix epoch (0 if the clock predates it)./// Local to the audio manager so the receiver-loop stamping and the piggyback@@ -164,6 +166,9 @@ pub struct AudioManager {engine_builds: EngineBuildCoordinator,/// Handle to the reconciliation background task so we can abort it on shutdown.reconciliation_handle: Arc<RwLock<Option<JoinHandle<()>>>>,+ /// Wakes the background reconciliation task when a foreground sweep fills+ /// its 50-chunk cap, so a meeting backlog keeps draining immediately.+ reconciliation_wakeup: Arc<Notify>,/// Output devices temporarily stopped due to DRM content detection./// Stored so they can be restarted when DRM clears.drm_stopped_devices: Arc<RwLock<Vec<AudioDevice>>>,@@ -375,6 +380,7 @@ impl AudioManager {engine: Arc::new(RwLock::new(None)),engine_builds: EngineBuildCoordinator::new(),reconciliation_handle: Arc::new(RwLock::new(None)),+ reconciliation_wakeup: Arc::new(Notify::new()),drm_stopped_devices: Arc::new(RwLock::new(Vec::new())),user_disabled_devices: Arc::new(RwLock::new(HashSet::new())),session_devices: Arc::new(std::sync::RwLock::new(HashSet::new())),@@ -500,9 +506,15 @@ impl AudioManager {let output_path_bg = self.options.read().await.output_path.clone();let metrics_bg = self.metrics.clone();let meeting_detector_bg = self.meeting_detector().await;+ let reconciliation_wakeup = self.reconciliation_wakeup.clone();let handle = tokio::spawn(async move {- // Wait for model to load + initial recordings- tokio::time::sleep(Duration::from_secs(120)).await;+ // Wait for model load + initial recordings, unless a foreground+ // session-end sweep tells us its 50-chunk cap left more work.+ tokio::select! {+ _ = tokio::time::sleep(RECONCILIATION_IDLE_INTERVAL) => {}+ _ = reconciliation_wakeup.notified() => {}+ }+ let mut consecutive_full_sweeps = 0usize;loop {// Contain a panic inside a sweep so it cannot kill this// long-lived worker (issue #3498: a single panic used to@@ -519,7 +531,7 @@ impl AudioManager {debug!("reconciliation: skipping background sweep during active audio session");- return;+ return false;}}@@ -530,7 +542,7 @@ impl AudioManager {let batch_max_dur = opts.batch_max_duration_secs;drop(opts);- let count = super::reconciliation::reconcile_untranscribed(+ let sweep = super::reconciliation::reconcile_untranscribed(&db,transcription_engine,on_insert_bg.as_ref(),@@ -541,25 +553,48 @@ impl AudioManager {Some(metrics_bg.clone()),).await;+ let count = sweep.processed_chunks;if count > 0 {info!("reconciliation: transcribed {} orphaned chunks", count);}+ return sweep.hit_candidate_limit;}+ false}).catch_unwind().await;- if let Err(panic) = swept {- let reason = panic- .downcast_ref::<&str>()- .copied()- .or_else(|| panic.downcast_ref::<String>().map(String::as_str))- .unwrap_or("unknown cause");- error!(- "reconciliation: sweep panicked, worker continues: {}",- reason+ let swept_full_batch = match swept {+ Ok(full) => full,+ Err(panic) => {+ let reason = panic+ .downcast_ref::<&str>()+ .copied()+ .or_else(|| panic.downcast_ref::<String>().map(String::as_str))+ .unwrap_or("unknown cause");+ error!(+ "reconciliation: sweep panicked, worker continues: {}",+ reason+ );+ false+ }+ };++ if swept_full_batch+ && consecutive_full_sweeps + 1 < MAX_IMMEDIATE_RECONCILIATION_SWEEPS+ {+ consecutive_full_sweeps += 1;+ info!(+ "reconciliation: sweep hit the chunk cap; continuing backlog drain immediately");+ tokio::task::yield_now().await;+ continue;+ }++ consecutive_full_sweeps = 0;+ tokio::select! {+ _ = tokio::time::sleep(RECONCILIATION_IDLE_INTERVAL) => {}+ _ = reconciliation_wakeup.notified() => {}}- tokio::time::sleep(Duration::from_secs(120)).await;}});*self.reconciliation_handle.write().await = Some(handle);@@ -1153,6 +1188,7 @@ impl AudioManager {let meeting_audio_tap = self.meeting_audio_tap.clone();let db = self.db.clone();let on_insert_session = self.on_transcription_insert.clone();+ let reconciliation_wakeup = self.reconciliation_wakeup.clone();// Session streams (Meeting Tap, piggyback mic) bypass the meetings-only// drop gate below — they exist only during a meeting by construction.let session_devices = self.session_devices.clone();@@ -1406,7 +1442,7 @@ impl AudioManager {"batch mode: audio session ended, transcribing accumulated audio");let data_dir = output_path.as_deref();- let count = super::reconciliation::reconcile_untranscribed(+ let sweep = super::reconciliation::reconcile_untranscribed(&db,&engine,on_insert_session.as_ref(),@@ -1417,9 +1453,13 @@ impl AudioManager {Some(metrics.clone()),).await;+ let count = sweep.processed_chunks;for _ in 0..count {metrics.record_segment_batch_processed();}+ if sweep.hit_candidate_limit {+ reconciliation_wakeup.notify_one();+ }info!("batch mode: transcribed {} chunks", count);} else if now_in_session {if deferral_started.is_none() {@@ -2478,6 +2518,16 @@ mod tests {/// Real-model smoke for the release-only Apple Silicon MLX path. This is/// ignored because it requires the multi-gigabyte model to already be in/// the Hugging Face cache and the Xcode Metal toolchain to be installed.+ #[cfg(all(target_os = "macos", target_arch = "aarch64", feature = "parakeet-mlx"))]+ fn cached_parakeet_mlx_model_available() -> bool {+ const MODEL_REPO: &str = "mlx-community/parakeet-tdt-0.6b-v3";++ let cache = hf_hub::Cache::default().repo(hf_hub::Repo::model(MODEL_REPO.to_string()));+ cache.get("model.safetensors").is_some()+ && cache.get("config.json").is_some()+ && (cache.get("vocab.txt").is_some() || cache.get("tokenizer.model").is_some())+ }+#[cfg(all(target_os = "macos", target_arch = "aarch64", feature = "parakeet-mlx"))]#[tokio::test]#[ignore = "requires cached Parakeet MLX weights and the Xcode Metal toolchain"]@@ -2485,11 +2535,7 @@ mod tests {const MODEL_REPO: &str = "mlx-community/parakeet-tdt-0.6b-v3";const MAX_REUSE_DELTA_BYTES: usize = 64 * 1024 * 1024;- let cache = hf_hub::Cache::from_env().repo(hf_hub::Repo::model(MODEL_REPO.to_string()));- let model_cached = cache.get("model.safetensors").is_some()- && cache.get("config.json").is_some()- && (cache.get("vocab.txt").is_some() || cache.get("tokenizer.model").is_some());- if !model_cached {+ if !cached_parakeet_mlx_model_available() {eprintln!("skipping cached Parakeet MLX smoke: {MODEL_REPO} is not complete in the HF cache");diff --git a/crates/screenpipe-audio/src/audio_manager/reconciliation.rs b/crates/screenpipe-audio/src/audio_manager/reconciliation.rsindex 998f1035d..eb54ba81e 100644--- a/crates/screenpipe-audio/src/audio_manager/reconciliation.rs+++ b/crates/screenpipe-audio/src/audio_manager/reconciliation.rs@@ -33,6 +33,12 @@ const RECONCILIATION_LOOKBACK_HOURS: i64 = 24 * 7;const RECONCILIATION_FRESHNESS_DELAY_SECS: i64 = 10 * 60;const RECONCILIATION_CHUNKS_PER_SWEEP: i64 = 50;+#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]+pub struct ReconciliationSweep {+ pub processed_chunks: usize,+ pub hit_candidate_limit: bool,+}+use crate::core::engine::AudioTranscriptionEngine;use crate::metrics::AudioPipelineMetrics;use crate::segmentation::segmentation_manager::SegmentationManager;@@ -166,7 +172,8 @@ fn replace_with_merged_audio(/// This gives Whisper 2-5 minutes of context instead of 30s, significantly/// improving transcription quality — the whole point of batch mode.///-/// Returns the number of chunks successfully transcribed.+/// Returns both the number of chunks processed and whether the candidate query+/// hit its cap, which tells the scheduler to keep draining without a 120s nap.#[allow(clippy::too_many_arguments)]pub async fn reconcile_untranscribed(db: &DatabaseManager,@@ -177,7 +184,7 @@ pub async fn reconcile_untranscribed(data_dir: Option<&Path>,batch_max_duration_secs: Option<u64>,metrics: Option<Arc<AudioPipelineMetrics>>,-) -> usize {+) -> ReconciliationSweep {// Prevent concurrent reconciliation runs — two Whisper sessions = 200%+ CPU.// Acquired *before* the transcription-disabled check because orphaned-chunk// recovery (below) must run regardless of the engine, and still needs to be@@ -187,7 +194,7 @@ pub async fn reconcile_untranscribed(.is_err(){debug!("reconciliation: skipping — another reconciliation is already running");- return 0;+ return ReconciliationSweep::default();}// Ensure we always release the lockstruct Guard;@@ -211,7 +218,7 @@ pub async fn reconcile_untranscribed(// Transcription-specific reconciliation is skipped when transcription is// disabled — this also avoids the silent-audio deletion path nuking files.if *audio_engine == AudioTranscriptionEngine::Disabled {- return 0;+ return ReconciliationSweep::default();}// Retry any previously failed transcriptions before processing new chunks@@ -234,13 +241,14 @@ pub async fn reconcile_untranscribed("reconciliation: failed to query untranscribed chunks: {}",e);- return 0;+ return ReconciliationSweep::default();}};if chunks.is_empty() {- return 0;+ return ReconciliationSweep::default();}+ let hit_candidate_limit = chunks.len() >= RECONCILIATION_CHUNKS_PER_SWEEP as usize;debug!("reconciliation: found {} old untranscribed audio chunks (older_than={})",@@ -623,7 +631,10 @@ pub async fn reconcile_untranscribed(}}- success_count+ ReconciliationSweep {+ processed_chunks: success_count,+ hit_candidate_limit,+ }}/// Returns the path to the pending-transcriptions directory, creating it if needed.@@ -2387,4 +2398,205 @@ mod tests {let updated = backfill_missing_speakers(&db, segmentation_manager, 24, 50).await;assert_eq!(updated, 0);}++ #[cfg(all(target_os = "macos", target_arch = "aarch64", feature = "parakeet-mlx"))]+ fn cached_parakeet_mlx_model_available() -> bool {+ const MODEL_REPO: &str = "mlx-community/parakeet-tdt-0.6b-v3";++ let cache = hf_hub::Cache::default().repo(hf_hub::Repo::model(MODEL_REPO.to_string()));+ cache.get("model.safetensors").is_some()+ && cache.get("config.json").is_some()+ && (cache.get("vocab.txt").is_some() || cache.get("tokenizer.model").is_some())+ }++ #[cfg(all(target_os = "macos", target_arch = "aarch64", feature = "parakeet-mlx"))]+ #[allow(deprecated)]+ fn resident_size_bytes() -> u64 {+ unsafe {+ let mut info: libc::mach_task_basic_info = std::mem::zeroed();+ let mut count = libc::MACH_TASK_BASIC_INFO_COUNT;+ let result = libc::task_info(+ libc::mach_task_self(),+ libc::MACH_TASK_BASIC_INFO as libc::task_flavor_t,+ (&mut info as *mut libc::mach_task_basic_info).cast::<libc::integer_t>(),+ &mut count,+ );+ assert_eq!(result, 0, "task_info(MACH_TASK_BASIC_INFO) failed");+ std::ptr::addr_of!(info.resident_size).read_unaligned()+ }+ }++ #[cfg(all(target_os = "macos", target_arch = "aarch64", feature = "parakeet-mlx"))]+ fn mib(bytes: u64) -> f64 {+ bytes as f64 / 1024.0 / 1024.0+ }++ #[cfg(all(target_os = "macos", target_arch = "aarch64", feature = "parakeet-mlx"))]+ fn write_float_wav(path: &Path, samples: &[f32], sample_rate: u32) {+ let spec = hound::WavSpec {+ channels: 1,+ sample_rate,+ bits_per_sample: 32,+ sample_format: hound::SampleFormat::Float,+ };+ let mut writer = hound::WavWriter::create(path, spec).expect("create repro wav");+ for sample in samples {+ writer+ .write_sample(*sample)+ .expect("write repro wav sample");+ }+ writer.finalize().expect("finalize repro wav");+ }++ /// Full-path memory regression for the production reconciliation function:+ /// temporary SQLite DB, candidate query, ffmpeg decode, per-batch session,+ /// Parakeet MLX transcription, ONNX speaker diarization, and persistence.+ ///+ /// `cargo test --release -p screenpipe-audio --features parakeet-mlx cached_parakeet_mlx_reconciliation_memory_plateaus -- --ignored --nocapture --test-threads=1`+ #[cfg(all(target_os = "macos", target_arch = "aarch64", feature = "parakeet-mlx"))]+ #[tokio::test(flavor = "current_thread")]+ #[ignore = "real-model full reconciliation memory plateau regression"]+ async fn cached_parakeet_mlx_reconciliation_memory_plateaus() {+ const MODEL_REPO: &str = "mlx-community/parakeet-tdt-0.6b-v3";+ const WARMUP_CALLS: usize = 4;+ const CANDIDATES_PER_SWEEP: [usize; 4] = [50, 21, 21, 21];+ const SWEEP_COUNT: usize = CANDIDATES_PER_SWEEP.len();+ const MAX_POST_WARMUP_ENDPOINT_GROWTH_BYTES: u64 = 64 * 1024 * 1024;++ if !cached_parakeet_mlx_model_available() {+ eprintln!(+ "skipping cached Parakeet MLX reconciliation repro: {MODEL_REPO} is not complete in the HF cache"+ );+ return;+ }++ let fixture = Path::new(env!("CARGO_MANIFEST_DIR")).join("test_data/accuracy1.wav");+ let (audio, sample_rate) = read_audio_from_file(&fixture)+ .unwrap_or_else(|error| panic!("failed to decode {}: {error}", fixture.display()));+ let thirty_seconds = sample_rate as usize * 30;+ assert!(audio.len() >= thirty_seconds);++ let engine = TranscriptionEngine::new(+ Arc::new(AudioTranscriptionEngine::ParakeetMlx),+ None,+ None,+ vec![screenpipe_core::Language::English],+ Vec::new(),+ )+ .await+ .expect("failed to load cached Parakeet MLX model");++ for call in 1..=WARMUP_CALLS {+ let mut session = engine+ .create_session()+ .expect("failed to create Parakeet MLX warm-up session");+ session+ .transcribe(+ &audio[..thirty_seconds],+ sample_rate,+ "reconciliation-memory-warmup",+ )+ .await+ .unwrap_or_else(|error| panic!("warm-up transcription {call} failed: {error}"));+ }++ let temp = tempfile::tempdir().expect("create reconciliation repro dir");+ let db = temp_db(temp.path()).await;+ let model_dir = Path::new(env!("CARGO_MANIFEST_DIR"))+ .join("models")+ .join("pyannote");+ let embedding_model_path = model_dir.join("wespeaker_en_voxceleb_CAM++.onnx");+ let segmentation_model_path = model_dir.join("segmentation-3.0.onnx");+ let segmentation_manager = Arc::new(SegmentationManager {+ embedding_manager: Arc::new(std::sync::Mutex::new(+ crate::speaker::embedding_manager::EmbeddingManager::new(usize::MAX),+ )),+ embedding_extractor: tokio::sync::Mutex::new(Some(Arc::new(std::sync::Mutex::new(+ crate::speaker::embedding::EmbeddingExtractor::new(&embedding_model_path)+ .expect("load cached speaker embedding model"),+ )))),+ embedding_model_path: tokio::sync::Mutex::new(Some(embedding_model_path)),+ segmentation_model_path: tokio::sync::Mutex::new(Some(segmentation_model_path)),+ });+ let base_timestamp = Utc::now() - chrono::Duration::hours(6);+ let mut sweep_endpoints = Vec::with_capacity(SWEEP_COUNT);+ let mut next_index = 0usize;++ for (sweep, candidate_count) in CANDIDATES_PER_SWEEP.into_iter().enumerate() {+ for offset in 0..candidate_count {+ let index = next_index;+ next_index += 1;+ let duration_step = offset % 21;+ let sample_count =+ sample_rate as usize * 20 + duration_step * sample_rate as usize / 2;+ let path = temp.path().join(format!(+ "Display (output)_2026-07-23_{sweep:02}-{offset:02}-00.wav"+ ));+ write_float_wav(&path, &audio[..sample_count], sample_rate);+ let timestamp = base_timestamp + chrono::Duration::seconds(index as i64 * 65);+ db.get_or_insert_audio_chunk(&path.to_string_lossy(), Some(timestamp))+ .await+ .expect("insert reconciliation candidate");+ }++ let candidates = db+ .get_reconciliation_candidate_chunks(+ Utc::now() - chrono::Duration::days(7),+ Utc::now() - chrono::Duration::minutes(10),+ RECONCILIATION_CHUNKS_PER_SWEEP,+ )+ .await+ .expect("query reconciliation candidates");+ assert_eq!(candidates.len(), candidate_count);++ let sweep_start = resident_size_bytes();+ let result = reconcile_untranscribed(+ &db,+ &engine,+ None,+ Arc::new(AudioTranscriptionEngine::Parakeet),+ Some(segmentation_manager.clone()),+ Some(temp.path()),+ None,+ None,+ )+ .await;+ let sweep_end = resident_size_bytes();+ assert_eq!(result.processed_chunks, candidate_count);+ assert_eq!(+ result.hit_candidate_limit,+ candidate_count == RECONCILIATION_CHUNKS_PER_SWEEP as usize+ );+ sweep_endpoints.push(sweep_end);+ eprintln!(+ "[mlx-reconciliation-memory] sweep={}/{SWEEP_COUNT} reconciled={}/{candidate_count} rss_start={:.1} MiB rss_end={:.1} MiB working_set_delta={:.1} MiB active_mlx={:.1} MiB",+ sweep + 1,+ result.processed_chunks,+ mib(sweep_start),+ mib(sweep_end),+ mib(sweep_end.saturating_sub(sweep_start)),+ crate::transcription::engine::mlx_active_memory_bytes_for_test() as f64+ / 1024.0+ / 1024.0,+ );+ }++ // A sweep's start can be lower because macOS reclaims empty malloc pages+ // between passes. Compare completed sweep endpoints instead: a leak keeps+ // raising that high-water mark, while reusable working memory plateaus.+ let warmup_endpoint = sweep_endpoints[0];+ let post_warmup_peak = *sweep_endpoints[1..]+ .iter()+ .max()+ .expect("at least one measured reconciliation sweep");+ let endpoint_growth = post_warmup_peak.saturating_sub(warmup_endpoint);+ assert!(+ endpoint_growth <= MAX_POST_WARMUP_ENDPOINT_GROWTH_BYTES,+ "Parakeet MLX reconciliation endpoints did not plateau: {SWEEP_COUNT} real backlog \+ sweeps grew the post-warm-up high-water mark by \+ {:.1} MiB (limit: {:.1} MiB)",+ mib(endpoint_growth),+ mib(MAX_POST_WARMUP_ENDPOINT_GROWTH_BYTES),+ );+ }}