YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
Avoid retaining raw audio in reconciliation diarization (#5017)
Louis Beaumont367234909342
diff --git a/crates/screenpipe-audio/src/audio_manager/reconciliation.rs b/crates/screenpipe-audio/src/audio_manager/reconciliation.rsindex 423b7da46..998f1035d 100644--- a/crates/screenpipe-audio/src/audio_manager/reconciliation.rs+++ b/crates/screenpipe-audio/src/audio_manager/reconciliation.rs@@ -36,7 +36,7 @@ const RECONCILIATION_CHUNKS_PER_SWEEP: i64 = 50;use crate::core::engine::AudioTranscriptionEngine;use crate::metrics::AudioPipelineMetrics;use crate::segmentation::segmentation_manager::SegmentationManager;-use crate::speaker::segment::{get_segments, SpeechSegment};+use crate::speaker::segment::{get_segments_without_samples, SpeechSegment};use crate::transcription::engine::{TranscriptionEngine, TranscriptionSession};use crate::transcription::get_or_create_speaker_from_embedding;use crate::transcription::{AudioInsertCallback, AudioInsertInfo, TranscriptionDiarizationSegment};@@ -1275,7 +1275,7 @@ async fn extract_local_diarization_segments(}};- let segments = match get_segments(+ let segments = match get_segments_without_samples(samples,sample_rate,segmentation_model_path,@@ -1405,7 +1405,7 @@ async fn extract_speaker_id(}};- let segments = match get_segments(+ let segments = match get_segments_without_samples(samples,sample_rate,segmentation_model_path,diff --git a/crates/screenpipe-audio/src/speaker/segment.rs b/crates/screenpipe-audio/src/speaker/segment.rsindex 27bf7f802..a712752a5 100644--- a/crates/screenpipe-audio/src/speaker/segment.rs+++ b/crates/screenpipe-audio/src/speaker/segment.rs@@ -57,6 +57,7 @@ fn create_speech_segment_from_range(padded_samples: &[f32],embedding_extractor: Arc<Mutex<EmbeddingExtractor>>,embedding_manager: &Arc<Mutex<EmbeddingManager>>,+ retain_samples: bool,) -> Result<SpeechSegment> {let mut segment_vec;@@ -98,7 +99,11 @@ fn create_speech_segment_from_range(Ok(SpeechSegment {start: start_idx as f64 / sample_rate as f64,end: end_idx as f64 / sample_rate as f64,- samples: segment_samples.to_vec(),+ samples: if retain_samples {+ segment_samples.to_vec()+ } else {+ Vec::new()+ },speaker,embedding,sample_rate,@@ -109,16 +114,17 @@ fn create_speech_segments(start_offset: f64,offset: i32,sample_rate: u32,- samples: &[f32],+ sample_len: usize,padded_samples: &[f32],embedding_extractor: Arc<Mutex<EmbeddingExtractor>>,embedding_manager: &Arc<Mutex<EmbeddingManager>>,+ retain_samples: bool,) -> Result<Vec<SpeechSegment>> {let max_segment_samples = ((sample_rate as f64) * MAX_EMBEDDING_SEGMENT_SECONDS) as usize;let mut start_idx = start_offset.max(0.0)- .min((samples.len().saturating_sub(1)) as f64) as usize;- let end_idx = (offset.max(0) as usize).min(samples.len());+ .min((sample_len.saturating_sub(1)) as f64) as usize;+ let end_idx = (offset.max(0) as usize).min(sample_len);let mut segments = Vec::new();while start_idx < end_idx {@@ -140,6 +146,7 @@ fn create_speech_segments(padded_samples,embedding_extractor.clone(),embedding_manager,+ retain_samples,)?);start_idx = split_end_idx;}@@ -210,7 +217,7 @@ fn cached_seg_session(model_path: &Path) -> Result<Arc<SegSession>> {}pub struct SegmentIterator {- samples: Vec<f32>,+ sample_len: usize,sample_rate: u32,/// Shared, cached segmentation session (see `cached_seg_session`).seg: Arc<SegSession>,@@ -226,6 +233,7 @@ pub struct SegmentIterator {current_segment: Option<SpeechSegment>,pending_segments: VecDeque<SpeechSegment>,padded_samples: Vec<f32>,+ retain_samples: bool,}impl SegmentIterator {@@ -235,18 +243,33 @@ impl SegmentIterator {model_path: P,embedding_extractor: Arc<Mutex<EmbeddingExtractor>>,embedding_manager: Arc<Mutex<EmbeddingManager>>,+ ) -> Result<Self> {+ Self::new_with_sample_payload(+ samples,+ sample_rate,+ model_path,+ embedding_extractor,+ embedding_manager,+ true,+ )+ }++ fn new_with_sample_payload<P: AsRef<Path>>(+ mut samples: Vec<f32>,+ sample_rate: u32,+ model_path: P,+ embedding_extractor: Arc<Mutex<EmbeddingExtractor>>,+ embedding_manager: Arc<Mutex<EmbeddingManager>>,+ retain_samples: bool,) -> Result<Self> {let seg = cached_seg_session(model_path.as_ref())?;let window_size = (sample_rate * 10) as usize;+ let sample_len = samples.len();- let padded_samples = {- let mut padded = samples.clone();- padded.extend(vec![0.0; window_size - (samples.len() % window_size)]);- padded- };+ samples.extend(vec![0.0; window_size - (sample_len % window_size)]);Ok(Self {- samples,+ sample_len,sample_rate,seg,embedding_extractor,@@ -260,7 +283,8 @@ impl SegmentIterator {active_overlap: None,current_segment: None,pending_segments: VecDeque::new(),- padded_samples,+ padded_samples: samples,+ retain_samples,})}@@ -269,10 +293,11 @@ impl SegmentIterator {self.start_offset,self.offset,self.sample_rate,- &self.samples,+ self.sample_len,&self.padded_samples,self.embedding_extractor.clone(),&self.embedding_manager,+ self.retain_samples,)?;for new_segment in new_segments {@@ -408,6 +433,23 @@ pub fn get_segments<P: AsRef<Path>>()}+pub(crate) fn get_segments_without_samples<P: AsRef<Path>>(+ samples: &[f32],+ sample_rate: u32,+ model_path: P,+ embedding_extractor: Arc<Mutex<EmbeddingExtractor>>,+ embedding_manager: Arc<Mutex<EmbeddingManager>>,+) -> Result<SegmentIterator> {+ SegmentIterator::new_with_sample_payload(+ samples.to_vec(),+ sample_rate,+ model_path,+ embedding_extractor,+ embedding_manager,+ false,+ )+}+fn get_speaker_embedding(embedding_extractor: Arc<Mutex<EmbeddingExtractor>>,samples: &[f32],@@ -433,7 +475,18 @@ pub fn get_speaker_from_embedding(#[cfg(test)]mod tests {- use super::is_overlap_class;+ use super::{handle_new_segment, is_overlap_class, SpeechSegment};++ fn test_segment(start: f64, end: f64, sample_count: usize, speaker: &str) -> SpeechSegment {+ SpeechSegment {+ start,+ end,+ samples: vec![0.25; sample_count],+ speaker: speaker.to_string(),+ embedding: vec![0.0; 192],+ sample_rate: 16_000,+ }+ }#[test]fn pyannote_powerset_overlap_classes_start_after_single_speaker_classes() {@@ -445,4 +498,30 @@ mod tests {assert!(is_overlap_class(5));assert!(is_overlap_class(6));}++ #[test]+ fn metadata_only_same_speaker_merge_retains_no_audio_payload() {+ let segment_count = 50usize;++ let mut current_segment = None;+ let mut flushed_segments = Vec::new();+ let mut start = 0.0;++ for _ in 0..segment_count {+ let end = start + 2.0;+ let segment = test_segment(start, end, 0, "speaker-1");+ current_segment = handle_new_segment(current_segment, segment, &mut flushed_segments);++ let current_samples = current_segment+ .as_ref()+ .map(|segment| segment.samples.len())+ .unwrap_or_default();+ assert_eq!(current_samples, 0);++ start = end + 0.01;+ }++ assert!(flushed_segments.is_empty());+ assert_eq!(current_segment.unwrap().samples.len(), 0);+ }}