Open source Loom alternative. Beautiful, shareable screen recordings.
Rust appcapcossloommacnextjsnextjs15open-sourceossreactrecordrustscreen-capturescreen-recorderscreenshotsolidjstauritauri-apptypescriptvite
Merge remote-tracking branch 'origin/main' into perf/parakeet-chunked-captions
Yash Rajb9acf0d9fb21
diff --cc apps/desktop-gpui/src/transcription.rsindex 38326aea5,9f23b4352..bc7ad223b--- a/apps/desktop-gpui/src/transcription.rs+++ b/apps/desktop-gpui/src/transcription.rs@@@ -1581,57 -1602,41 +1634,70 @@@ fn process_with_parakeet}};- let result = {+ let mut reader = hound::WavReader::open(audio_path)+ .map_err(|e| format!("Failed to open audio file: {e} at path: {audio_path:?}"))?;+ let spec = reader.spec();+ if spec.channels != 1 || spec.sample_rate != WHISPER_SAMPLE_RATE {+ return Err(format!(+ "Expected 16 kHz mono audio, got {} Hz with {} channels",+ spec.sample_rate, spec.channels+ ));+ }+ let sample_rate = spec.sample_rate;+ let samples = reader+ .samples::<i16>()+ .map(|sample| sample.map(|sample| f32::from(sample) / 32768.0))+ .collect::<Result<Vec<_>, _>>()+ .map_err(|e| format!("Failed to read audio file: {e}"))?;+ if samples.is_empty() {+ return Err("No speech detected in the audio".to_string());+ }+- let mut words = Vec::new();++ let mut tokens = Vec::new();+ {let mut parakeet = model_arc.lock().map_err(|e| format!("Failed to lock Parakeet model: {e}"))?;- parakeet- .transcribe_file(audio_path, Some(TimestampMode::Words))- .map_err(|e| format!("Parakeet transcription failed: {e}"))?- };+ for range in parakeet_chunk_ranges(&samples, sample_rate as usize) {+ let offset = range.start as f32 / sample_rate as f32;+ let result = parakeet+ .transcribe_samples(+ samples[range].to_vec(),+ sample_rate,+ 1,+ Some(TimestampMode::Words),+ )+ .map_err(|e| format!("Parakeet transcription failed: {e}"))?;- words.extend(- result- .tokens- .into_iter()- .filter(|token| !token.text.trim().is_empty())- .map(|token| CaptionWord {- text: token.text.trim().to_string(),- start: token.start + offset,- end: token.end + offset,- }),- );++ tokens.extend(result.tokens.into_iter().map(|token| CaptionWord {++ text: token.text,++ start: token.start + offset,++ end: token.end + offset,++ }));+ }+ }- tracing::info!("Parakeet produced {} timed tokens", words.len());- tracing::info!("Parakeet produced {} timed tokens", result.tokens.len());++ tracing::info!("Parakeet produced {} timed tokens", tokens.len());- let words = normalize_caption_words(words);- caption_segments_from_tokens(result.tokens.iter().map(|token| CaptionWord {- text: token.text.clone(),- start: token.start,- end: token.end,- }))++ caption_segments_from_tokens(tokens)+ }++ /// Parakeet's timed word tokens -> caption segments: drop empty tokens,+ /// normalize the words and chunk them.+ #[cfg(any(test, not(all(target_os = "macos", target_arch = "x86_64"))))]+ fn caption_segments_from_tokens(+ tokens: impl IntoIterator<Item = CaptionWord>,+ ) -> Result<Vec<CaptionSegment>, String> {+ let words = normalize_caption_words(+ tokens+ .into_iter()+ .filter(|token| !token.text.trim().is_empty())+ .map(|token| CaptionWord {+ text: token.text.trim().to_string(),+ start: token.start,+ end: token.end,+ })+ .collect(),+ );if words.is_empty() {return Err("No speech detected in the audio".to_string());