Skip to content

armgit / Cap

Public Imported from GitHub
0

Open source Loom alternative. Beautiful, shareable screen recordings.

Merge remote-tracking branch 'origin/main' into perf/parakeet-chunked-captions

Yash Rajb9acf0d9fb21
diff --cc apps/desktop-gpui/src/transcription.rs
index 38326aea5,9f23b4352..bc7ad223b
--- a/apps/desktop-gpui/src/transcription.rs
+++ b/apps/desktop-gpui/src/transcription.rs
@@@ -1581,57 -1602,41 +1634,70 @@@ fn process_with_parakeet
}
};
- let result = {
+ let mut reader = hound::WavReader::open(audio_path)
+ .map_err(|e| format!("Failed to open audio file: {e} at path: {audio_path:?}"))?;
+ let spec = reader.spec();
+ if spec.channels != 1 || spec.sample_rate != WHISPER_SAMPLE_RATE {
+ return Err(format!(
+ "Expected 16 kHz mono audio, got {} Hz with {} channels",
+ spec.sample_rate, spec.channels
+ ));
+ }
+ let sample_rate = spec.sample_rate;
+ let samples = reader
+ .samples::<i16>()
+ .map(|sample| sample.map(|sample| f32::from(sample) / 32768.0))
+ .collect::<Result<Vec<_>, _>>()
+ .map_err(|e| format!("Failed to read audio file: {e}"))?;
+ if samples.is_empty() {
+ return Err("No speech detected in the audio".to_string());
+ }
+
- let mut words = Vec::new();
++ let mut tokens = Vec::new();
+ {
let mut parakeet = model_arc
.lock()
.map_err(|e| format!("Failed to lock Parakeet model: {e}"))?;
- parakeet
- .transcribe_file(audio_path, Some(TimestampMode::Words))
- .map_err(|e| format!("Parakeet transcription failed: {e}"))?
- };
+ for range in parakeet_chunk_ranges(&samples, sample_rate as usize) {
+ let offset = range.start as f32 / sample_rate as f32;
+ let result = parakeet
+ .transcribe_samples(
+ samples[range].to_vec(),
+ sample_rate,
+ 1,
+ Some(TimestampMode::Words),
+ )
+ .map_err(|e| format!("Parakeet transcription failed: {e}"))?;
- words.extend(
- result
- .tokens
- .into_iter()
- .filter(|token| !token.text.trim().is_empty())
- .map(|token| CaptionWord {
- text: token.text.trim().to_string(),
- start: token.start + offset,
- end: token.end + offset,
- }),
- );
++ tokens.extend(result.tokens.into_iter().map(|token| CaptionWord {
++ text: token.text,
++ start: token.start + offset,
++ end: token.end + offset,
++ }));
+ }
+ }
- tracing::info!("Parakeet produced {} timed tokens", words.len());
- tracing::info!("Parakeet produced {} timed tokens", result.tokens.len());
++ tracing::info!("Parakeet produced {} timed tokens", tokens.len());
- let words = normalize_caption_words(words);
- caption_segments_from_tokens(result.tokens.iter().map(|token| CaptionWord {
- text: token.text.clone(),
- start: token.start,
- end: token.end,
- }))
++ caption_segments_from_tokens(tokens)
+ }
+
+ /// Parakeet's timed word tokens -> caption segments: drop empty tokens,
+ /// normalize the words and chunk them.
+ #[cfg(any(test, not(all(target_os = "macos", target_arch = "x86_64"))))]
+ fn caption_segments_from_tokens(
+ tokens: impl IntoIterator<Item = CaptionWord>,
+ ) -> Result<Vec<CaptionSegment>, String> {
+ let words = normalize_caption_words(
+ tokens
+ .into_iter()
+ .filter(|token| !token.text.trim().is_empty())
+ .map(|token| CaptionWord {
+ text: token.text.trim().to_string(),
+ start: token.start,
+ end: token.end,
+ })
+ .collect(),
+ );
if words.is_empty() {
return Err("No speech detected in the audio".to_string());