Open source Loom alternative. Beautiful, shareable screen recordings.
Rust appcapcossloommacnextjsnextjs15open-sourceossreactrecordrustscreen-capturescreen-recorderscreenshotsolidjstauritauri-apptypescriptvite
fix(desktop-gpui): transcribe kept ranges in one model run
Aditya Garud4ce9922ca0b2
diff --git a/apps/desktop-gpui/src/transcription.rs b/apps/desktop-gpui/src/transcription.rsindex d6ba04c84..4fe8db857 100644--- a/apps/desktop-gpui/src/transcription.rs+++ b/apps/desktop-gpui/src/transcription.rs@@ -980,7 +980,7 @@ fn extract_project_audio(studio: &StudioRecordingMeta,recording_durations: &[f64],range: Range<f64>,- output_path: &Path,+ final_samples: &mut Vec<f32>,) -> Result<bool, String> {let tracks: Vec<Vec<(&cap_project::AudioMeta, f64)>> = match studio {StudioRecordingMeta::SingleSegment { segment } => {@@ -1007,7 +1007,7 @@ fn extract_project_audio(if tracks.len() != recording_durations.len() {return Err("Recording durations do not match the recording metadata".to_string());}- let mut final_samples = Vec::new();+ let initial_len = final_samples.len();let mut any_audio = false;let mut take_start = 0.0;for (tracks, duration) in tracks.into_iter().zip(recording_durations) {@@ -1038,20 +1038,13 @@ fn extract_project_audio(display_duration_secs: end - start,sources,};- append_transcription_audio(&mut final_samples, &take, DECODE_SAMPLE_RATE)+ append_transcription_audio(final_samples, &take, DECODE_SAMPLE_RATE).map_err(|error| format!("Failed to assemble transcription audio: {error}"))?;}if !any_audio {- return Ok(false);+ final_samples.truncate(initial_len);}-- let gain = normalize_audio_for_transcription(&mut final_samples);- if (gain - 1.0).abs() > 0.01 {- tracing::info!("Applied transcription audio gain: {gain:.2}x");- }-- write_wav_16k_mono(&final_samples, output_path)?;- Ok(true)+ Ok(any_audio)}/// The WAV encode half of the recording-directory arm@@ -1551,6 +1544,63 @@ fn process_with_parakeet(static TEMP_COUNTER: AtomicU64 = AtomicU64::new(0);+struct TranscriptionAudioOffset {+ concatenated_start: f64,+ source_start: f64,+ length: f64,+}++fn restore_source_timestamps(+ segments: Vec<CaptionSegment>,+ offsets: &[TranscriptionAudioOffset],+) -> Vec<CaptionSegment> {+ let map_times = |start: f32, end: f32| {+ let start = f64::from(start);+ let index = offsets+ .partition_point(|offset| offset.concatenated_start <= start)+ .checked_sub(1)?;+ let offset = &offsets[index];+ let local_start = start - offset.concatenated_start;+ if local_start >= offset.length {+ return None;+ }+ let local_end =+ (f64::from(end) - offset.concatenated_start).clamp(local_start, offset.length);+ Some((+ (offset.source_start + local_start) as f32,+ (offset.source_start + local_end) as f32,+ ))+ };+ segments+ .into_iter()+ .filter_map(|mut segment| {+ let word_count = segment.words.len();+ segment.words = segment+ .words+ .into_iter()+ .filter_map(|mut word| {+ (word.start, word.end) = map_times(word.start, word.end)?;+ Some(word)+ })+ .collect();+ if word_count != segment.words.len() {+ segment.text = caption_text_from_words(&segment.words);+ }+ (segment.start, segment.end) = match (segment.words.first(), segment.words.last()) {+ (Some(first), Some(last)) => (first.start, last.end),+ _ if word_count == 0 => map_times(segment.start, segment.end)?,+ _ => return None,+ };+ Some(segment)+ })+ .enumerate()+ .map(|(index, mut segment)| {+ segment.id = format!("segment-{index}");+ segment+ })+ .collect()+}+/// `transcribe_audio` (`captions.rs:1315-1439`), minus the Tauri command/// plumbing. Blocking -- run it on the tokio blocking pool.pub fn transcribe_blocking(@@ -1601,7 +1651,7 @@ fn transcribe_project_with(project_path: &Path,timeline: Option<&TimelineConfiguration>,recording_durations: &[f64],- mut engine: impl FnMut(&Path) -> Result<Vec<CaptionSegment>, String>,+ engine: impl FnOnce(&Path) -> Result<Vec<CaptionSegment>, String>,) -> Result<Vec<CaptionSegment>, String> {ffmpeg::init().map_err(|e| format!("Failed to initialise ffmpeg: {e}"))?;let meta = RecordingMeta::load_for_project(project_path)@@ -1620,34 +1670,35 @@ fn transcribe_project_with(let audio_path = temp_dir.join("audio.wav");let result = (|| {- let mut segments = Vec::new();- let mut any_audio = false;+ let mut samples = Vec::new();+ let mut offsets = Vec::new();for range in cap_project::transcription_source_ranges(timeline, recording_durations) {+ let start = samples.len();if !extract_project_audio(&meta,studio,recording_durations,range.clone(),- &audio_path,+ &mut samples,)? {continue;}- any_audio = true;- for mut segment in engine(&audio_path)? {- segment.id = format!("segment-{}", segments.len());- segment.start += range.start as f32;- segment.end += range.start as f32;- for word in &mut segment.words {- word.start += range.start as f32;- word.end += range.start as f32;- }- segments.push(segment);- }+ offsets.push(TranscriptionAudioOffset {+ concatenated_start: start as f64 / f64::from(DECODE_SAMPLE_RATE),+ source_start: range.start,+ length: (samples.len() - start) as f64 / f64::from(DECODE_SAMPLE_RATE),+ });}- if !any_audio {+ if offsets.is_empty() {return Err("No audio sources found in the selected timeline".to_string());}- Ok(segments)+ let gain = normalize_audio_for_transcription(&mut samples);+ if (gain - 1.0).abs() > 0.01 {+ tracing::info!("Applied transcription audio gain: {gain:.2}x");+ }+ write_wav_16k_mono(&samples, &audio_path)?;+ drop(samples);+ Ok(restore_source_timestamps(engine(&audio_path)?, &offsets))})();let _ = std::fs::remove_dir_all(&temp_dir);diff --git a/apps/desktop-gpui/src/transcription/verify_tests.rs b/apps/desktop-gpui/src/transcription/verify_tests.rsindex e00f2dbf8..f247012ba 100644--- a/apps/desktop-gpui/src/transcription/verify_tests.rs+++ b/apps/desktop-gpui/src/transcription/verify_tests.rs@@ -690,29 +690,41 @@ fn trimmed_transcription_extracts_only_used_audio_and_restores_source_times() {.unwrap();let mut called = 0;let segments = transcribe_project_with(&project, Some(&timeline), &durations, |path| {- let range = &expected_ranges[called];+ let expected_duration: f64 = expected_ranges+ .iter()+ .map(|range| range.end - range.start)+ .sum();let audio = media::decode_audio(path, Some(MARKER_RATE)).unwrap();let duration = audio.mono().len() as f64 / f64::from(MARKER_RATE);assert!(- (duration - (range.end - range.start)).abs() < 0.002,- "extracted {duration}s, expected {}s",- range.end - range.start+ (duration - expected_duration).abs() < 0.002,+ "extracted {duration}s, expected {expected_duration}s");called += 1;- Ok(vec![CaptionSegment {- id: "segment-0".into(),- start: 0.25,- end: 0.5,- text: "word".into(),- words: vec![CaptionWord {- text: "word".into(),- start: 0.25,- end: 0.5,- }],- }])+ let mut offset = 0.0;+ Ok(expected_ranges+ .iter()+ .map(|range| {+ let start = offset + 0.25;+ let end = offset + 0.5;+ offset += (range.end - range.start) as f32;+ CaptionSegment {+ id: "segment-0".into(),+ start,+ end,+ text: "word".into(),+ words: vec![CaptionWord {+ text: "word".into(),+ start,+ end,+ }],+ }+ })+ .collect())}).unwrap();- assert_eq!(called, expected_ranges.len());+ assert_eq!(called, 1);+ assert_eq!(segments.len(), expected_ranges.len());for (segment, range) in segments.iter().zip(expected_ranges) {assert!((f64::from(segment.start) - range.start - 0.25).abs() < 1e-5);assert!((f64::from(segment.end) - range.start - 0.5).abs() < 1e-5);@@ -800,17 +812,14 @@ fn transcription_keeps_speech_after_empty_ranges_and_cleans_up_on_failure() {let result = transcribe_project_with(&project, Some(&timeline), &durations, |path| {temporary = path.to_path_buf();calls += 1;- if calls == 1 {- return Ok(Vec::new());- }Ok(caption_segments_from_tokens([CaptionWord {text: "speech".into(),- start: 0.1,- end: 0.2,+ start: 1.1,+ end: 1.2,}]))}).unwrap();- assert_eq!(calls, 2);+ assert_eq!(calls, 1);assert!((result[0].words[0].start - 3.1).abs() < 1e-5);assert!(!temporary.parent().unwrap().exists());let error = transcribe_project_with(&project, Some(&timeline), &durations, |path| {@@ -832,3 +841,133 @@ fn transcription_keeps_speech_after_empty_ranges_and_cleans_up_on_failure() {.unwrap_err();assert!(error.contains("No audio sources"));}++#[test]+fn source_timestamps_restore_words_in_two_ranges() {+ let offsets = [+ TranscriptionAudioOffset {+ concatenated_start: 0.0,+ source_start: 10.0,+ length: 2.0,+ },+ TranscriptionAudioOffset {+ concatenated_start: 2.0,+ source_start: 20.0,+ length: 3.0,+ },+ ];+ let segments = restore_source_timestamps(+ caption_segments_from_tokens([+ CaptionWord {+ text: "first".into(),+ start: 0.25,+ end: 0.5,+ },+ CaptionWord {+ text: "second".into(),+ start: 2.25,+ end: 2.5,+ },+ ]),+ &offsets,+ );+ assert_eq!(segments.len(), 1);+ let segment = &segments[0];+ assert_eq!((segment.start, segment.end), (10.25, 20.5));+ assert_eq!(+ (segment.words[0].start, segment.words[0].end),+ (10.25, 10.5)+ );+ assert_eq!(+ (segment.words[1].start, segment.words[1].end),+ (20.25, 20.5)+ );+}++#[test]+fn source_timestamps_clamp_words_and_wordless_segments_at_junctions() {+ let offsets = [+ TranscriptionAudioOffset {+ concatenated_start: 0.0,+ source_start: 10.0,+ length: 2.0,+ },+ TranscriptionAudioOffset {+ concatenated_start: 2.0,+ source_start: 20.0,+ length: 3.0,+ },+ ];+ let mut segments = caption_segments_from_tokens([+ CaptionWord {+ text: "across".into(),+ start: 1.75,+ end: 2.25,+ },+ CaptionWord {+ text: "junction".into(),+ start: 2.0,+ end: 2.25,+ },+ ]);+ segments.push(CaptionSegment {+ start: 1.75,+ end: 2.25,+ text: "no words".into(),+ ..Default::default()+ });+ let segments = restore_source_timestamps(segments, &offsets);+ assert_eq!(segments.len(), 2);+ assert_eq!(+ (segments[0].words[0].start, segments[0].words[0].end),+ (11.75, 12.0)+ );+ assert_eq!(+ (segments[0].words[1].start, segments[0].words[1].end),+ (20.0, 20.25)+ );+ assert_eq!((segments[0].start, segments[0].end), (11.75, 20.25));+ assert_eq!((segments[1].start, segments[1].end), (11.75, 12.0));+}++#[test]+fn source_timestamps_for_one_range_match_a_source_offset() {+ let offsets = [TranscriptionAudioOffset {+ concatenated_start: 0.0,+ source_start: 37.0,+ length: 4.0,+ }];+ let segments = caption_segments_from_tokens([CaptionWord {+ text: "word".into(),+ start: 0.25,+ end: 0.5,+ }]);+ let mut reference = segments.clone();+ for segment in &mut reference {+ segment.start += 37.0;+ segment.end += 37.0;+ for word in &mut segment.words {+ word.start += 37.0;+ word.end += 37.0;+ }+ }+ let segments = restore_source_timestamps(segments, &offsets);+ assert!(compare_segments(&segments, &reference, 0.0).is_empty());+}++#[test]+fn source_timestamps_with_empty_ranges_produce_no_captions() {+ let mut segments = caption_segments_from_tokens([CaptionWord {+ text: "word".into(),+ start: 0.25,+ end: 0.5,+ }]);+ segments.push(CaptionSegment {+ start: 0.25,+ end: 0.5,+ text: "no words".into(),+ ..Default::default()+ });+ assert!(restore_source_timestamps(segments, &[]).is_empty());+ assert!(restore_source_timestamps(Vec::new(), &[]).is_empty());+}