Skip to content

armgit / Cap

Public Imported from GitHub
0

Open source Loom alternative. Beautiful, shareable screen recordings.

fix(desktop-gpui): transcribe kept ranges in one model run

Aditya Garud4ce9922ca0b2
diff --git a/apps/desktop-gpui/src/transcription.rs b/apps/desktop-gpui/src/transcription.rs
index d6ba04c84..4fe8db857 100644
--- a/apps/desktop-gpui/src/transcription.rs
+++ b/apps/desktop-gpui/src/transcription.rs
@@ -980,7 +980,7 @@ fn extract_project_audio(
studio: &StudioRecordingMeta,
recording_durations: &[f64],
range: Range<f64>,
- output_path: &Path,
+ final_samples: &mut Vec<f32>,
) -> Result<bool, String> {
let tracks: Vec<Vec<(&cap_project::AudioMeta, f64)>> = match studio {
StudioRecordingMeta::SingleSegment { segment } => {
@@ -1007,7 +1007,7 @@ fn extract_project_audio(
if tracks.len() != recording_durations.len() {
return Err("Recording durations do not match the recording metadata".to_string());
}
- let mut final_samples = Vec::new();
+ let initial_len = final_samples.len();
let mut any_audio = false;
let mut take_start = 0.0;
for (tracks, duration) in tracks.into_iter().zip(recording_durations) {
@@ -1038,20 +1038,13 @@ fn extract_project_audio(
display_duration_secs: end - start,
sources,
};
- append_transcription_audio(&mut final_samples, &take, DECODE_SAMPLE_RATE)
+ append_transcription_audio(final_samples, &take, DECODE_SAMPLE_RATE)
.map_err(|error| format!("Failed to assemble transcription audio: {error}"))?;
}
if !any_audio {
- return Ok(false);
+ final_samples.truncate(initial_len);
}
-
- let gain = normalize_audio_for_transcription(&mut final_samples);
- if (gain - 1.0).abs() > 0.01 {
- tracing::info!("Applied transcription audio gain: {gain:.2}x");
- }
-
- write_wav_16k_mono(&final_samples, output_path)?;
- Ok(true)
+ Ok(any_audio)
}
/// The WAV encode half of the recording-directory arm
@@ -1551,6 +1544,63 @@ fn process_with_parakeet(
static TEMP_COUNTER: AtomicU64 = AtomicU64::new(0);
+struct TranscriptionAudioOffset {
+ concatenated_start: f64,
+ source_start: f64,
+ length: f64,
+}
+
+fn restore_source_timestamps(
+ segments: Vec<CaptionSegment>,
+ offsets: &[TranscriptionAudioOffset],
+) -> Vec<CaptionSegment> {
+ let map_times = |start: f32, end: f32| {
+ let start = f64::from(start);
+ let index = offsets
+ .partition_point(|offset| offset.concatenated_start <= start)
+ .checked_sub(1)?;
+ let offset = &offsets[index];
+ let local_start = start - offset.concatenated_start;
+ if local_start >= offset.length {
+ return None;
+ }
+ let local_end =
+ (f64::from(end) - offset.concatenated_start).clamp(local_start, offset.length);
+ Some((
+ (offset.source_start + local_start) as f32,
+ (offset.source_start + local_end) as f32,
+ ))
+ };
+ segments
+ .into_iter()
+ .filter_map(|mut segment| {
+ let word_count = segment.words.len();
+ segment.words = segment
+ .words
+ .into_iter()
+ .filter_map(|mut word| {
+ (word.start, word.end) = map_times(word.start, word.end)?;
+ Some(word)
+ })
+ .collect();
+ if word_count != segment.words.len() {
+ segment.text = caption_text_from_words(&segment.words);
+ }
+ (segment.start, segment.end) = match (segment.words.first(), segment.words.last()) {
+ (Some(first), Some(last)) => (first.start, last.end),
+ _ if word_count == 0 => map_times(segment.start, segment.end)?,
+ _ => return None,
+ };
+ Some(segment)
+ })
+ .enumerate()
+ .map(|(index, mut segment)| {
+ segment.id = format!("segment-{index}");
+ segment
+ })
+ .collect()
+}
+
/// `transcribe_audio` (`captions.rs:1315-1439`), minus the Tauri command
/// plumbing. Blocking -- run it on the tokio blocking pool.
pub fn transcribe_blocking(
@@ -1601,7 +1651,7 @@ fn transcribe_project_with(
project_path: &Path,
timeline: Option<&TimelineConfiguration>,
recording_durations: &[f64],
- mut engine: impl FnMut(&Path) -> Result<Vec<CaptionSegment>, String>,
+ engine: impl FnOnce(&Path) -> Result<Vec<CaptionSegment>, String>,
) -> Result<Vec<CaptionSegment>, String> {
ffmpeg::init().map_err(|e| format!("Failed to initialise ffmpeg: {e}"))?;
let meta = RecordingMeta::load_for_project(project_path)
@@ -1620,34 +1670,35 @@ fn transcribe_project_with(
let audio_path = temp_dir.join("audio.wav");
let result = (|| {
- let mut segments = Vec::new();
- let mut any_audio = false;
+ let mut samples = Vec::new();
+ let mut offsets = Vec::new();
for range in cap_project::transcription_source_ranges(timeline, recording_durations) {
+ let start = samples.len();
if !extract_project_audio(
&meta,
studio,
recording_durations,
range.clone(),
- &audio_path,
+ &mut samples,
)? {
continue;
}
- any_audio = true;
- for mut segment in engine(&audio_path)? {
- segment.id = format!("segment-{}", segments.len());
- segment.start += range.start as f32;
- segment.end += range.start as f32;
- for word in &mut segment.words {
- word.start += range.start as f32;
- word.end += range.start as f32;
- }
- segments.push(segment);
- }
+ offsets.push(TranscriptionAudioOffset {
+ concatenated_start: start as f64 / f64::from(DECODE_SAMPLE_RATE),
+ source_start: range.start,
+ length: (samples.len() - start) as f64 / f64::from(DECODE_SAMPLE_RATE),
+ });
}
- if !any_audio {
+ if offsets.is_empty() {
return Err("No audio sources found in the selected timeline".to_string());
}
- Ok(segments)
+ let gain = normalize_audio_for_transcription(&mut samples);
+ if (gain - 1.0).abs() > 0.01 {
+ tracing::info!("Applied transcription audio gain: {gain:.2}x");
+ }
+ write_wav_16k_mono(&samples, &audio_path)?;
+ drop(samples);
+ Ok(restore_source_timestamps(engine(&audio_path)?, &offsets))
})();
let _ = std::fs::remove_dir_all(&temp_dir);
diff --git a/apps/desktop-gpui/src/transcription/verify_tests.rs b/apps/desktop-gpui/src/transcription/verify_tests.rs
index e00f2dbf8..f247012ba 100644
--- a/apps/desktop-gpui/src/transcription/verify_tests.rs
+++ b/apps/desktop-gpui/src/transcription/verify_tests.rs
@@ -690,29 +690,41 @@ fn trimmed_transcription_extracts_only_used_audio_and_restores_source_times() {
.unwrap();
let mut called = 0;
let segments = transcribe_project_with(&project, Some(&timeline), &durations, |path| {
- let range = &expected_ranges[called];
+ let expected_duration: f64 = expected_ranges
+ .iter()
+ .map(|range| range.end - range.start)
+ .sum();
let audio = media::decode_audio(path, Some(MARKER_RATE)).unwrap();
let duration = audio.mono().len() as f64 / f64::from(MARKER_RATE);
assert!(
- (duration - (range.end - range.start)).abs() < 0.002,
- "extracted {duration}s, expected {}s",
- range.end - range.start
+ (duration - expected_duration).abs() < 0.002,
+ "extracted {duration}s, expected {expected_duration}s"
);
called += 1;
- Ok(vec![CaptionSegment {
- id: "segment-0".into(),
- start: 0.25,
- end: 0.5,
- text: "word".into(),
- words: vec![CaptionWord {
- text: "word".into(),
- start: 0.25,
- end: 0.5,
- }],
- }])
+ let mut offset = 0.0;
+ Ok(expected_ranges
+ .iter()
+ .map(|range| {
+ let start = offset + 0.25;
+ let end = offset + 0.5;
+ offset += (range.end - range.start) as f32;
+ CaptionSegment {
+ id: "segment-0".into(),
+ start,
+ end,
+ text: "word".into(),
+ words: vec![CaptionWord {
+ text: "word".into(),
+ start,
+ end,
+ }],
+ }
+ })
+ .collect())
})
.unwrap();
- assert_eq!(called, expected_ranges.len());
+ assert_eq!(called, 1);
+ assert_eq!(segments.len(), expected_ranges.len());
for (segment, range) in segments.iter().zip(expected_ranges) {
assert!((f64::from(segment.start) - range.start - 0.25).abs() < 1e-5);
assert!((f64::from(segment.end) - range.start - 0.5).abs() < 1e-5);
@@ -800,17 +812,14 @@ fn transcription_keeps_speech_after_empty_ranges_and_cleans_up_on_failure() {
let result = transcribe_project_with(&project, Some(&timeline), &durations, |path| {
temporary = path.to_path_buf();
calls += 1;
- if calls == 1 {
- return Ok(Vec::new());
- }
Ok(caption_segments_from_tokens([CaptionWord {
text: "speech".into(),
- start: 0.1,
- end: 0.2,
+ start: 1.1,
+ end: 1.2,
}]))
})
.unwrap();
- assert_eq!(calls, 2);
+ assert_eq!(calls, 1);
assert!((result[0].words[0].start - 3.1).abs() < 1e-5);
assert!(!temporary.parent().unwrap().exists());
let error = transcribe_project_with(&project, Some(&timeline), &durations, |path| {
@@ -832,3 +841,133 @@ fn transcription_keeps_speech_after_empty_ranges_and_cleans_up_on_failure() {
.unwrap_err();
assert!(error.contains("No audio sources"));
}
+
+#[test]
+fn source_timestamps_restore_words_in_two_ranges() {
+ let offsets = [
+ TranscriptionAudioOffset {
+ concatenated_start: 0.0,
+ source_start: 10.0,
+ length: 2.0,
+ },
+ TranscriptionAudioOffset {
+ concatenated_start: 2.0,
+ source_start: 20.0,
+ length: 3.0,
+ },
+ ];
+ let segments = restore_source_timestamps(
+ caption_segments_from_tokens([
+ CaptionWord {
+ text: "first".into(),
+ start: 0.25,
+ end: 0.5,
+ },
+ CaptionWord {
+ text: "second".into(),
+ start: 2.25,
+ end: 2.5,
+ },
+ ]),
+ &offsets,
+ );
+ assert_eq!(segments.len(), 1);
+ let segment = &segments[0];
+ assert_eq!((segment.start, segment.end), (10.25, 20.5));
+ assert_eq!(
+ (segment.words[0].start, segment.words[0].end),
+ (10.25, 10.5)
+ );
+ assert_eq!(
+ (segment.words[1].start, segment.words[1].end),
+ (20.25, 20.5)
+ );
+}
+
+#[test]
+fn source_timestamps_clamp_words_and_wordless_segments_at_junctions() {
+ let offsets = [
+ TranscriptionAudioOffset {
+ concatenated_start: 0.0,
+ source_start: 10.0,
+ length: 2.0,
+ },
+ TranscriptionAudioOffset {
+ concatenated_start: 2.0,
+ source_start: 20.0,
+ length: 3.0,
+ },
+ ];
+ let mut segments = caption_segments_from_tokens([
+ CaptionWord {
+ text: "across".into(),
+ start: 1.75,
+ end: 2.25,
+ },
+ CaptionWord {
+ text: "junction".into(),
+ start: 2.0,
+ end: 2.25,
+ },
+ ]);
+ segments.push(CaptionSegment {
+ start: 1.75,
+ end: 2.25,
+ text: "no words".into(),
+ ..Default::default()
+ });
+ let segments = restore_source_timestamps(segments, &offsets);
+ assert_eq!(segments.len(), 2);
+ assert_eq!(
+ (segments[0].words[0].start, segments[0].words[0].end),
+ (11.75, 12.0)
+ );
+ assert_eq!(
+ (segments[0].words[1].start, segments[0].words[1].end),
+ (20.0, 20.25)
+ );
+ assert_eq!((segments[0].start, segments[0].end), (11.75, 20.25));
+ assert_eq!((segments[1].start, segments[1].end), (11.75, 12.0));
+}
+
+#[test]
+fn source_timestamps_for_one_range_match_a_source_offset() {
+ let offsets = [TranscriptionAudioOffset {
+ concatenated_start: 0.0,
+ source_start: 37.0,
+ length: 4.0,
+ }];
+ let segments = caption_segments_from_tokens([CaptionWord {
+ text: "word".into(),
+ start: 0.25,
+ end: 0.5,
+ }]);
+ let mut reference = segments.clone();
+ for segment in &mut reference {
+ segment.start += 37.0;
+ segment.end += 37.0;
+ for word in &mut segment.words {
+ word.start += 37.0;
+ word.end += 37.0;
+ }
+ }
+ let segments = restore_source_timestamps(segments, &offsets);
+ assert!(compare_segments(&segments, &reference, 0.0).is_empty());
+}
+
+#[test]
+fn source_timestamps_with_empty_ranges_produce_no_captions() {
+ let mut segments = caption_segments_from_tokens([CaptionWord {
+ text: "word".into(),
+ start: 0.25,
+ end: 0.5,
+ }]);
+ segments.push(CaptionSegment {
+ start: 0.25,
+ end: 0.5,
+ text: "no words".into(),
+ ..Default::default()
+ });
+ assert!(restore_source_timestamps(segments, &[]).is_empty());
+ assert!(restore_source_timestamps(Vec::new(), &[]).is_empty());
+}