Skip to content

armgit / Cap

Public Imported from GitHub
0

Open source Loom alternative. Beautiful, shareable screen recordings.

fix(desktop-gpui): transcribe only retained source audio

Aditya Garudfbbe4166047e
diff --git a/apps/desktop-gpui/src/editor_tabs.rs b/apps/desktop-gpui/src/editor_tabs.rs
index a79364748..3a808180d 100644
--- a/apps/desktop-gpui/src/editor_tabs.rs
+++ b/apps/desktop-gpui/src/editor_tabs.rs
@@ -3176,6 +3176,7 @@ impl EditorWindow {
};
let recording_durations = summary.clip_display_durations.clone();
let recording_duration = summary.recording_duration;
+ let timeline = self.project.timeline.clone();
let project_path = self.project_path.clone();
let model = self.selected_caption_model().name;
let language = self.sidebar.caption_language;
@@ -3187,9 +3188,16 @@ impl EditorWindow {
cx.spawn_in(window, async move |this, cx| {
let task_path = project_path.clone();
+ let task_durations = recording_durations.clone();
let task = gpui_tokio::Tokio::spawn(cx, async move {
tokio::task::spawn_blocking(move || {
- transcription::transcribe_blocking(&task_path, model, language)
+ transcription::transcribe_blocking(
+ &task_path,
+ model,
+ language,
+ timeline.as_ref(),
+ &task_durations,
+ )
})
.await
.map_err(|error| format!("Transcription task panicked: {error}"))?
diff --git a/apps/desktop-gpui/src/transcription.rs b/apps/desktop-gpui/src/transcription.rs
index bc7ad223b..d6ba04c84 100644
--- a/apps/desktop-gpui/src/transcription.rs
+++ b/apps/desktop-gpui/src/transcription.rs
@@ -26,6 +26,7 @@ use std::{
collections::{HashMap, HashSet},
fs::File,
io::{Read, Write},
+ ops::Range,
path::{Path, PathBuf},
sync::{
Arc, LazyLock, Mutex, PoisonError,
@@ -40,7 +41,6 @@ use cap_project::{
ProjectConfiguration, RecordingMeta, StudioRecordingMeta, TimelineConfiguration,
TimelineSegment,
};
-use cap_rendering::Video;
use ffmpeg::{
ChannelLayout, codec as avcodec,
format::{self as avformat},
@@ -923,162 +923,23 @@ const DECODE_SAMPLE_RATE: u32 = 48_000;
/// `WHISPER_SAMPLE_RATE` (`captions.rs:66`).
const WHISPER_SAMPLE_RATE: u32 = 16_000;
-/// `AudioData::from_file` (`crates/audio/src/audio_data.rs:22-95`) transcribed
-/// against the same ffmpeg fork: any input decoded to packed f32 at 48kHz,
-/// mono stays mono, everything else downmixes to stereo.
-fn decode_audio_file(path: &Path) -> Result<(Vec<f32>, usize), String> {
- let mut input_ctx = avformat::input(&path).map_err(|e| format!("Input Open / {e}"))?;
- let input_stream = input_ctx
- .streams()
- .best(ffmpeg::media::Type::Audio)
- .ok_or_else(|| "No Stream".to_string())?;
-
- let decoder_ctx = avcodec::Context::from_parameters(input_stream.parameters())
- .map_err(|e| format!("Decoder Parameters / {e}"))?;
- let mut decoder = decoder_ctx
- .decoder()
- .audio()
- .map_err(|e| format!("Set Parameters / {e}"))?;
-
- let source_channels = decoder.channels().max(1);
- if decoder.channel_layout().is_empty() {
- decoder.set_channel_layout(ChannelLayout::default(i32::from(source_channels)));
- }
- decoder.set_packet_time_base(input_stream.time_base());
-
- let target_channels: u16 = if source_channels <= 1 { 1 } else { 2 };
- let target_channel_layout = ChannelLayout::default(i32::from(target_channels));
- let mut options = ffmpeg::Dictionary::new();
- options.set("filter_size", "128");
- options.set("cutoff", "0.97");
-
- let mut resampler = resampling::Context::get_with(
- decoder.format(),
- decoder.channel_layout(),
- decoder.rate(),
- avformat::Sample::F32(avformat::sample::Type::Packed),
- target_channel_layout,
- DECODE_SAMPLE_RATE,
- options,
- )
- .map_err(|e| format!("Resampler / {e}"))?;
-
- let index = input_stream.index();
-
- let mut decoded_frame = ffmpeg::frame::Audio::empty();
- let mut samples: Vec<f32> = Vec::new();
-
- for (stream, packet) in input_ctx.packets() {
- if stream.index() != index {
- continue;
- }
-
- decoder
- .send_packet(&packet)
- .map_err(|e| format!("Send Packet / {e}"))?;
-
- while decoder.receive_frame(&mut decoded_frame).is_ok() {
- run_decode_resampler(&mut resampler, &decoded_frame, &mut samples)?;
- }
- }
-
- decoder.send_eof().map_err(|e| format!("Send EOF / {e}"))?;
-
- while decoder.receive_frame(&mut decoded_frame).is_ok() {
- run_decode_resampler(&mut resampler, &decoded_frame, &mut samples)?;
- }
-
- flush_decode_resampler(&mut resampler, &mut samples)?;
-
- Ok((samples, usize::from(target_channels)))
-}
-
-fn run_decode_resampler(
- resampler: &mut resampling::Context,
- decoded_frame: &ffmpeg::frame::Audio,
- samples: &mut Vec<f32>,
-) -> Result<(), String> {
- let target = *resampler.output();
- let capacity = decode_resample_capacity(resampler, decoded_frame.samples());
- let mut resampled_frame =
- ffmpeg::frame::Audio::new(target.format, capacity, target.channel_layout);
-
- resampler
- .run(decoded_frame, &mut resampled_frame)
- .map_err(|e| format!("Run Resampler / {e}"))?;
-
- append_resampled_frame(samples, &resampled_frame)
-}
-
-fn flush_decode_resampler(
- resampler: &mut resampling::Context,
- samples: &mut Vec<f32>,
-) -> Result<(), String> {
- for _ in 0..64 {
- let Some(delay) = resampler.delay() else {
- break;
- };
- let target = *resampler.output();
- let capacity = delay
- .output
- .max(1)
- .saturating_add(16)
- .min(i64::from(i32::MAX)) as usize;
- let mut resampled_frame =
- ffmpeg::frame::Audio::new(target.format, capacity, target.channel_layout);
- let remaining = resampler
- .flush(&mut resampled_frame)
- .map_err(|e| format!("Flush Resampler / {e}"))?;
-
- let output_samples = resampled_frame.samples();
- append_resampled_frame(samples, &resampled_frame)?;
-
- if remaining.is_none() || output_samples == 0 {
- break;
- }
- }
- Ok(())
-}
-
-fn decode_resample_capacity(resampler: &resampling::Context, input_samples: usize) -> usize {
- let src_rate = u64::from(resampler.input().rate.max(1));
- let dst_rate = u64::from(resampler.output().rate.max(1));
- let pending_output_samples = resampler
- .delay()
- .map(|d| d.output.max(0) as u64)
- .unwrap_or(0);
- let resampled_from_input = (input_samples as u64)
- .saturating_mul(dst_rate)
- .div_ceil(src_rate);
-
- pending_output_samples
- .saturating_add(resampled_from_input)
- .saturating_add(16)
- .min(i32::MAX as u64) as usize
-}
-
-fn append_resampled_frame(
- samples: &mut Vec<f32>,
- frame: &ffmpeg::frame::Audio,
-) -> Result<(), String> {
- if frame.samples() == 0 {
- return Ok(());
- }
-
- let byte_len = frame
- .samples()
- .saturating_mul(frame.channels() as usize)
- .saturating_mul(std::mem::size_of::<f32>());
- let data = frame
- .data(0)
- .get(..byte_len)
- .ok_or_else(|| "Resampled frame data shorter than expected".to_string())?;
-
- // SAFETY: ffmpeg's frame buffers are allocated with av_malloc alignment,
- // and the slice length is a whole number of f32s by construction.
- samples
- .extend(unsafe { std::slice::from_raw_parts(data.as_ptr() as *const f32, byte_len / 4) });
- Ok(())
+fn decode_transcription_source(
+ path: &Path,
+ offset_secs: f64,
+ range: Range<f64>,
+) -> Result<TranscriptionAudioSource, String> {
+ let rate = f64::from(DECODE_SAMPLE_RATE);
+ let start = (range.start * rate).round() as i64;
+ let end = (range.end * rate).round() as i64;
+ let offset = (offset_secs * rate).round() as i64;
+ let source_start = (start + offset).max(0) as usize;
+ let source_end = (end + offset).max(0) as usize;
+ let decoded = cap_audio::AudioData::from_file_range(path, source_start, source_end)?;
+ Ok(TranscriptionAudioSource {
+ samples: decoded.samples().to_vec(),
+ channels: usize::from(decoded.channels()),
+ offset_secs: (start + offset - source_start as i64) as f64 / rate,
+ })
}
/// `normalize_audio_for_transcription` (`captions.rs:885-915`).
@@ -1114,112 +975,74 @@ fn normalize_audio_for_transcription(samples: &mut [f32]) -> f32 {
gain
}
-/// The recording-directory arm of `extract_audio_from_video`
-/// (`captions.rs:250-517`): per segment, decode system audio then mic
-/// (`captions.rs:281-283`), downmix each to mono, average them together,
-/// concatenate the segments, normalize, and write a 16kHz mono s16le WAV.
-fn extract_project_audio(project_path: &Path, output_path: &Path) -> Result<(), String> {
- ffmpeg::init().map_err(|e| format!("Failed to initialise ffmpeg: {e}"))?;
-
- let meta = RecordingMeta::load_for_project(project_path)
- .map_err(|e| format!("Failed to read recording metadata: {e}"))?;
- let Some(studio) = meta.studio_meta() else {
- return Err("Only studio recordings can be transcribed".to_string());
+fn extract_project_audio(
+ meta: &RecordingMeta,
+ studio: &StudioRecordingMeta,
+ recording_durations: &[f64],
+ range: Range<f64>,
+ output_path: &Path,
+) -> Result<bool, String> {
+ let tracks: Vec<Vec<(&cap_project::AudioMeta, f64)>> = match studio {
+ StudioRecordingMeta::SingleSegment { segment } => {
+ vec![segment.audio.iter().map(|audio| (audio, 0.0)).collect()]
+ }
+ StudioRecordingMeta::MultipleSegments { inner } => inner
+ .segments
+ .iter()
+ .map(|segment| {
+ let offsets = segment.calculate_audio_offsets();
+ [
+ (
+ segment.system_audio.as_ref(),
+ f64::from(offsets.system_audio),
+ ),
+ (segment.mic.as_ref(), f64::from(offsets.mic)),
+ ]
+ .into_iter()
+ .filter_map(|(audio, offset)| Some((audio?, offset)))
+ .collect()
+ })
+ .collect(),
};
-
- studio.ensure_ordinary_media_access(&meta.project_path)?;
+ if tracks.len() != recording_durations.len() {
+ return Err("Recording durations do not match the recording metadata".to_string());
+ }
let mut final_samples = Vec::new();
let mut any_audio = false;
- match studio {
- StudioRecordingMeta::SingleSegment { segment } => {
- let display_duration = Video::new(
- segment.display.path.to_path(&meta.project_path),
- segment.display.start_time.unwrap_or_default(),
- )
- .map_err(|error| format!("Failed to read display video: {error}"))?
- .duration;
- let mut sources = Vec::new();
- if let Some(audio) = &segment.audio {
- let path = meta.path(&audio.path);
- if path.exists() {
- match decode_audio_file(&path) {
- Ok((samples, channels)) => {
- any_audio = true;
- sources.push(TranscriptionAudioSource {
- samples,
- channels,
- offset_secs: 0.0,
- });
- }
- Err(error) => tracing::warn!(
- path = %path.display(),
- "Failed to process audio source: {error}"
- ),
- }
- }
- }
- let take = TranscriptionAudioTake {
- display_duration_secs: display_duration,
- sources,
- };
- append_transcription_audio(&mut final_samples, &take, DECODE_SAMPLE_RATE)
- .map_err(|error| format!("Failed to assemble transcription audio: {error}"))?;
+ let mut take_start = 0.0;
+ for (tracks, duration) in tracks.into_iter().zip(recording_durations) {
+ let start = (range.start - take_start).max(0.0);
+ let end = (range.end - take_start).min(*duration);
+ take_start += duration;
+ if start >= end {
+ continue;
}
- StudioRecordingMeta::MultipleSegments { inner } => {
- for (segment_idx, segment) in inner.segments.iter().enumerate() {
- let display_duration = Video::new(
- segment.display.path.to_path(&meta.project_path),
- segment.display.start_time.unwrap_or_default(),
- )
- .map_err(|error| {
- format!("Failed to read display video for segment {segment_idx}: {error}")
- })?
- .duration;
- let offsets = segment.calculate_audio_offsets();
- let mut sources = Vec::new();
- let mut push_source = |audio: &cap_project::AudioMeta, offset_secs: f64| {
- let path = meta.path(&audio.path);
- if !path.exists() {
- return;
- }
- match decode_audio_file(&path) {
- Ok((samples, channels)) => {
- any_audio = true;
- sources.push(TranscriptionAudioSource {
- samples,
- channels,
- offset_secs,
- });
- }
- Err(error) => tracing::warn!(
- path = %path.display(),
- "Failed to process audio source: {error}"
- ),
- }
- };
-
- if let Some(system_audio) = &segment.system_audio {
- push_source(system_audio, f64::from(offsets.system_audio));
- }
- if let Some(mic) = &segment.mic {
- push_source(mic, f64::from(offsets.mic));
+ let mut sources = Vec::new();
+ for (audio, offset) in tracks {
+ let path = meta.path(&audio.path);
+ if !path.exists() {
+ continue;
+ }
+ match decode_transcription_source(&path, offset, start..end) {
+ Ok(source) => {
+ any_audio = true;
+ sources.push(source);
}
- let take = TranscriptionAudioTake {
- display_duration_secs: display_duration,
- sources,
- };
- append_transcription_audio(&mut final_samples, &take, DECODE_SAMPLE_RATE)
- .map_err(|error| format!("Failed to assemble transcription audio: {error}"))?;
+ Err(error) => tracing::warn!(
+ path = %path.display(),
+ "Failed to process audio source: {error}"
+ ),
}
}
+ let take = TranscriptionAudioTake {
+ display_duration_secs: end - start,
+ sources,
+ };
+ append_transcription_audio(&mut final_samples, &take, DECODE_SAMPLE_RATE)
+ .map_err(|error| format!("Failed to assemble transcription audio: {error}"))?;
}
-
if !any_audio {
- return Err("No audio sources found in the recording metadata".to_string());
- }
-
- if final_samples.is_empty() {
- return Err("Failed to process any audio sources".to_string());
+ return Ok(false);
}
let gain = normalize_audio_for_transcription(&mut final_samples);
@@ -1227,7 +1050,8 @@ fn extract_project_audio(project_path: &Path, output_path: &Path) -> Result<(),
tracing::info!("Applied transcription audio gain: {gain:.2}x");
}
- write_wav_16k_mono(&final_samples, output_path)
+ write_wav_16k_mono(&final_samples, output_path)?;
+ Ok(true)
}
/// The WAV encode half of the recording-directory arm
@@ -1650,7 +1474,7 @@ fn process_with_parakeet(
.collect::<Result<Vec<_>, _>>()
.map_err(|e| format!("Failed to read audio file: {e}"))?;
if samples.is_empty() {
- return Err("No speech detected in the audio".to_string());
+ return Ok(Vec::new());
}
let mut tokens = Vec::new();
@@ -1678,7 +1502,7 @@ fn process_with_parakeet(
tracing::info!("Parakeet produced {} timed tokens", tokens.len());
- caption_segments_from_tokens(tokens)
+ Ok(caption_segments_from_tokens(tokens))
}
/// Parakeet's timed word tokens -> caption segments: drop empty tokens,
@@ -1686,7 +1510,7 @@ fn process_with_parakeet(
#[cfg(any(test, not(all(target_os = "macos", target_arch = "x86_64"))))]
fn caption_segments_from_tokens(
tokens: impl IntoIterator<Item = CaptionWord>,
-) -> Result<Vec<CaptionSegment>, String> {
+) -> Vec<CaptionSegment> {
let words = normalize_caption_words(
tokens
.into_iter()
@@ -1699,10 +1523,6 @@ fn caption_segments_from_tokens(
.collect(),
);
- if words.is_empty() {
- return Err("No speech detected in the audio".to_string());
- }
-
let mut segments = Vec::new();
for (chunk_idx, chunk) in caption_word_chunks(&words).into_iter().enumerate() {
segments.push(CaptionSegment {
@@ -1714,7 +1534,7 @@ fn caption_segments_from_tokens(
});
}
- Ok(segments)
+ segments
}
#[cfg(all(target_os = "macos", target_arch = "x86_64"))]
@@ -1737,6 +1557,8 @@ pub fn transcribe_blocking(
project_path: &Path,
model: &str,
language: &str,
+ timeline: Option<&TimelineConfiguration>,
+ recording_durations: &[f64],
) -> Result<Vec<CaptionSegment>, String> {
let model_file = model_path(model);
if !project_path.exists() {
@@ -1752,30 +1574,42 @@ pub fn transcribe_blocking(
));
}
- transcribe_project_with(project_path, |audio_path| {
- let _guard = lock_transcription_worker_slot();
- if is_parakeet_model(model) {
- tracing::info!("Using Parakeet TDT engine");
- let result = process_with_parakeet(audio_path, model_file.to_string_lossy().as_ref());
- schedule_parakeet_release();
- result
- } else {
- tracing::info!("Using Whisper engine");
- let hints = transcription_hints();
- let context = get_whisper_context(model_file.to_string_lossy().as_ref())?;
- let result = process_with_whisper(audio_path, context, language, &hints);
- release_whisper_context_after_transcription();
- result
- }
- })
+ let _guard = lock_transcription_worker_slot();
+ let result =
+ transcribe_project_with(project_path, timeline, recording_durations, |audio_path| {
+ if is_parakeet_model(model) {
+ tracing::info!("Using Parakeet TDT engine");
+ process_with_parakeet(audio_path, model_file.to_string_lossy().as_ref())
+ } else {
+ tracing::info!("Using Whisper engine");
+ let hints = transcription_hints();
+ let context = get_whisper_context(model_file.to_string_lossy().as_ref())?;
+ process_with_whisper(audio_path, context, language, &hints)
+ }
+ });
+ if is_parakeet_model(model) {
+ schedule_parakeet_release();
+ } else {
+ release_whisper_context_after_transcription();
+ }
+ result
}
/// Extracts the project's transcription audio to a temporary 16 kHz mono WAV
/// and runs `engine` on it.
fn transcribe_project_with(
project_path: &Path,
- engine: impl FnOnce(&Path) -> Result<Vec<CaptionSegment>, String>,
+ timeline: Option<&TimelineConfiguration>,
+ recording_durations: &[f64],
+ mut engine: impl FnMut(&Path) -> Result<Vec<CaptionSegment>, String>,
) -> Result<Vec<CaptionSegment>, String> {
+ ffmpeg::init().map_err(|e| format!("Failed to initialise ffmpeg: {e}"))?;
+ let meta = RecordingMeta::load_for_project(project_path)
+ .map_err(|e| format!("Failed to read recording metadata: {e}"))?;
+ let Some(studio) = meta.studio_meta() else {
+ return Err("Only studio recordings can be transcribed".to_string());
+ };
+ studio.ensure_ordinary_media_access(&meta.project_path)?;
let temp_dir = std::env::temp_dir().join(format!(
"cap-gpui-transcription-{}-{}",
std::process::id(),
@@ -1786,8 +1620,34 @@ fn transcribe_project_with(
let audio_path = temp_dir.join("audio.wav");
let result = (|| {
- extract_audio_from_video(project_path, &audio_path)?;
- engine(&audio_path)
+ let mut segments = Vec::new();
+ let mut any_audio = false;
+ for range in cap_project::transcription_source_ranges(timeline, recording_durations) {
+ if !extract_project_audio(
+ &meta,
+ studio,
+ recording_durations,
+ range.clone(),
+ &audio_path,
+ )? {
+ continue;
+ }
+ any_audio = true;
+ for mut segment in engine(&audio_path)? {
+ segment.id = format!("segment-{}", segments.len());
+ segment.start += range.start as f32;
+ segment.end += range.start as f32;
+ for word in &mut segment.words {
+ word.start += range.start as f32;
+ word.end += range.start as f32;
+ }
+ segments.push(segment);
+ }
+ }
+ if !any_audio {
+ return Err("No audio sources found in the selected timeline".to_string());
+ }
+ Ok(segments)
})();
let _ = std::fs::remove_dir_all(&temp_dir);
@@ -1799,21 +1659,6 @@ fn transcribe_project_with(
Ok(segments)
}
-/// `resolve_audio_extraction_source` (`captions.rs:221-243`), directory arm
-/// only -- the gpui editor always opens `.cap` bundles, so a plain media file
-/// is refused rather than decoded.
-fn extract_audio_from_video(video_path: &Path, output_path: &Path) -> Result<(), String> {
- let metadata = std::fs::metadata(video_path)
- .map_err(|e| format!("Failed to read video path metadata: {e}"))?;
- if !metadata.is_dir() {
- return Err("Video path is not a recording directory".to_string());
- }
- if !video_path.join("recording-meta.json").is_file() {
- return Err("Recording directory is missing recording-meta.json".to_string());
- }
- extract_project_audio(video_path, output_path)
-}
-
// ---------------------------------------------------------------------------
// Caption words & chunking
// ---------------------------------------------------------------------------
diff --git a/apps/desktop-gpui/src/transcription/verify_tests.rs b/apps/desktop-gpui/src/transcription/verify_tests.rs
index 39a2af3be..e00f2dbf8 100644
--- a/apps/desktop-gpui/src/transcription/verify_tests.rs
+++ b/apps/desktop-gpui/src/transcription/verify_tests.rs
@@ -36,6 +36,8 @@ use cap_verify::{
report::{self, Case, Tier},
};
+use cap_rendering::Video;
+
use super::*;
const MAX_WORD_SECS: f32 = 2.5;
@@ -335,10 +337,15 @@ fn takes_fixture() -> FixtureSpec {
/// segments.
fn check_project(case: &mut Case, project: &Path, takes: &Takes) {
let mut tokens = Vec::new();
- let result = transcribe_project_with(project, |audio_path| {
- tokens = fake_tokens(audio_path)?;
- caption_segments_from_tokens(tokens.clone())
- });
+ let result = transcribe_project_with(
+ project,
+ None,
+ &takes.iter().map(|take| take.0).collect::<Vec<_>>(),
+ |audio_path| {
+ tokens = fake_tokens(audio_path)?;
+ Ok(caption_segments_from_tokens(tokens.clone()))
+ },
+ );
let segments = match result {
Ok(segments) => segments,
Err(e) => {
@@ -526,7 +533,7 @@ fn instant_project_is_refused() {
),
}
let mut called = false;
- let result = transcribe_project_with(&project, |_| {
+ let result = transcribe_project_with(&project, None, &[], |_| {
called = true;
Ok(Vec::new())
});
@@ -606,33 +613,25 @@ fn segmentation_case(id: &str, unicode: bool) {
unicode,
);
let reference = reference_segments(&tokens);
- match caption_segments_from_tokens(tokens.clone()) {
- Err(e) if reference.is_empty() && e.contains("No speech") => empty += 1,
- Err(e) => {
- failed += 1;
+ let product = caption_segments_from_tokens(tokens.clone());
+ if product.is_empty() {
+ empty += 1;
+ }
+ segments += product.len();
+ let problems = compare_segments(&product, &reference, 0.0);
+ if !problems.is_empty() {
+ failed += 1;
+ if failed <= 3 {
case.problem(
"caption-segments",
- format!("seed {seed}: unexpected error {e}"),
+ format!(
+ "seed {seed}: {} problems (first: {}); tokens {:?}",
+ problems.len(),
+ problems[0],
+ tokens.iter().map(|t| t.text.as_str()).collect::<Vec<_>>()
+ ),
);
}
- Ok(product) => {
- segments += product.len();
- let problems = compare_segments(&product, &reference, 0.0);
- if !problems.is_empty() {
- failed += 1;
- if failed <= 3 {
- case.problem(
- "caption-segments",
- format!(
- "seed {seed}: {} problems (first: {}); tokens {:?}",
- problems.len(),
- problems[0],
- tokens.iter().map(|t| t.text.as_str()).collect::<Vec<_>>()
- ),
- );
- }
- }
- }
}
}
case.note(format!(
@@ -650,3 +649,186 @@ fn segmentation_matches_reference() {
fn segmentation_matches_reference_unicode() {
segmentation_case("captions/app/segmentation/unicode", true);
}
+
+#[test]
+fn trimmed_transcription_extracts_only_used_audio_and_restores_source_times() {
+ let spec = takes_fixture();
+ let project = fixture::ensure_fixture(&spec, fixture::write_fixture).expect("fixture");
+ let takes = multi_segment_takes(&project, &spec);
+ let durations: Vec<f64> = takes.iter().map(|take| take.0).collect();
+ for (clips, expected_ranges, expected_starts) in [
+ (
+ serde_json::json!([
+ {"recordingSegment": 0, "start": 1.0, "end": 3.0, "timescale": 2.0}
+ ]),
+ vec![1.0..3.0],
+ vec![0.125],
+ ),
+ (
+ serde_json::json!([
+ {"recordingSegment": 1, "start": 1.0, "end": 3.0, "timescale": 0.5},
+ {"recordingSegment": 0, "start": 3.0, "end": 4.0, "timescale": 2.0},
+ {"recordingSegment": 0, "start": 0.5, "end": 1.5, "timescale": 1.0},
+ {"recordingSegment": 0, "start": 1.0, "end": 2.0, "timescale": 1.0},
+ {"recordingSegment": 0, "start": 3.0, "end": 4.0, "timescale": 1.0}
+ ]),
+ vec![0.5..2.0, 3.0..4.0, durations[0] + 1.0..durations[0] + 3.0],
+ vec![0.5, 4.125, 4.75, 6.75],
+ ),
+ (
+ serde_json::json!([
+ {"recordingSegment":0,"start":4.0,"end":durations[0],"timescale":1.0},
+ {"recordingSegment":1,"start":0.0,"end":1.0,"timescale":1.0}
+ ]),
+ vec![4.0..durations[0] + 1.0],
+ vec![0.25],
+ ),
+ ] {
+ let timeline: TimelineConfiguration = serde_json::from_value(serde_json::json!({
+ "segments": clips, "zoomSegments": []
+ }))
+ .unwrap();
+ let mut called = 0;
+ let segments = transcribe_project_with(&project, Some(&timeline), &durations, |path| {
+ let range = &expected_ranges[called];
+ let audio = media::decode_audio(path, Some(MARKER_RATE)).unwrap();
+ let duration = audio.mono().len() as f64 / f64::from(MARKER_RATE);
+ assert!(
+ (duration - (range.end - range.start)).abs() < 0.002,
+ "extracted {duration}s, expected {}s",
+ range.end - range.start
+ );
+ called += 1;
+ Ok(vec![CaptionSegment {
+ id: "segment-0".into(),
+ start: 0.25,
+ end: 0.5,
+ text: "word".into(),
+ words: vec![CaptionWord {
+ text: "word".into(),
+ start: 0.25,
+ end: 0.5,
+ }],
+ }])
+ })
+ .unwrap();
+ assert_eq!(called, expected_ranges.len());
+ for (segment, range) in segments.iter().zip(expected_ranges) {
+ assert!((f64::from(segment.start) - range.start - 0.25).abs() < 1e-5);
+ assert!((f64::from(segment.end) - range.start - 0.5).abs() < 1e-5);
+ assert_eq!(segment.words[0].start, segment.start);
+ assert_eq!(segment.words[0].end, segment.end);
+ }
+ let ids: std::collections::HashSet<_> =
+ segments.iter().map(|segment| &segment.id).collect();
+ assert_eq!(ids.len(), segments.len());
+ let tracks = derive_caption_track_segments(&segments, &timeline, &durations);
+ assert_eq!(tracks.len(), expected_starts.len());
+ for (track, expected) in tracks.iter().zip(expected_starts) {
+ assert!((track.start - expected).abs() < 1e-5);
+ }
+ }
+}
+
+#[test]
+fn trimmed_sources_match_full_audio_with_positive_and_negative_offsets() {
+ let spec = takes_fixture();
+ let project = fixture::ensure_fixture(&spec, fixture::write_fixture).unwrap();
+ let meta = RecordingMeta::load_for_project(&project).unwrap();
+ let Some(StudioRecordingMeta::MultipleSegments { inner }) = meta.studio_meta() else {
+ panic!("studio")
+ };
+ let segment = &inner.segments[0];
+ for audio in [
+ segment.mic.as_ref().unwrap(),
+ segment.system_audio.as_ref().unwrap(),
+ ] {
+ let path = meta.path(&audio.path);
+ let decoded = cap_audio::AudioData::from_file(&path).unwrap();
+ for offset_secs in [-0.4, 0.3] {
+ let full = cap_editor::assemble_transcription_audio(
+ &[TranscriptionAudioTake {
+ display_duration_secs: 5.0,
+ sources: vec![TranscriptionAudioSource {
+ samples: decoded.samples().to_vec(),
+ channels: usize::from(decoded.channels()),
+ offset_secs,
+ }],
+ }],
+ DECODE_SAMPLE_RATE,
+ )
+ .unwrap();
+ for range in [0.0..1.0, 1.25..2.5, 3.0..4.0] {
+ let source =
+ decode_transcription_source(&path, offset_secs, range.clone()).unwrap();
+ let trimmed = cap_editor::assemble_transcription_audio(
+ &[TranscriptionAudioTake {
+ display_duration_secs: range.end - range.start,
+ sources: vec![source],
+ }],
+ DECODE_SAMPLE_RATE,
+ )
+ .unwrap();
+ let start = (range.start * f64::from(DECODE_SAMPLE_RATE)).round() as usize;
+ let reference = &full[start..start + trimmed.len()];
+ let worst = trimmed
+ .iter()
+ .zip(reference)
+ .fold(0.0_f32, |worst, (a, b)| worst.max((a - b).abs()));
+ assert!(
+ worst < 0.001,
+ "offset {offset_secs}, range {range:?}, worst sample error {worst}"
+ );
+ }
+ }
+ }
+}
+
+#[test]
+fn transcription_keeps_speech_after_empty_ranges_and_cleans_up_on_failure() {
+ let spec = takes_fixture();
+ let project = fixture::ensure_fixture(&spec, fixture::write_fixture).unwrap();
+ let takes = multi_segment_takes(&project, &spec);
+ let durations: Vec<_> = takes.iter().map(|take| take.0).collect();
+ let timeline = serde_json::from_value(serde_json::json!({"segments": [
+ {"start":0.0,"end":1.0,"timescale":1.0},
+ {"start":3.0,"end":4.0,"timescale":1.0}
+ ], "zoomSegments": []}))
+ .unwrap();
+ let mut calls = 0;
+ let mut temporary = PathBuf::new();
+ let result = transcribe_project_with(&project, Some(&timeline), &durations, |path| {
+ temporary = path.to_path_buf();
+ calls += 1;
+ if calls == 1 {
+ return Ok(Vec::new());
+ }
+ Ok(caption_segments_from_tokens([CaptionWord {
+ text: "speech".into(),
+ start: 0.1,
+ end: 0.2,
+ }]))
+ })
+ .unwrap();
+ assert_eq!(calls, 2);
+ assert!((result[0].words[0].start - 3.1).abs() < 1e-5);
+ assert!(!temporary.parent().unwrap().exists());
+ let error = transcribe_project_with(&project, Some(&timeline), &durations, |path| {
+ temporary = path.to_path_buf();
+ Err("test engine failure".into())
+ })
+ .unwrap_err();
+ assert_eq!(error, "test engine failure");
+ assert!(!temporary.parent().unwrap().exists());
+ let empty = transcribe_project_with(&project, Some(&timeline), &durations, |_| Ok(Vec::new()))
+ .unwrap_err();
+ assert_eq!(empty, "No speech detected in the audio");
+ assert!(caption_segments_from_tokens(Vec::new()).is_empty());
+ let empty_timeline =
+ serde_json::from_value(serde_json::json!({"segments":[],"zoomSegments":[]})).unwrap();
+ let error = transcribe_project_with(&project, Some(&empty_timeline), &durations, |_| {
+ panic!("an empty timeline must not run the model")
+ })
+ .unwrap_err();
+ assert!(error.contains("No audio sources"));
+}
diff --git a/crates/project/src/caption_timing.rs b/crates/project/src/caption_timing.rs
index 63700fc2a..6b6b5e622 100644
--- a/crates/project/src/caption_timing.rs
+++ b/crates/project/src/caption_timing.rs
@@ -152,6 +152,40 @@ fn build_source_to_edited_mappings(
.collect()
}
+pub fn transcription_source_ranges(
+ timeline: Option<&TimelineConfiguration>,
+ recording_durations: &[f64],
+) -> Vec<std::ops::Range<f64>> {
+ let Some(timeline) = timeline else {
+ let duration = recording_durations.iter().sum();
+ return if duration > 0.0 {
+ vec![0.0..duration]
+ } else {
+ Vec::new()
+ };
+ };
+ let mut ranges: Vec<_> = build_source_to_edited_mappings(timeline, recording_durations)
+ .into_iter()
+ .zip(&timeline.segments)
+ .filter_map(|(mapping, segment)| {
+ let duration = *recording_durations.get(segment.recording_clip as usize)?;
+ let offset = mapping.source_start - segment.start;
+ let start = segment.start.max(0.0);
+ let end = segment.end.min(duration);
+ (start < end).then_some(offset + start..offset + end)
+ })
+ .collect();
+ ranges.sort_by(|a, b| a.start.total_cmp(&b.start));
+ let mut merged: Vec<std::ops::Range<f64>> = Vec::with_capacity(ranges.len());
+ for range in ranges {
+ match merged.last_mut() {
+ Some(last) if range.start <= last.end => last.end = last.end.max(range.end),
+ _ => merged.push(range),
+ }
+ }
+ merged
+}
+
fn map_time_range_within_mapping(
start: f64,
end: f64,
@@ -429,6 +463,48 @@ mod tests {
use super::*;
use crate::CaptionsData;
+ #[test]
+ fn transcription_ranges_follow_source_cuts_independent_of_speed_and_clip_order() {
+ let timeline = serde_json::from_value(serde_json::json!({
+ "segments": [
+ {"recordingSegment":1,"start":1.0,"end":3.0,"timescale":0.5},
+ {"recordingSegment":0,"start":5.0,"end":6.0,"timescale":2.0},
+ {"recordingSegment":0,"start":1.0,"end":3.0,"timescale":1.0},
+ {"recordingSegment":0,"start":2.0,"end":4.0,"timescale":1.0},
+ {"recordingSegment":0,"start":4.0,"end":4.5,"timescale":1.0},
+ {"recordingSegment":0,"start":1.0,"end":3.0,"timescale":2.0}
+ ], "zoomSegments": []
+ }))
+ .unwrap();
+ assert_eq!(
+ transcription_source_ranges(Some(&timeline), &[10.0, 4.0]),
+ vec![1.0..4.5, 5.0..6.0, 11.0..13.0]
+ );
+ }
+
+ #[test]
+ fn transcription_ranges_preserve_empty_timelines_and_clip_to_sources() {
+ let mut timeline: TimelineConfiguration =
+ serde_json::from_value(serde_json::json!({"segments": [], "zoomSegments": []}))
+ .unwrap();
+ assert!(transcription_source_ranges(Some(&timeline), &[30.0]).is_empty());
+ assert_eq!(
+ transcription_source_ranges(None, &[30.0, 10.0]),
+ vec![0.0..40.0]
+ );
+ timeline.segments = serde_json::from_value(serde_json::json!([
+ {"recordingSegment":0,"start":-1.0,"end":4.0,"timescale":1.0},
+ {"recordingSegment":0,"start":8.0,"end":40.0,"timescale":1.0},
+ {"recordingSegment":2,"start":0.0,"end":3.0,"timescale":1.0},
+ {"recordingSegment":0,"start":5.0,"end":5.0,"timescale":1.0}
+ ]))
+ .unwrap();
+ assert_eq!(
+ transcription_source_ranges(Some(&timeline), &[10.0]),
+ vec![0.0..4.0, 8.0..10.0]
+ );
+ }
+
fn project(source_timed: bool, start: f32, end: f32) -> ProjectConfiguration {
ProjectConfiguration {
timeline: Some(serde_json::from_value(serde_json::json!({
diff --git a/crates/project/src/lib.rs b/crates/project/src/lib.rs
index dea389ee1..6e82e759a 100644
--- a/crates/project/src/lib.rs
+++ b/crates/project/src/lib.rs
@@ -6,7 +6,10 @@ pub mod keyboard;
mod meta;
pub use animated_gradient::*;
-pub use caption_timing::{derive_caption_track_segments, source_caption_id, synchronize_captions};
+pub use caption_timing::{
+ derive_caption_track_segments, source_caption_id, synchronize_captions,
+ transcription_source_ranges,
+};
pub use configuration::*;
pub use cursor::*;
pub use keyboard::*;