Open source Loom alternative. Beautiful, shareable screen recordings.
Rust appcapcossloommacnextjsnextjs15open-sourceossreactrecordrustscreen-capturescreen-recorderscreenshotsolidjstauritauri-apptypescriptvite
fix(desktop-gpui): transcribe only retained source audio
Aditya Garudfbbe4166047e
diff --git a/apps/desktop-gpui/src/editor_tabs.rs b/apps/desktop-gpui/src/editor_tabs.rsindex a79364748..3a808180d 100644--- a/apps/desktop-gpui/src/editor_tabs.rs+++ b/apps/desktop-gpui/src/editor_tabs.rs@@ -3176,6 +3176,7 @@ impl EditorWindow {};let recording_durations = summary.clip_display_durations.clone();let recording_duration = summary.recording_duration;+ let timeline = self.project.timeline.clone();let project_path = self.project_path.clone();let model = self.selected_caption_model().name;let language = self.sidebar.caption_language;@@ -3187,9 +3188,16 @@ impl EditorWindow {cx.spawn_in(window, async move |this, cx| {let task_path = project_path.clone();+ let task_durations = recording_durations.clone();let task = gpui_tokio::Tokio::spawn(cx, async move {tokio::task::spawn_blocking(move || {- transcription::transcribe_blocking(&task_path, model, language)+ transcription::transcribe_blocking(+ &task_path,+ model,+ language,+ timeline.as_ref(),+ &task_durations,+ )}).await.map_err(|error| format!("Transcription task panicked: {error}"))?diff --git a/apps/desktop-gpui/src/transcription.rs b/apps/desktop-gpui/src/transcription.rsindex bc7ad223b..d6ba04c84 100644--- a/apps/desktop-gpui/src/transcription.rs+++ b/apps/desktop-gpui/src/transcription.rs@@ -26,6 +26,7 @@ use std::{collections::{HashMap, HashSet},fs::File,io::{Read, Write},+ ops::Range,path::{Path, PathBuf},sync::{Arc, LazyLock, Mutex, PoisonError,@@ -40,7 +41,6 @@ use cap_project::{ProjectConfiguration, RecordingMeta, StudioRecordingMeta, TimelineConfiguration,TimelineSegment,};-use cap_rendering::Video;use ffmpeg::{ChannelLayout, codec as avcodec,format::{self as avformat},@@ -923,162 +923,23 @@ const DECODE_SAMPLE_RATE: u32 = 48_000;/// `WHISPER_SAMPLE_RATE` (`captions.rs:66`).const WHISPER_SAMPLE_RATE: u32 = 16_000;-/// `AudioData::from_file` (`crates/audio/src/audio_data.rs:22-95`) transcribed-/// against the same ffmpeg fork: any input decoded to packed f32 at 48kHz,-/// mono stays mono, everything else downmixes to stereo.-fn decode_audio_file(path: &Path) -> Result<(Vec<f32>, usize), String> {- let mut input_ctx = avformat::input(&path).map_err(|e| format!("Input Open / {e}"))?;- let input_stream = input_ctx- .streams()- .best(ffmpeg::media::Type::Audio)- .ok_or_else(|| "No Stream".to_string())?;-- let decoder_ctx = avcodec::Context::from_parameters(input_stream.parameters())- .map_err(|e| format!("Decoder Parameters / {e}"))?;- let mut decoder = decoder_ctx- .decoder()- .audio()- .map_err(|e| format!("Set Parameters / {e}"))?;-- let source_channels = decoder.channels().max(1);- if decoder.channel_layout().is_empty() {- decoder.set_channel_layout(ChannelLayout::default(i32::from(source_channels)));- }- decoder.set_packet_time_base(input_stream.time_base());-- let target_channels: u16 = if source_channels <= 1 { 1 } else { 2 };- let target_channel_layout = ChannelLayout::default(i32::from(target_channels));- let mut options = ffmpeg::Dictionary::new();- options.set("filter_size", "128");- options.set("cutoff", "0.97");-- let mut resampler = resampling::Context::get_with(- decoder.format(),- decoder.channel_layout(),- decoder.rate(),- avformat::Sample::F32(avformat::sample::Type::Packed),- target_channel_layout,- DECODE_SAMPLE_RATE,- options,- )- .map_err(|e| format!("Resampler / {e}"))?;-- let index = input_stream.index();-- let mut decoded_frame = ffmpeg::frame::Audio::empty();- let mut samples: Vec<f32> = Vec::new();-- for (stream, packet) in input_ctx.packets() {- if stream.index() != index {- continue;- }-- decoder- .send_packet(&packet)- .map_err(|e| format!("Send Packet / {e}"))?;-- while decoder.receive_frame(&mut decoded_frame).is_ok() {- run_decode_resampler(&mut resampler, &decoded_frame, &mut samples)?;- }- }-- decoder.send_eof().map_err(|e| format!("Send EOF / {e}"))?;-- while decoder.receive_frame(&mut decoded_frame).is_ok() {- run_decode_resampler(&mut resampler, &decoded_frame, &mut samples)?;- }-- flush_decode_resampler(&mut resampler, &mut samples)?;-- Ok((samples, usize::from(target_channels)))-}--fn run_decode_resampler(- resampler: &mut resampling::Context,- decoded_frame: &ffmpeg::frame::Audio,- samples: &mut Vec<f32>,-) -> Result<(), String> {- let target = *resampler.output();- let capacity = decode_resample_capacity(resampler, decoded_frame.samples());- let mut resampled_frame =- ffmpeg::frame::Audio::new(target.format, capacity, target.channel_layout);-- resampler- .run(decoded_frame, &mut resampled_frame)- .map_err(|e| format!("Run Resampler / {e}"))?;-- append_resampled_frame(samples, &resampled_frame)-}--fn flush_decode_resampler(- resampler: &mut resampling::Context,- samples: &mut Vec<f32>,-) -> Result<(), String> {- for _ in 0..64 {- let Some(delay) = resampler.delay() else {- break;- };- let target = *resampler.output();- let capacity = delay- .output- .max(1)- .saturating_add(16)- .min(i64::from(i32::MAX)) as usize;- let mut resampled_frame =- ffmpeg::frame::Audio::new(target.format, capacity, target.channel_layout);- let remaining = resampler- .flush(&mut resampled_frame)- .map_err(|e| format!("Flush Resampler / {e}"))?;-- let output_samples = resampled_frame.samples();- append_resampled_frame(samples, &resampled_frame)?;-- if remaining.is_none() || output_samples == 0 {- break;- }- }- Ok(())-}--fn decode_resample_capacity(resampler: &resampling::Context, input_samples: usize) -> usize {- let src_rate = u64::from(resampler.input().rate.max(1));- let dst_rate = u64::from(resampler.output().rate.max(1));- let pending_output_samples = resampler- .delay()- .map(|d| d.output.max(0) as u64)- .unwrap_or(0);- let resampled_from_input = (input_samples as u64)- .saturating_mul(dst_rate)- .div_ceil(src_rate);-- pending_output_samples- .saturating_add(resampled_from_input)- .saturating_add(16)- .min(i32::MAX as u64) as usize-}--fn append_resampled_frame(- samples: &mut Vec<f32>,- frame: &ffmpeg::frame::Audio,-) -> Result<(), String> {- if frame.samples() == 0 {- return Ok(());- }-- let byte_len = frame- .samples()- .saturating_mul(frame.channels() as usize)- .saturating_mul(std::mem::size_of::<f32>());- let data = frame- .data(0)- .get(..byte_len)- .ok_or_else(|| "Resampled frame data shorter than expected".to_string())?;-- // SAFETY: ffmpeg's frame buffers are allocated with av_malloc alignment,- // and the slice length is a whole number of f32s by construction.- samples- .extend(unsafe { std::slice::from_raw_parts(data.as_ptr() as *const f32, byte_len / 4) });- Ok(())+fn decode_transcription_source(+ path: &Path,+ offset_secs: f64,+ range: Range<f64>,+) -> Result<TranscriptionAudioSource, String> {+ let rate = f64::from(DECODE_SAMPLE_RATE);+ let start = (range.start * rate).round() as i64;+ let end = (range.end * rate).round() as i64;+ let offset = (offset_secs * rate).round() as i64;+ let source_start = (start + offset).max(0) as usize;+ let source_end = (end + offset).max(0) as usize;+ let decoded = cap_audio::AudioData::from_file_range(path, source_start, source_end)?;+ Ok(TranscriptionAudioSource {+ samples: decoded.samples().to_vec(),+ channels: usize::from(decoded.channels()),+ offset_secs: (start + offset - source_start as i64) as f64 / rate,+ })}/// `normalize_audio_for_transcription` (`captions.rs:885-915`).@@ -1114,112 +975,74 @@ fn normalize_audio_for_transcription(samples: &mut [f32]) -> f32 {gain}-/// The recording-directory arm of `extract_audio_from_video`-/// (`captions.rs:250-517`): per segment, decode system audio then mic-/// (`captions.rs:281-283`), downmix each to mono, average them together,-/// concatenate the segments, normalize, and write a 16kHz mono s16le WAV.-fn extract_project_audio(project_path: &Path, output_path: &Path) -> Result<(), String> {- ffmpeg::init().map_err(|e| format!("Failed to initialise ffmpeg: {e}"))?;-- let meta = RecordingMeta::load_for_project(project_path)- .map_err(|e| format!("Failed to read recording metadata: {e}"))?;- let Some(studio) = meta.studio_meta() else {- return Err("Only studio recordings can be transcribed".to_string());+fn extract_project_audio(+ meta: &RecordingMeta,+ studio: &StudioRecordingMeta,+ recording_durations: &[f64],+ range: Range<f64>,+ output_path: &Path,+) -> Result<bool, String> {+ let tracks: Vec<Vec<(&cap_project::AudioMeta, f64)>> = match studio {+ StudioRecordingMeta::SingleSegment { segment } => {+ vec![segment.audio.iter().map(|audio| (audio, 0.0)).collect()]+ }+ StudioRecordingMeta::MultipleSegments { inner } => inner+ .segments+ .iter()+ .map(|segment| {+ let offsets = segment.calculate_audio_offsets();+ [+ (+ segment.system_audio.as_ref(),+ f64::from(offsets.system_audio),+ ),+ (segment.mic.as_ref(), f64::from(offsets.mic)),+ ]+ .into_iter()+ .filter_map(|(audio, offset)| Some((audio?, offset)))+ .collect()+ })+ .collect(),};-- studio.ensure_ordinary_media_access(&meta.project_path)?;+ if tracks.len() != recording_durations.len() {+ return Err("Recording durations do not match the recording metadata".to_string());+ }let mut final_samples = Vec::new();let mut any_audio = false;- match studio {- StudioRecordingMeta::SingleSegment { segment } => {- let display_duration = Video::new(- segment.display.path.to_path(&meta.project_path),- segment.display.start_time.unwrap_or_default(),- )- .map_err(|error| format!("Failed to read display video: {error}"))?- .duration;- let mut sources = Vec::new();- if let Some(audio) = &segment.audio {- let path = meta.path(&audio.path);- if path.exists() {- match decode_audio_file(&path) {- Ok((samples, channels)) => {- any_audio = true;- sources.push(TranscriptionAudioSource {- samples,- channels,- offset_secs: 0.0,- });- }- Err(error) => tracing::warn!(- path = %path.display(),- "Failed to process audio source: {error}"- ),- }- }- }- let take = TranscriptionAudioTake {- display_duration_secs: display_duration,- sources,- };- append_transcription_audio(&mut final_samples, &take, DECODE_SAMPLE_RATE)- .map_err(|error| format!("Failed to assemble transcription audio: {error}"))?;+ let mut take_start = 0.0;+ for (tracks, duration) in tracks.into_iter().zip(recording_durations) {+ let start = (range.start - take_start).max(0.0);+ let end = (range.end - take_start).min(*duration);+ take_start += duration;+ if start >= end {+ continue;}- StudioRecordingMeta::MultipleSegments { inner } => {- for (segment_idx, segment) in inner.segments.iter().enumerate() {- let display_duration = Video::new(- segment.display.path.to_path(&meta.project_path),- segment.display.start_time.unwrap_or_default(),- )- .map_err(|error| {- format!("Failed to read display video for segment {segment_idx}: {error}")- })?- .duration;- let offsets = segment.calculate_audio_offsets();- let mut sources = Vec::new();- let mut push_source = |audio: &cap_project::AudioMeta, offset_secs: f64| {- let path = meta.path(&audio.path);- if !path.exists() {- return;- }- match decode_audio_file(&path) {- Ok((samples, channels)) => {- any_audio = true;- sources.push(TranscriptionAudioSource {- samples,- channels,- offset_secs,- });- }- Err(error) => tracing::warn!(- path = %path.display(),- "Failed to process audio source: {error}"- ),- }- };-- if let Some(system_audio) = &segment.system_audio {- push_source(system_audio, f64::from(offsets.system_audio));- }- if let Some(mic) = &segment.mic {- push_source(mic, f64::from(offsets.mic));+ let mut sources = Vec::new();+ for (audio, offset) in tracks {+ let path = meta.path(&audio.path);+ if !path.exists() {+ continue;+ }+ match decode_transcription_source(&path, offset, start..end) {+ Ok(source) => {+ any_audio = true;+ sources.push(source);}- let take = TranscriptionAudioTake {- display_duration_secs: display_duration,- sources,- };- append_transcription_audio(&mut final_samples, &take, DECODE_SAMPLE_RATE)- .map_err(|error| format!("Failed to assemble transcription audio: {error}"))?;+ Err(error) => tracing::warn!(+ path = %path.display(),+ "Failed to process audio source: {error}"+ ),}}+ let take = TranscriptionAudioTake {+ display_duration_secs: end - start,+ sources,+ };+ append_transcription_audio(&mut final_samples, &take, DECODE_SAMPLE_RATE)+ .map_err(|error| format!("Failed to assemble transcription audio: {error}"))?;}-if !any_audio {- return Err("No audio sources found in the recording metadata".to_string());- }-- if final_samples.is_empty() {- return Err("Failed to process any audio sources".to_string());+ return Ok(false);}let gain = normalize_audio_for_transcription(&mut final_samples);@@ -1227,7 +1050,8 @@ fn extract_project_audio(project_path: &Path, output_path: &Path) -> Result<(),tracing::info!("Applied transcription audio gain: {gain:.2}x");}- write_wav_16k_mono(&final_samples, output_path)+ write_wav_16k_mono(&final_samples, output_path)?;+ Ok(true)}/// The WAV encode half of the recording-directory arm@@ -1650,7 +1474,7 @@ fn process_with_parakeet(.collect::<Result<Vec<_>, _>>().map_err(|e| format!("Failed to read audio file: {e}"))?;if samples.is_empty() {- return Err("No speech detected in the audio".to_string());+ return Ok(Vec::new());}let mut tokens = Vec::new();@@ -1678,7 +1502,7 @@ fn process_with_parakeet(tracing::info!("Parakeet produced {} timed tokens", tokens.len());- caption_segments_from_tokens(tokens)+ Ok(caption_segments_from_tokens(tokens))}/// Parakeet's timed word tokens -> caption segments: drop empty tokens,@@ -1686,7 +1510,7 @@ fn process_with_parakeet(#[cfg(any(test, not(all(target_os = "macos", target_arch = "x86_64"))))]fn caption_segments_from_tokens(tokens: impl IntoIterator<Item = CaptionWord>,-) -> Result<Vec<CaptionSegment>, String> {+) -> Vec<CaptionSegment> {let words = normalize_caption_words(tokens.into_iter()@@ -1699,10 +1523,6 @@ fn caption_segments_from_tokens(.collect(),);- if words.is_empty() {- return Err("No speech detected in the audio".to_string());- }-let mut segments = Vec::new();for (chunk_idx, chunk) in caption_word_chunks(&words).into_iter().enumerate() {segments.push(CaptionSegment {@@ -1714,7 +1534,7 @@ fn caption_segments_from_tokens(});}- Ok(segments)+ segments}#[cfg(all(target_os = "macos", target_arch = "x86_64"))]@@ -1737,6 +1557,8 @@ pub fn transcribe_blocking(project_path: &Path,model: &str,language: &str,+ timeline: Option<&TimelineConfiguration>,+ recording_durations: &[f64],) -> Result<Vec<CaptionSegment>, String> {let model_file = model_path(model);if !project_path.exists() {@@ -1752,30 +1574,42 @@ pub fn transcribe_blocking());}- transcribe_project_with(project_path, |audio_path| {- let _guard = lock_transcription_worker_slot();- if is_parakeet_model(model) {- tracing::info!("Using Parakeet TDT engine");- let result = process_with_parakeet(audio_path, model_file.to_string_lossy().as_ref());- schedule_parakeet_release();- result- } else {- tracing::info!("Using Whisper engine");- let hints = transcription_hints();- let context = get_whisper_context(model_file.to_string_lossy().as_ref())?;- let result = process_with_whisper(audio_path, context, language, &hints);- release_whisper_context_after_transcription();- result- }- })+ let _guard = lock_transcription_worker_slot();+ let result =+ transcribe_project_with(project_path, timeline, recording_durations, |audio_path| {+ if is_parakeet_model(model) {+ tracing::info!("Using Parakeet TDT engine");+ process_with_parakeet(audio_path, model_file.to_string_lossy().as_ref())+ } else {+ tracing::info!("Using Whisper engine");+ let hints = transcription_hints();+ let context = get_whisper_context(model_file.to_string_lossy().as_ref())?;+ process_with_whisper(audio_path, context, language, &hints)+ }+ });+ if is_parakeet_model(model) {+ schedule_parakeet_release();+ } else {+ release_whisper_context_after_transcription();+ }+ result}/// Extracts the project's transcription audio to a temporary 16 kHz mono WAV/// and runs `engine` on it.fn transcribe_project_with(project_path: &Path,- engine: impl FnOnce(&Path) -> Result<Vec<CaptionSegment>, String>,+ timeline: Option<&TimelineConfiguration>,+ recording_durations: &[f64],+ mut engine: impl FnMut(&Path) -> Result<Vec<CaptionSegment>, String>,) -> Result<Vec<CaptionSegment>, String> {+ ffmpeg::init().map_err(|e| format!("Failed to initialise ffmpeg: {e}"))?;+ let meta = RecordingMeta::load_for_project(project_path)+ .map_err(|e| format!("Failed to read recording metadata: {e}"))?;+ let Some(studio) = meta.studio_meta() else {+ return Err("Only studio recordings can be transcribed".to_string());+ };+ studio.ensure_ordinary_media_access(&meta.project_path)?;let temp_dir = std::env::temp_dir().join(format!("cap-gpui-transcription-{}-{}",std::process::id(),@@ -1786,8 +1620,34 @@ fn transcribe_project_with(let audio_path = temp_dir.join("audio.wav");let result = (|| {- extract_audio_from_video(project_path, &audio_path)?;- engine(&audio_path)+ let mut segments = Vec::new();+ let mut any_audio = false;+ for range in cap_project::transcription_source_ranges(timeline, recording_durations) {+ if !extract_project_audio(+ &meta,+ studio,+ recording_durations,+ range.clone(),+ &audio_path,+ )? {+ continue;+ }+ any_audio = true;+ for mut segment in engine(&audio_path)? {+ segment.id = format!("segment-{}", segments.len());+ segment.start += range.start as f32;+ segment.end += range.start as f32;+ for word in &mut segment.words {+ word.start += range.start as f32;+ word.end += range.start as f32;+ }+ segments.push(segment);+ }+ }+ if !any_audio {+ return Err("No audio sources found in the selected timeline".to_string());+ }+ Ok(segments)})();let _ = std::fs::remove_dir_all(&temp_dir);@@ -1799,21 +1659,6 @@ fn transcribe_project_with(Ok(segments)}-/// `resolve_audio_extraction_source` (`captions.rs:221-243`), directory arm-/// only -- the gpui editor always opens `.cap` bundles, so a plain media file-/// is refused rather than decoded.-fn extract_audio_from_video(video_path: &Path, output_path: &Path) -> Result<(), String> {- let metadata = std::fs::metadata(video_path)- .map_err(|e| format!("Failed to read video path metadata: {e}"))?;- if !metadata.is_dir() {- return Err("Video path is not a recording directory".to_string());- }- if !video_path.join("recording-meta.json").is_file() {- return Err("Recording directory is missing recording-meta.json".to_string());- }- extract_project_audio(video_path, output_path)-}-// ---------------------------------------------------------------------------// Caption words & chunking// ---------------------------------------------------------------------------diff --git a/apps/desktop-gpui/src/transcription/verify_tests.rs b/apps/desktop-gpui/src/transcription/verify_tests.rsindex 39a2af3be..e00f2dbf8 100644--- a/apps/desktop-gpui/src/transcription/verify_tests.rs+++ b/apps/desktop-gpui/src/transcription/verify_tests.rs@@ -36,6 +36,8 @@ use cap_verify::{report::{self, Case, Tier},};+use cap_rendering::Video;+use super::*;const MAX_WORD_SECS: f32 = 2.5;@@ -335,10 +337,15 @@ fn takes_fixture() -> FixtureSpec {/// segments.fn check_project(case: &mut Case, project: &Path, takes: &Takes) {let mut tokens = Vec::new();- let result = transcribe_project_with(project, |audio_path| {- tokens = fake_tokens(audio_path)?;- caption_segments_from_tokens(tokens.clone())- });+ let result = transcribe_project_with(+ project,+ None,+ &takes.iter().map(|take| take.0).collect::<Vec<_>>(),+ |audio_path| {+ tokens = fake_tokens(audio_path)?;+ Ok(caption_segments_from_tokens(tokens.clone()))+ },+ );let segments = match result {Ok(segments) => segments,Err(e) => {@@ -526,7 +533,7 @@ fn instant_project_is_refused() {),}let mut called = false;- let result = transcribe_project_with(&project, |_| {+ let result = transcribe_project_with(&project, None, &[], |_| {called = true;Ok(Vec::new())});@@ -606,33 +613,25 @@ fn segmentation_case(id: &str, unicode: bool) {unicode,);let reference = reference_segments(&tokens);- match caption_segments_from_tokens(tokens.clone()) {- Err(e) if reference.is_empty() && e.contains("No speech") => empty += 1,- Err(e) => {- failed += 1;+ let product = caption_segments_from_tokens(tokens.clone());+ if product.is_empty() {+ empty += 1;+ }+ segments += product.len();+ let problems = compare_segments(&product, &reference, 0.0);+ if !problems.is_empty() {+ failed += 1;+ if failed <= 3 {case.problem("caption-segments",- format!("seed {seed}: unexpected error {e}"),+ format!(+ "seed {seed}: {} problems (first: {}); tokens {:?}",+ problems.len(),+ problems[0],+ tokens.iter().map(|t| t.text.as_str()).collect::<Vec<_>>()+ ),);}- Ok(product) => {- segments += product.len();- let problems = compare_segments(&product, &reference, 0.0);- if !problems.is_empty() {- failed += 1;- if failed <= 3 {- case.problem(- "caption-segments",- format!(- "seed {seed}: {} problems (first: {}); tokens {:?}",- problems.len(),- problems[0],- tokens.iter().map(|t| t.text.as_str()).collect::<Vec<_>>()- ),- );- }- }- }}}case.note(format!(@@ -650,3 +649,186 @@ fn segmentation_matches_reference() {fn segmentation_matches_reference_unicode() {segmentation_case("captions/app/segmentation/unicode", true);}++#[test]+fn trimmed_transcription_extracts_only_used_audio_and_restores_source_times() {+ let spec = takes_fixture();+ let project = fixture::ensure_fixture(&spec, fixture::write_fixture).expect("fixture");+ let takes = multi_segment_takes(&project, &spec);+ let durations: Vec<f64> = takes.iter().map(|take| take.0).collect();+ for (clips, expected_ranges, expected_starts) in [+ (+ serde_json::json!([+ {"recordingSegment": 0, "start": 1.0, "end": 3.0, "timescale": 2.0}+ ]),+ vec![1.0..3.0],+ vec![0.125],+ ),+ (+ serde_json::json!([+ {"recordingSegment": 1, "start": 1.0, "end": 3.0, "timescale": 0.5},+ {"recordingSegment": 0, "start": 3.0, "end": 4.0, "timescale": 2.0},+ {"recordingSegment": 0, "start": 0.5, "end": 1.5, "timescale": 1.0},+ {"recordingSegment": 0, "start": 1.0, "end": 2.0, "timescale": 1.0},+ {"recordingSegment": 0, "start": 3.0, "end": 4.0, "timescale": 1.0}+ ]),+ vec![0.5..2.0, 3.0..4.0, durations[0] + 1.0..durations[0] + 3.0],+ vec![0.5, 4.125, 4.75, 6.75],+ ),+ (+ serde_json::json!([+ {"recordingSegment":0,"start":4.0,"end":durations[0],"timescale":1.0},+ {"recordingSegment":1,"start":0.0,"end":1.0,"timescale":1.0}+ ]),+ vec![4.0..durations[0] + 1.0],+ vec![0.25],+ ),+ ] {+ let timeline: TimelineConfiguration = serde_json::from_value(serde_json::json!({+ "segments": clips, "zoomSegments": []+ }))+ .unwrap();+ let mut called = 0;+ let segments = transcribe_project_with(&project, Some(&timeline), &durations, |path| {+ let range = &expected_ranges[called];+ let audio = media::decode_audio(path, Some(MARKER_RATE)).unwrap();+ let duration = audio.mono().len() as f64 / f64::from(MARKER_RATE);+ assert!(+ (duration - (range.end - range.start)).abs() < 0.002,+ "extracted {duration}s, expected {}s",+ range.end - range.start+ );+ called += 1;+ Ok(vec![CaptionSegment {+ id: "segment-0".into(),+ start: 0.25,+ end: 0.5,+ text: "word".into(),+ words: vec![CaptionWord {+ text: "word".into(),+ start: 0.25,+ end: 0.5,+ }],+ }])+ })+ .unwrap();+ assert_eq!(called, expected_ranges.len());+ for (segment, range) in segments.iter().zip(expected_ranges) {+ assert!((f64::from(segment.start) - range.start - 0.25).abs() < 1e-5);+ assert!((f64::from(segment.end) - range.start - 0.5).abs() < 1e-5);+ assert_eq!(segment.words[0].start, segment.start);+ assert_eq!(segment.words[0].end, segment.end);+ }+ let ids: std::collections::HashSet<_> =+ segments.iter().map(|segment| &segment.id).collect();+ assert_eq!(ids.len(), segments.len());+ let tracks = derive_caption_track_segments(&segments, &timeline, &durations);+ assert_eq!(tracks.len(), expected_starts.len());+ for (track, expected) in tracks.iter().zip(expected_starts) {+ assert!((track.start - expected).abs() < 1e-5);+ }+ }+}++#[test]+fn trimmed_sources_match_full_audio_with_positive_and_negative_offsets() {+ let spec = takes_fixture();+ let project = fixture::ensure_fixture(&spec, fixture::write_fixture).unwrap();+ let meta = RecordingMeta::load_for_project(&project).unwrap();+ let Some(StudioRecordingMeta::MultipleSegments { inner }) = meta.studio_meta() else {+ panic!("studio")+ };+ let segment = &inner.segments[0];+ for audio in [+ segment.mic.as_ref().unwrap(),+ segment.system_audio.as_ref().unwrap(),+ ] {+ let path = meta.path(&audio.path);+ let decoded = cap_audio::AudioData::from_file(&path).unwrap();+ for offset_secs in [-0.4, 0.3] {+ let full = cap_editor::assemble_transcription_audio(+ &[TranscriptionAudioTake {+ display_duration_secs: 5.0,+ sources: vec![TranscriptionAudioSource {+ samples: decoded.samples().to_vec(),+ channels: usize::from(decoded.channels()),+ offset_secs,+ }],+ }],+ DECODE_SAMPLE_RATE,+ )+ .unwrap();+ for range in [0.0..1.0, 1.25..2.5, 3.0..4.0] {+ let source =+ decode_transcription_source(&path, offset_secs, range.clone()).unwrap();+ let trimmed = cap_editor::assemble_transcription_audio(+ &[TranscriptionAudioTake {+ display_duration_secs: range.end - range.start,+ sources: vec![source],+ }],+ DECODE_SAMPLE_RATE,+ )+ .unwrap();+ let start = (range.start * f64::from(DECODE_SAMPLE_RATE)).round() as usize;+ let reference = &full[start..start + trimmed.len()];+ let worst = trimmed+ .iter()+ .zip(reference)+ .fold(0.0_f32, |worst, (a, b)| worst.max((a - b).abs()));+ assert!(+ worst < 0.001,+ "offset {offset_secs}, range {range:?}, worst sample error {worst}"+ );+ }+ }+ }+}++#[test]+fn transcription_keeps_speech_after_empty_ranges_and_cleans_up_on_failure() {+ let spec = takes_fixture();+ let project = fixture::ensure_fixture(&spec, fixture::write_fixture).unwrap();+ let takes = multi_segment_takes(&project, &spec);+ let durations: Vec<_> = takes.iter().map(|take| take.0).collect();+ let timeline = serde_json::from_value(serde_json::json!({"segments": [+ {"start":0.0,"end":1.0,"timescale":1.0},+ {"start":3.0,"end":4.0,"timescale":1.0}+ ], "zoomSegments": []}))+ .unwrap();+ let mut calls = 0;+ let mut temporary = PathBuf::new();+ let result = transcribe_project_with(&project, Some(&timeline), &durations, |path| {+ temporary = path.to_path_buf();+ calls += 1;+ if calls == 1 {+ return Ok(Vec::new());+ }+ Ok(caption_segments_from_tokens([CaptionWord {+ text: "speech".into(),+ start: 0.1,+ end: 0.2,+ }]))+ })+ .unwrap();+ assert_eq!(calls, 2);+ assert!((result[0].words[0].start - 3.1).abs() < 1e-5);+ assert!(!temporary.parent().unwrap().exists());+ let error = transcribe_project_with(&project, Some(&timeline), &durations, |path| {+ temporary = path.to_path_buf();+ Err("test engine failure".into())+ })+ .unwrap_err();+ assert_eq!(error, "test engine failure");+ assert!(!temporary.parent().unwrap().exists());+ let empty = transcribe_project_with(&project, Some(&timeline), &durations, |_| Ok(Vec::new()))+ .unwrap_err();+ assert_eq!(empty, "No speech detected in the audio");+ assert!(caption_segments_from_tokens(Vec::new()).is_empty());+ let empty_timeline =+ serde_json::from_value(serde_json::json!({"segments":[],"zoomSegments":[]})).unwrap();+ let error = transcribe_project_with(&project, Some(&empty_timeline), &durations, |_| {+ panic!("an empty timeline must not run the model")+ })+ .unwrap_err();+ assert!(error.contains("No audio sources"));+}diff --git a/crates/project/src/caption_timing.rs b/crates/project/src/caption_timing.rsindex 63700fc2a..6b6b5e622 100644--- a/crates/project/src/caption_timing.rs+++ b/crates/project/src/caption_timing.rs@@ -152,6 +152,40 @@ fn build_source_to_edited_mappings(.collect()}+pub fn transcription_source_ranges(+ timeline: Option<&TimelineConfiguration>,+ recording_durations: &[f64],+) -> Vec<std::ops::Range<f64>> {+ let Some(timeline) = timeline else {+ let duration = recording_durations.iter().sum();+ return if duration > 0.0 {+ vec![0.0..duration]+ } else {+ Vec::new()+ };+ };+ let mut ranges: Vec<_> = build_source_to_edited_mappings(timeline, recording_durations)+ .into_iter()+ .zip(&timeline.segments)+ .filter_map(|(mapping, segment)| {+ let duration = *recording_durations.get(segment.recording_clip as usize)?;+ let offset = mapping.source_start - segment.start;+ let start = segment.start.max(0.0);+ let end = segment.end.min(duration);+ (start < end).then_some(offset + start..offset + end)+ })+ .collect();+ ranges.sort_by(|a, b| a.start.total_cmp(&b.start));+ let mut merged: Vec<std::ops::Range<f64>> = Vec::with_capacity(ranges.len());+ for range in ranges {+ match merged.last_mut() {+ Some(last) if range.start <= last.end => last.end = last.end.max(range.end),+ _ => merged.push(range),+ }+ }+ merged+}+fn map_time_range_within_mapping(start: f64,end: f64,@@ -429,6 +463,48 @@ mod tests {use super::*;use crate::CaptionsData;+ #[test]+ fn transcription_ranges_follow_source_cuts_independent_of_speed_and_clip_order() {+ let timeline = serde_json::from_value(serde_json::json!({+ "segments": [+ {"recordingSegment":1,"start":1.0,"end":3.0,"timescale":0.5},+ {"recordingSegment":0,"start":5.0,"end":6.0,"timescale":2.0},+ {"recordingSegment":0,"start":1.0,"end":3.0,"timescale":1.0},+ {"recordingSegment":0,"start":2.0,"end":4.0,"timescale":1.0},+ {"recordingSegment":0,"start":4.0,"end":4.5,"timescale":1.0},+ {"recordingSegment":0,"start":1.0,"end":3.0,"timescale":2.0}+ ], "zoomSegments": []+ }))+ .unwrap();+ assert_eq!(+ transcription_source_ranges(Some(&timeline), &[10.0, 4.0]),+ vec![1.0..4.5, 5.0..6.0, 11.0..13.0]+ );+ }++ #[test]+ fn transcription_ranges_preserve_empty_timelines_and_clip_to_sources() {+ let mut timeline: TimelineConfiguration =+ serde_json::from_value(serde_json::json!({"segments": [], "zoomSegments": []}))+ .unwrap();+ assert!(transcription_source_ranges(Some(&timeline), &[30.0]).is_empty());+ assert_eq!(+ transcription_source_ranges(None, &[30.0, 10.0]),+ vec![0.0..40.0]+ );+ timeline.segments = serde_json::from_value(serde_json::json!([+ {"recordingSegment":0,"start":-1.0,"end":4.0,"timescale":1.0},+ {"recordingSegment":0,"start":8.0,"end":40.0,"timescale":1.0},+ {"recordingSegment":2,"start":0.0,"end":3.0,"timescale":1.0},+ {"recordingSegment":0,"start":5.0,"end":5.0,"timescale":1.0}+ ]))+ .unwrap();+ assert_eq!(+ transcription_source_ranges(Some(&timeline), &[10.0]),+ vec![0.0..4.0, 8.0..10.0]+ );+ }+fn project(source_timed: bool, start: f32, end: f32) -> ProjectConfiguration {ProjectConfiguration {timeline: Some(serde_json::from_value(serde_json::json!({diff --git a/crates/project/src/lib.rs b/crates/project/src/lib.rsindex dea389ee1..6e82e759a 100644--- a/crates/project/src/lib.rs+++ b/crates/project/src/lib.rs@@ -6,7 +6,10 @@ pub mod keyboard;mod meta;pub use animated_gradient::*;-pub use caption_timing::{derive_caption_track_segments, source_caption_id, synchronize_captions};+pub use caption_timing::{+ derive_caption_track_segments, source_caption_id, synchronize_captions,+ transcription_source_ranges,+};pub use configuration::*;pub use cursor::*;pub use keyboard::*;