Open source Loom alternative. Beautiful, shareable screen recordings.
Rust appcapcossloommacnextjsnextjs15open-sourceossreactrecordrustscreen-capturescreen-recorderscreenshotsolidjstauritauri-apptypescriptvite
refactor: move caption word utilities into cap-project (#3274)
Ker102e1ded277d4cb
diff --git a/apps/desktop-gpui/src/editor_window.rs b/apps/desktop-gpui/src/editor_window.rsindex e89dee550..046a18b1f 100644--- a/apps/desktop-gpui/src/editor_window.rs+++ b/apps/desktop-gpui/src/editor_window.rs@@ -49,6 +49,7 @@ use std::{use cap_cursor_info::CursorFamily;use cap_editor::{EditorFrameOutput, EditorInstance, EditorState};+use cap_project::caption_words::caption_text_from_words;use cap_project::{ClipSpeedAudioMode, Cursors, ProjectConfiguration, RecordingMeta, RecordingMetaInner,StudioRecordingMeta, TimelineConfiguration, XY,@@ -5346,44 +5347,6 @@ pub(crate) struct EditorNotice {/// How long a notice stays up.const EDITOR_NOTICE_DURATION: Duration = Duration::from_millis(2500);-fn caption_text_from_words(words: &[cap_project::CaptionWord]) -> String {- let mut text = String::new();- for word in words {- let value = word.text.trim();- if value.is_empty() {- continue;- }- let attaches = value.chars().next().is_some_and(|value| {- matches!(- value,- ',' | '.'- | '!'- | '?'- | ';'- | ':'- | '%'- | ')'- | ']'- | '}'- | '\''- | '’'- | '、'- | '。'- | '!'- | '?'- | ';'- | ':'- | ','- )- });- if !text.is_empty() && !attaches {- text.push(' ');- }- text.push_str(value);- }- text-}-fn fresh_caption_source_id(project: &ProjectConfiguration) -> String {use std::hash::{BuildHasher, Hasher};diff --git a/apps/desktop-gpui/src/transcription.rs b/apps/desktop-gpui/src/transcription.rsindex 5ea9ccdaa..92fa942a9 100644--- a/apps/desktop-gpui/src/transcription.rs+++ b/apps/desktop-gpui/src/transcription.rs@@ -1856,167 +1856,10 @@ fn extract_audio_from_video(video_path: &Path, output_path: &Path) -> Result<(),// Caption words & chunking// ----------------------------------------------------------------------------/// `MAX_CAPTION_WORD_DURATION` (`captions.rs:75`): Whisper/Parakeet sometimes-/// stretch a trailing word's end across a following silence, which leaves the-/// rendered caption stuck on screen and duplicates the word across timeline-/// cuts once projected.-pub const MAX_CAPTION_WORD_DURATION: f32 = 2.5;-/// `captions.rs:67-69`.-const TARGET_CAPTION_WORDS_PER_SEGMENT: usize = 6;-const MAX_CAPTION_WORDS_PER_SEGMENT: usize = 8;-const MIN_FINAL_CAPTION_WORDS: usize = 3;--/// `caption_char_attaches_to_previous` (`captions.rs:745-767`).-fn caption_char_attaches_to_previous(value: char) -> bool {- matches!(- value,- ',' | '.'- | '!'- | '?'- | ';'- | ':'- | '%'- | ')'- | ']'- | '}'- | '\''- | '’'- | '、'- | '。'- | '!'- | '?'- | ';'- | ':'- | ','- )-}--fn caption_token_attaches_to_previous(text: &str) -> bool {- text.trim()- .chars()- .next()- .is_some_and(caption_char_attaches_to_previous)-}--/// `caption_boundary_word_is_weak` (`captions.rs:769-801`).-fn caption_boundary_word_is_weak(word: &CaptionWord) -> bool {- let normalized = word- .text- .trim()- .chars()- .filter(|c| c.is_alphanumeric())- .collect::<String>()- .to_lowercase();-- normalized.chars().count() <= 1- || matches!(- normalized.as_str(),- "an" | "as"- | "at"- | "be"- | "by"- | "do"- | "he"- | "if"- | "in"- | "is"- | "it"- | "me"- | "my"- | "of"- | "on"- | "or"- | "so"- | "to"- | "up"- | "we"- )-}--/// `normalize_caption_words` (`captions.rs:803-831`).-fn normalize_caption_words(words: Vec<CaptionWord>) -> Vec<CaptionWord> {- let mut normalized: Vec<CaptionWord> = Vec::with_capacity(words.len());-- for word in words {- let text = word.text.trim();- if text.is_empty() {- continue;- }-- if caption_token_attaches_to_previous(text)- && let Some(previous) = normalized.last_mut()- {- previous.text.push_str(text);- previous.end = word.end;- } else {- normalized.push(CaptionWord {- text: text.to_string(),- start: word.start,- end: word.end,- });- }- }-- for word in &mut normalized {- word.end = word.end.min(word.start + MAX_CAPTION_WORD_DURATION);- }-- normalized-}--/// `caption_text_from_words` (`captions.rs:833-849`).-fn caption_text_from_words<'a>(words: impl IntoIterator<Item = &'a CaptionWord>) -> String {- let mut text = String::new();-- for word in words {- let word_text = word.text.trim();- if word_text.is_empty() {- continue;- }-- if !text.is_empty() && !caption_token_attaches_to_previous(word_text) {- text.push(' ');- }- text.push_str(word_text);- }-- text-}--/// `caption_word_chunks` (`captions.rs:851-883`).-fn caption_word_chunks(words: &[CaptionWord]) -> Vec<&[CaptionWord]> {- let mut chunks = Vec::new();- let mut start = 0;-- while start < words.len() {- let remaining = words.len() - start;- if remaining <= TARGET_CAPTION_WORDS_PER_SEGMENT {- chunks.push(&words[start..]);- break;- }-- let mut end = (start + TARGET_CAPTION_WORDS_PER_SEGMENT).min(words.len());- while end < words.len()- && caption_boundary_word_is_weak(&words[end - 1])- && end - start < MAX_CAPTION_WORDS_PER_SEGMENT- {- end += 1;- }-- let remaining_after = words.len() - end;- if remaining_after > 0- && remaining_after < MIN_FINAL_CAPTION_WORDS- && caption_boundary_word_is_weak(&words[end])- {- end = words.len();- }-- chunks.push(&words[start..end]);- start = end;- }-- chunks-}+use cap_project::caption_words::{+ caption_text_from_words, caption_word_chunks, normalize_caption_words,+ MAX_CAPTION_WORD_DURATION,+};// ---------------------------------------------------------------------------// Track derivation -- deriveCaptionTrackSegments, in Rustdiff --git a/apps/desktop/src-tauri/src/captions.rs b/apps/desktop/src-tauri/src/captions.rsindex b64305aa4..90d6712ab 100644--- a/apps/desktop/src-tauri/src/captions.rs+++ b/apps/desktop/src-tauri/src/captions.rs@@ -74,15 +74,11 @@ lazy_static::lazy_static! {}const WHISPER_SAMPLE_RATE: u32 = 16000;-const TARGET_CAPTION_WORDS_PER_SEGMENT: usize = 6;-const MAX_CAPTION_WORDS_PER_SEGMENT: usize = 8;-const MIN_FINAL_CAPTION_WORDS: usize = 3;-// Whisper/Parakeet sometimes stretch a trailing word's end across a following-// silence (e.g. a 16s "seconds."), which leaves the rendered caption stuck on-// screen and duplicates the word across timeline cuts once projected. Real-// spoken words never approach this, so cap each word's duration to keep timing-// tied to speech rather than silence.-const MAX_CAPTION_WORD_DURATION: f32 = 2.5;++use cap_project::caption_words::{+ caption_text_from_words, caption_word_chunks, normalize_caption_words,+ MAX_CAPTION_WORD_DURATION,+};#[cfg(not(all(target_os = "macos", target_arch = "x86_64")))]struct CachedParakeetContext {@@ -769,154 +765,6 @@ fn is_special_token(token_text: &str) -> bool {is_special}-fn caption_token_attaches_to_previous(text: &str) -> bool {- let Some(first_char) = text.trim().chars().next() else {- return false;- };-- caption_char_attaches_to_previous(first_char)-}--fn caption_char_attaches_to_previous(value: char) -> bool {- matches!(- value,- ',' | '.'- | '!'- | '?'- | ';'- | ':'- | '%'- | ')'- | ']'- | '}'- | '\''- | '’'- | '、'- | '。'- | '!'- | '?'- | ';'- | ':'- | ','- )-}--fn caption_boundary_word_is_weak(word: &CaptionWord) -> bool {- let normalized = word- .text- .trim()- .chars()- .filter(|c| c.is_alphanumeric())- .collect::<String>()- .to_lowercase();-- normalized.len() <= 1- || matches!(- normalized.as_str(),- "an" | "as"- | "at"- | "be"- | "by"- | "do"- | "he"- | "if"- | "in"- | "is"- | "it"- | "me"- | "my"- | "of"- | "on"- | "or"- | "so"- | "to"- | "up"- | "we"- )-}--fn normalize_caption_words(words: Vec<CaptionWord>) -> Vec<CaptionWord> {- let mut normalized: Vec<CaptionWord> = Vec::with_capacity(words.len());-- for word in words {- let text = word.text.trim();- if text.is_empty() {- continue;- }-- if caption_token_attaches_to_previous(text)- && let Some(previous) = normalized.last_mut()- {- previous.text.push_str(text);- previous.end = word.end;- } else {- normalized.push(CaptionWord {- text: text.to_string(),- start: word.start,- end: word.end,- });- }- }-- for word in &mut normalized {- word.end = word.end.min(word.start + MAX_CAPTION_WORD_DURATION);- }-- normalized-}--fn caption_text_from_words<'a>(words: impl IntoIterator<Item = &'a CaptionWord>) -> String {- let mut text = String::new();-- for word in words {- let word_text = word.text.trim();- if word_text.is_empty() {- continue;- }-- if !text.is_empty() && !caption_token_attaches_to_previous(word_text) {- text.push(' ');- }- text.push_str(word_text);- }-- text-}--fn caption_word_chunks(words: &[CaptionWord]) -> Vec<&[CaptionWord]> {- let mut chunks = Vec::new();- let mut start = 0;-- while start < words.len() {- let remaining = words.len() - start;- if remaining <= TARGET_CAPTION_WORDS_PER_SEGMENT {- chunks.push(&words[start..]);- break;- }-- let mut end = (start + TARGET_CAPTION_WORDS_PER_SEGMENT).min(words.len());- while end < words.len()- && caption_boundary_word_is_weak(&words[end - 1])- && end - start < MAX_CAPTION_WORDS_PER_SEGMENT- {- end += 1;- }-- let remaining_after = words.len() - end;- if remaining_after > 0- && remaining_after < MIN_FINAL_CAPTION_WORDS- && caption_boundary_word_is_weak(&words[end])- {- end = words.len();- }-- chunks.push(&words[start..end]);- start = end;- }-- chunks-}-fn normalize_audio_for_transcription(samples: &mut [f32]) -> f32 {if samples.is_empty() {return 1.0;diff --git a/crates/project/src/caption_timing.rs b/crates/project/src/caption_timing.rsindex b938177af..5f9fa655b 100644--- a/crates/project/src/caption_timing.rs+++ b/crates/project/src/caption_timing.rs@@ -1,10 +1,9 @@+use crate::caption_words::{MAX_CAPTION_WORD_DURATION, caption_text_from_words};use crate::{CaptionSegment, CaptionTrackSegment, CaptionWord, ProjectConfiguration, TimelineConfiguration,};use std::collections::HashMap;-const MAX_CAPTION_WORD_DURATION: f32 = 2.5;-pub(crate) fn clip_timeline_offsets(timeline: &TimelineConfiguration) -> Vec<f64> {let mut offset = 0.0;timeline@@ -22,55 +21,6 @@ pub(crate) fn clip_timeline_offsets(timeline: &TimelineConfiguration) -> Vec<f64.collect()}-fn caption_char_attaches_to_previous(value: char) -> bool {- matches!(- value,- ',' | '.'- | '!'- | '?'- | ';'- | ':'- | '%'- | ')'- | ']'- | '}'- | '\''- | '’'- | '、'- | '。'- | '!'- | '?'- | ';'- | ':'- | ','- )-}--fn caption_token_attaches_to_previous(text: &str) -> bool {- text.trim()- .chars()- .next()- .is_some_and(caption_char_attaches_to_previous)-}--fn caption_text_from_words<'a>(words: impl IntoIterator<Item = &'a CaptionWord>) -> String {- let mut text = String::new();-- for word in words {- let word_text = word.text.trim();- if word_text.is_empty() {- continue;- }-- if !text.is_empty() && !caption_token_attaches_to_previous(word_text) {- text.push(' ');- }- text.push_str(word_text);- }-- text-}-const CAPTION_EDL_SEPARATOR: &str = "::edl";pub fn source_caption_id(track_id: &str) -> &str {diff --git a/crates/project/src/caption_words.rs b/crates/project/src/caption_words.rsnew file mode 100644index 000000000..767ed15ff--- /dev/null+++ b/crates/project/src/caption_words.rs@@ -0,0 +1,284 @@+//! Shared caption word normalization and segmentation.+//!+//! Single source of truth for how transcribed words are cleaned up and grouped+//! into on-screen caption segments. This used to be copy-pasted across the+//! Tauri captions pipeline (`apps/desktop/src-tauri/src/captions.rs`), the gpui+//! transcription pipeline (`apps/desktop-gpui/src/transcription.rs`),+//! `caption_timing`, and `editor_window`, and the copies drifted apart.++use crate::CaptionWord;++/// Whisper/Parakeet sometimes stretch a trailing word's end across a following+/// silence (e.g. a 16s "seconds."), which leaves the rendered caption stuck on+/// screen and duplicates the word across timeline cuts once projected. Real+/// spoken words never approach this, so cap each word's duration to keep timing+/// tied to speech rather than silence.+pub const MAX_CAPTION_WORD_DURATION: f32 = 2.5;+pub const TARGET_CAPTION_WORDS_PER_SEGMENT: usize = 6;+pub const MAX_CAPTION_WORDS_PER_SEGMENT: usize = 8;+pub const MIN_FINAL_CAPTION_WORDS: usize = 3;++fn caption_char_attaches_to_previous(value: char) -> bool {+ matches!(+ value,+ ',' | '.'+ | '!'+ | '?'+ | ';'+ | ':'+ | '%'+ | ')'+ | ']'+ | '}'+ | '\''+ | '’'+ | '、'+ | '。'+ | '!'+ | '?'+ | ';'+ | ':'+ | ','+ )+}++fn caption_token_attaches_to_previous(text: &str) -> bool {+ text.trim()+ .chars()+ .next()+ .is_some_and(caption_char_attaches_to_previous)+}++/// A caption segment should not end on a one letter word or a short connector+/// when more words follow, since that reads badly on screen. Comparing+/// `chars().count()` matters: a byte length check counts non-ASCII letters like+/// "à" as two and treats them as strong boundary words.+pub fn caption_boundary_word_is_weak(word: &CaptionWord) -> bool {+ let normalized = word+ .text+ .trim()+ .chars()+ .filter(|c| c.is_alphanumeric())+ .collect::<String>()+ .to_lowercase();++ normalized.chars().count() <= 1+ || matches!(+ normalized.as_str(),+ "an" | "as"+ | "at"+ | "be"+ | "by"+ | "do"+ | "he"+ | "if"+ | "in"+ | "is"+ | "it"+ | "me"+ | "my"+ | "of"+ | "on"+ | "or"+ | "so"+ | "to"+ | "up"+ | "we"+ )+}++/// Drops empty tokens, attaches punctuation-only tokens to the previous word,+/// and clamps inflated trailing word durations.+pub fn normalize_caption_words(words: Vec<CaptionWord>) -> Vec<CaptionWord> {+ let mut normalized: Vec<CaptionWord> = Vec::with_capacity(words.len());++ for word in words {+ let text = word.text.trim();+ if text.is_empty() {+ continue;+ }++ if caption_token_attaches_to_previous(text)+ && let Some(previous) = normalized.last_mut()+ {+ previous.text.push_str(text);+ previous.end = word.end;+ } else {+ normalized.push(CaptionWord {+ text: text.to_string(),+ start: word.start,+ end: word.end,+ });+ }+ }++ for word in &mut normalized {+ word.end = word.end.min(word.start + MAX_CAPTION_WORD_DURATION);+ }++ normalized+}++pub fn caption_text_from_words<'a>(words: impl IntoIterator<Item = &'a CaptionWord>) -> String {+ let mut text = String::new();++ for word in words {+ let word_text = word.text.trim();+ if word_text.is_empty() {+ continue;+ }++ if !text.is_empty() && !caption_token_attaches_to_previous(word_text) {+ text.push(' ');+ }+ text.push_str(word_text);+ }++ text+}++/// Groups normalized words into caption segments of at most+/// [`MAX_CAPTION_WORDS_PER_SEGMENT`] words, aiming for+/// [`TARGET_CAPTION_WORDS_PER_SEGMENT`]. The boundary word may move past the+/// target to avoid ending on a weak word, and a trailing fragment shorter than+/// [`MIN_FINAL_CAPTION_WORDS`] merges into the previous segment when it starts+/// with a weak word.+pub fn caption_word_chunks(words: &[CaptionWord]) -> Vec<&[CaptionWord]> {+ let mut chunks = Vec::new();+ let mut start = 0;++ while start < words.len() {+ let remaining = words.len() - start;+ if remaining <= TARGET_CAPTION_WORDS_PER_SEGMENT {+ chunks.push(&words[start..]);+ break;+ }++ let mut end = (start + TARGET_CAPTION_WORDS_PER_SEGMENT).min(words.len());+ while end < words.len()+ && caption_boundary_word_is_weak(&words[end - 1])+ && end - start < MAX_CAPTION_WORDS_PER_SEGMENT+ {+ end += 1;+ }++ let remaining_after = words.len() - end;+ if remaining_after > 0+ && remaining_after < MIN_FINAL_CAPTION_WORDS+ && caption_boundary_word_is_weak(&words[end])+ {+ end = words.len();+ }++ chunks.push(&words[start..end]);+ start = end;+ }++ chunks+}++#[cfg(test)]+mod tests {+ use super::*;++ fn word(text: &str, index: usize) -> CaptionWord {+ CaptionWord {+ text: text.to_string(),+ start: index as f32,+ end: index as f32 + 0.5,+ }+ }++ #[test]+ fn normalize_caption_words_attaches_punctuation() {+ let words = normalize_caption_words(vec![+ word("test", 0),+ word(",", 1),+ word("test", 2),+ word(".", 3),+ ]);++ assert_eq!(caption_text_from_words(&words), "test, test.");+ assert_eq!(words.len(), 2);+ }++ #[test]+ fn normalize_caption_words_clamps_inflated_trailing_word() {+ let words = normalize_caption_words(vec![CaptionWord {+ text: "seconds.".to_string(),+ start: 53.92,+ end: 70.16,+ }]);++ assert_eq!(words.len(), 1);+ assert!((words[0].end - (53.92 + MAX_CAPTION_WORD_DURATION)).abs() < 1e-4);+ }++ #[test]+ fn normalize_caption_words_keeps_normal_word_durations() {+ let words = normalize_caption_words(vec![CaptionWord {+ text: "hello".to_string(),+ start: 1.0,+ end: 1.4,+ }]);++ assert_eq!(words.len(), 1);+ assert!((words[0].end - 1.4).abs() < 1e-4);+ }++ #[test]+ fn caption_word_chunks_do_not_end_on_short_connector_when_more_words_follow() {+ for connector in ["I", "à", "à,", "a\u{300}", "日"] {+ let words = [+ "This", "is", "where", "we", "record", connector, "want", "clean", "captions",+ ]+ .iter()+ .enumerate()+ .map(|(index, text)| word(text, index))+ .collect::<Vec<_>>();++ let chunks = caption_word_chunks(&words);++ assert_eq!(+ caption_text_from_words(chunks[0]),+ format!("This is where we record {connector} want"),+ "connector {connector}"+ );+ assert_eq!(caption_text_from_words(chunks[1]), "clean captions");+ }+ }++ #[test]+ fn caption_word_chunks_merge_short_tail_starting_with_connector() {+ for connector in ["I", "à", "à,", "a\u{300}", "日"] {+ let words = [+ "These", "captions", "should", "stay", "grouped", "together", connector, "Paris",+ ]+ .iter()+ .enumerate()+ .map(|(index, text)| word(text, index))+ .collect::<Vec<_>>();++ let chunks = caption_word_chunks(&words);++ assert_eq!(chunks.len(), 1, "connector {connector}");+ assert_eq!(+ caption_text_from_words(chunks[0]),+ format!("These captions should stay grouped together {connector} Paris")+ );+ }+ }++ #[test]+ fn caption_word_chunks_target_size_groups_strong_boundaries() {+ let words = (0..12)+ .map(|index| word(&format!("w{index}"), index))+ .collect::<Vec<_>>();++ let chunks = caption_word_chunks(&words);++ assert_eq!(chunks.len(), 2);+ assert_eq!(chunks[0].len(), 6);+ assert_eq!(chunks[1].len(), 6);+ }+}diff --git a/crates/project/src/lib.rs b/crates/project/src/lib.rsindex 080355b05..f4bdcf586 100644--- a/crates/project/src/lib.rs+++ b/crates/project/src/lib.rs@@ -1,5 +1,6 @@mod animated_gradient;mod caption_timing;+pub mod caption_words;#[cfg(test)]mod config_write_tests;mod configuration;@@ -9,6 +10,11 @@ mod meta;pub use animated_gradient::*;pub use caption_timing::{derive_caption_track_segments, source_caption_id, synchronize_captions};+pub use caption_words::{+ MAX_CAPTION_WORD_DURATION, MAX_CAPTION_WORDS_PER_SEGMENT, MIN_FINAL_CAPTION_WORDS,+ TARGET_CAPTION_WORDS_PER_SEGMENT, caption_boundary_word_is_weak, caption_text_from_words,+ caption_word_chunks, normalize_caption_words,+};pub use configuration::*;pub use cursor::*;pub use keyboard::*;diff --git a/crates/rendering/src/layers/captions.rs b/crates/rendering/src/layers/captions.rsindex 5a6d10bbe..c26e715b3 100644--- a/crates/rendering/src/layers/captions.rs+++ b/crates/rendering/src/layers/captions.rs@@ -139,9 +139,9 @@ const BASE_TEXT_OPACITY: f32 = 0.8;const BOUNCE_OFFSET_PIXELS: f32 = 8.0;// Safety net for caption segments whose trailing word end was stretched across a// silence by transcription (e.g. a 16s "seconds."). Without this, such a segment-// stays on screen for the whole inflated duration. Kept in sync with-// MAX_CAPTION_WORD_DURATION in the desktop transcription/projection layers.-const MAX_CAPTION_WORD_DURATION: f64 = 2.5;+// stays on screen for the whole inflated duration. Value comes from the shared+// caption word utilities so it stays in sync with the transcription layers.+const MAX_CAPTION_WORD_DURATION: f64 = cap_project::caption_words::MAX_CAPTION_WORD_DURATION as f64;fn ease_out_cubic(t: f32) -> f32 {let t = t.clamp(0.0, 1.0);