The truly free, open-source, local-first, cross-platform CapCut replacement. Scriptable through a JSON-RPC API & MCP.
Rust ffmpegfree-video-editormodel-context-protocolnon-linear-editoroffline-firstopen-source-video-editorself-hostedvideo-editingvideo-editing-automationvideo-editing-softwarevideo-editing-toolsvideo-editorvoice-cloningyoutube-automationyoutube-shorts
feat(voices): a saved voice keeps a line about it, and a recording is listened to for where its voice is (#518)
Jareerfa2c51689c5c
diff --git a/src/crates/concat-host/src/voices.rs b/src/crates/concat-host/src/voices.rsindex ec130e64..72aab8c5 100644--- a/src/crates/concat-host/src/voices.rs+++ b/src/crates/concat-host/src/voices.rs@@ -8,7 +8,8 @@//! (Pocket TTS, Chatterbox). Kept, it is the few seconds of the recording//! the model listens to, cut out and written as a WAV of its own - not the//! file it came from, which may be a long video in one project - beside a-//! `voice.json` with its name and, when given one, a picture:+//! `voice.json` with its name and, when given them, a picture and a line+//! about it://!//! ```text//! <library>/voices/<id>/voice.json@@ -40,6 +41,32 @@ const PICTURE_CAP: u64 = 8 * 1024 * 1024;/// The pictures a voice can wear.pub const PICTURE_EXTENSIONS: [&str; 4] = ["png", "jpg", "jpeg", "webp"];+/// The recordings a voice can be imported from: the sound files the+/// library takes, and the videos the bin takes, for their soundtrack.+pub const RECORDING_EXTENSIONS: [&str; 15] = [+ "mp3", "wav", "aac", "m4a", "flac", "ogg", "opus", "aif", "aiff", "mp4", "mov", "mkv", "webm",+ "avi", "m4v",+];++/// The most characters a voice's description keeps: a line under its+/// name, not a biography.+pub const DESCRIPTION_CHARS: usize = 140;++/// The frame a recording is measured in when its voice is looked for, in+/// seconds: about the shortest sound of a syllable, the usual frame of a+/// voice detector.+const FRAME_SECONDS: f64 = 0.02;++/// How loud a frame must be to count as speech, against the loud end of+/// the recording (see [`densest`]): twenty decibels under it. Relative, so+/// a quiet take and a hot one are read alike, and the hiss of a room+/// between the sentences is not taken for them.+const SPEECH_FLOOR: f32 = 0.1;++/// A recording whose loud end is under this, about eighty decibels down,+/// is silent: there is no voice in it to look for.+const SILENCE: f32 = 1e-4;+/// One saved voice.#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]#[serde(rename_all = "camelCase")]@@ -51,6 +78,10 @@ pub struct SavedVoice {/// The picture's file name in its folder, when it has one.#[serde(default)]pub picture: Option<String>,+ /// A line about it, when the user wrote one: what the shelf says+ /// under its name.+ #[serde(default)]+ pub description: Option<String>,/// When it was saved, as Unix seconds: the shelf's order, newest first.#[serde(default)]pub created: u64,@@ -101,19 +132,25 @@ pub fn list(dir: &Path) -> Vec<SavedVoice> {}/// Saves `name` as a voice: [`SAMPLE_SECONDS`] of `source` from `start`,-/// mono, written as a WAV of its own, and `picture` copied beside it.-/// Slow - a decode - so a worker's.+/// mono, written as a WAV of its own, and `picture` copied beside it. The+/// description is kept trimmed and cut to [`DESCRIPTION_CHARS`]; a blank+/// one is none. Slow - a decode - so a worker's.pub fn save(dir: &Path,name: &str,source: &Path,start: f64,picture: Option<&Path>,+ description: Option<&str>,) -> Result<SavedVoice, String> {let name = name.trim();if name.is_empty() {return Err("a voice needs a name".to_owned());}+ let description = description+ .map(str::trim)+ .filter(|line| !line.is_empty())+ .map(|line| line.chars().take(DESCRIPTION_CHARS).collect::<String>());let options = concat_media::AudioOptions {start: (start.is_finite() && start > 0.0).then_some(start),duration: Some(SAMPLE_SECONDS),@@ -163,6 +200,7 @@ pub fn save(id: id.clone(),name: name.to_owned(),picture,+ description,created,dir: folder.clone(),};@@ -179,6 +217,96 @@ pub fn save(written}+/// What listening to a recording found: how long it is, and where in it+/// the [`SAMPLE_SECONDS`] most given to speech start.+#[derive(Clone, Copy, Debug, PartialEq)]+pub struct Heard {+ /// The recording's length, in seconds.+ pub duration: f64,+ /// Seconds in where the kept sample is best taken from.+ pub start: f64,+}++/// Listens to `source` for where its voice is, so a recording that opens+/// on silence, a room or a jingle is kept from where the speaking is and+/// not from its first second. Its sound is decoded as [`save`] decodes it,+/// mono at the models' rate, and measured a [`FRAME_SECONDS`] frame at a+/// time as it comes, so an hour of video costs a few hundred thousand+/// levels rather than the hour's samples; see [`densest`] for the window+/// taken. Slow - the whole file decoded - so a worker's.+pub fn listen(source: &Path) -> Result<Heard, String> {+ let options = concat_media::AudioOptions {+ rate: SAMPLE_RATE,+ channels: 1,+ format: concat_media::SampleFormat::F32,+ ..Default::default()+ };+ let unheard =+ |error: concat_media::Error| format!("could not listen to {}: {error}", source.display());+ let mut decoder = concat_media::AudioDecoder::open(source, &options).map_err(unheard)?;+ let frame = (f64::from(SAMPLE_RATE) * FRAME_SECONDS).round() as usize;+ let mut levels = Vec::new();+ let mut power = 0.0_f64;+ let mut filled = 0;+ let mut samples = 0_usize;+ while let Some(chunk) = decoder.next_f32().map_err(unheard)? {+ samples += chunk.len();+ for sample in chunk {+ power += f64::from(sample) * f64::from(sample);+ filled += 1;+ if filled == frame {+ levels.push((power / frame as f64).sqrt() as f32);+ power = 0.0;+ filled = 0;+ }+ }+ }+ let window = (SAMPLE_SECONDS / FRAME_SECONDS).round() as usize;+ Ok(Heard {+ duration: samples as f64 / f64::from(SAMPLE_RATE),+ start: densest(&levels, window) as f64 * FRAME_SECONDS,+ })+}++/// The first frame of the run of `window` frames most given to speech in+/// `levels`, each a frame's RMS. A frame is speech when it is louder than+/// [`SPEECH_FLOOR`] of the recording's loud end - the level a twentieth of+/// the frames are louder than, rather than the loudest, so one slammed+/// door does not set the bar - and the run with the most speech frames is+/// taken; between runs with as many, the one with the most energy, which+/// is the one spoken up in; between runs alike in both, the earliest. A+/// recording no longer than a window, or one with nothing in it louder+/// than [`SILENCE`], starts at its beginning.+fn densest(levels: &[f32], window: usize) -> usize {+ if window == 0 || levels.len() <= window {+ return 0;+ }+ let mut sorted = levels.to_vec();+ sorted.sort_by(f32::total_cmp);+ let loud = sorted[(sorted.len() - 1) * 19 / 20];+ if loud < SILENCE {+ return 0;+ }+ let floor = loud * SPEECH_FLOOR;+ // Running sums over the frames so far, so each run is the difference+ // of two rather than a fresh count of its window.+ let mut spoken = vec![0_usize; levels.len() + 1];+ let mut energy = vec![0.0_f64; levels.len() + 1];+ for (at, level) in levels.iter().enumerate() {+ spoken[at + 1] = spoken[at] + usize::from(*level > floor);+ energy[at + 1] = energy[at] + f64::from(*level) * f64::from(*level);+ }+ let mut best = (0, 0, 0.0);+ for first in 0..=levels.len() - window {+ let count = spoken[first + window] - spoken[first];+ let power = energy[first + window] - energy[first];+ if count > best.1 || (count == best.1 && power > best.2) {+ best = (first, count, power);+ }+ }+ best.0+}+/// Deletes a saved voice's folder.pub fn remove(voice: &SavedVoice) -> Result<(), String> {match std::fs::remove_dir_all(&voice.dir) {@@ -245,18 +373,22 @@ mod tests {dir}+ /// `seconds` of a tone at `level`, as a recording's samples.+ fn tone(seconds: f64, level: f32) -> Vec<f32> {+ (0..(seconds * f64::from(SAMPLE_RATE)) as usize)+ .map(|n| (n as f32 * 440.0 * std::f32::consts::TAU / SAMPLE_RATE as f32).sin() * level)+ .collect()+ }+/// A voice is saved from a recording - its sample cut out and written- /// on its own, its picture beside it - listed newest first, and gone- /// with its folder when removed.+ /// on its own, its picture beside it, its description trimmed - listed+ /// newest first, and gone with its folder when removed.#[test]fn a_voice_is_kept_listed_and_removed() {let dir = scratch("keep");// Three seconds of a tone, as the recording.- let tone: Vec<f32> = (0..SAMPLE_RATE * 3)- .map(|n| (n as f32 * 440.0 * std::f32::consts::TAU / SAMPLE_RATE as f32).sin() * 0.5)- .collect();let source = dir.join("take.wav");- std::fs::write(&source, wav(&tone, SAMPLE_RATE)).unwrap();+ std::fs::write(&source, wav(&tone(3.0, 0.5), SAMPLE_RATE)).unwrap();let picture = dir.join("face.PNG");std::fs::write(&picture, b"png").unwrap();@@ -266,22 +398,25 @@ mod tests {&source,0.5,Some(&picture),+ Some(" Warm, unhurried "),).expect("saves");assert_eq!(voice.name, "Narrator");assert!(voice.sample().is_file());assert_eq!(voice.picture.as_deref(), Some("picture.png"));+ assert_eq!(voice.description.as_deref(), Some("Warm, unhurried"));let listed = list(&dir.join("voices"));assert_eq!(listed.len(), 1);assert_eq!(listed[0].id, voice.id);assert!(listed[0].picture_path().unwrap().is_file());+ assert_eq!(listed[0].description, voice.description);assert!(- save(&dir.join("voices"), " ", &source, 0.0, None).is_err(),+ save(&dir.join("voices"), " ", &source, 0.0, None, None).is_err(),"a name is needed");assert!(- save(&dir.join("voices"), "Late", &source, 2.5, None).is_err(),+ save(&dir.join("voices"), "Late", &source, 2.5, None, None).is_err(),"half a second is too little");remove(&listed[0]).unwrap();@@ -289,6 +424,75 @@ mod tests {let _ = std::fs::remove_dir_all(&dir);}+ /// A description is a line: a blank one is none, a long one is cut at+ /// its cap by characters, never inside one; and a voice saved before+ /// voices had descriptions reads as one without.+ #[test]+ fn a_voice_description_is_one_capped_line() {+ let dir = scratch("describe");+ let source = dir.join("take.wav");+ std::fs::write(&source, wav(&tone(2.0, 0.5), SAMPLE_RATE)).unwrap();+ let voices = dir.join("voices");+ let blank = save(&voices, "Blank", &source, 0.0, None, Some(" ")).unwrap();+ assert_eq!(blank.description, None);+ let long = "é".repeat(DESCRIPTION_CHARS + 20);+ let cut = save(&voices, "Long", &source, 0.0, None, Some(&long)).unwrap();+ assert_eq!(+ cut.description.as_deref().map(|line| line.chars().count()),+ Some(DESCRIPTION_CHARS)+ );+ let old: SavedVoice =+ serde_json::from_str(r#"{"id":"a","name":"Old","created":1}"#).unwrap();+ assert_eq!(old.description, None);+ let _ = std::fs::remove_dir_all(&dir);+ }++ /// The window taken is where the speaking is: past a silent opening,+ /// the louder of two runs as full of it, and never one loud click.+ #[test]+ fn the_densest_window_is_where_the_speaking_is() {+ let run = |parts: &[(usize, f32)]| -> Vec<f32> {+ parts+ .iter()+ .flat_map(|&(frames, level)| std::iter::repeat_n(level, frames))+ .collect()+ };+ // Silence, then speech: the window starts where the speech does,+ // the earliest of the windows wholly in it.+ assert_eq!(densest(&run(&[(50, 0.0), (30, 0.5)]), 20), 50);+ // Two runs of speech as long: the one spoken up in.+ assert_eq!(+ densest(&run(&[(20, 0.2), (30, 0.0), (20, 0.5), (10, 0.0)]), 20),+ 50+ );+ // A door slammed in the silence before quiet speech: the bar is+ // set by the loud end of the recording, not its loudest frame, so+ // the speech still counts and the bang does not win on energy.+ let mut levels = run(&[(60, 0.0), (40, 0.0625), (20, 0.0)]);+ levels[5] = 1.0;+ assert_eq!(densest(&levels, 20), 60);+ // No longer than a window, or silent: from the start.+ assert_eq!(densest(&run(&[(10, 0.0), (10, 0.5)]), 20), 0);+ assert_eq!(densest(&run(&[(100, 0.0)]), 20), 0);+ }++ /// A recording that opens on silence is listened to from where its+ /// voice begins, and its length is the file's.+ #[test]+ fn a_recording_is_listened_to_for_its_voice() {+ let dir = scratch("listen");+ let mut samples = vec![0.0; SAMPLE_RATE as usize * 20];+ samples.extend(tone(12.5, 0.3));+ samples.extend(vec![0.0; SAMPLE_RATE as usize * 10]);+ let source = dir.join("interview.wav");+ std::fs::write(&source, wav(&samples, SAMPLE_RATE)).unwrap();+ let heard = listen(&source).expect("listens");+ assert!((heard.duration - 42.5).abs() < 0.05, "{heard:?}");+ assert!((19.98..=20.52).contains(&heard.start), "{heard:?}");+ assert!(listen(&dir.join("missing.wav")).is_err());+ let _ = std::fs::remove_dir_all(&dir);+ }+#[test]fn a_wav_header_says_what_follows() {let bytes = wav(&[0.0, 1.0, -1.0], 24_000);diff --git a/src/crates/concat/src/panes/sounds.rs b/src/crates/concat/src/panes/sounds.rsindex 0fdf8e10..b66afed3 100644--- a/src/crates/concat/src/panes/sounds.rs+++ b/src/crates/concat/src/panes/sounds.rs@@ -455,7 +455,12 @@ impl Studio {.map(|voice| VoiceCardData {id: format!("saved-{}", voice.id).into(),name: voice.name.as_str().into(),- detail: t("speech.savedVoice").into(),+ // The line the user wrote about it, or what it is.+ detail: voice+ .description+ .clone()+ .unwrap_or_else(|| t("speech.savedVoice"))+ .into(),avatar: self.speech.saved_avatar(voice),removable: true,})diff --git a/src/crates/concat/src/panes/speech.rs b/src/crates/concat/src/panes/speech.rsindex 4e60f5e8..bc0c4c9e 100644--- a/src/crates/concat/src/panes/speech.rs+++ b/src/crates/concat/src/panes/speech.rs@@ -1019,6 +1019,7 @@ impl SpeechPane {std::path::Path::new(&sample.path),start,picture.as_deref(),+ None,)},|studio, _, _, result| {