The truly free, open-source, local-first, cross-platform CapCut replacement. Scriptable through a JSON-RPC API & MCP.
Rust ffmpegfree-video-editormodel-context-protocolnon-linear-editoroffline-firstopen-source-video-editorself-hostedvideo-editingvideo-editing-automationvideo-editing-softwarevideo-editing-toolsvideo-editorvoice-cloningyoutube-automationyoutube-shorts
fix(speech): a voice the GPU cannot read is read again on the CPU (#291)
Jareere5d8197c400b
diff --git a/src/crates/concat-speech/src/chatterbox.rs b/src/crates/concat-speech/src/chatterbox.rsindex f66c7b5b..35cdd6a1 100644--- a/src/crates/concat-speech/src/chatterbox.rs+++ b/src/crates/concat-speech/src/chatterbox.rs@@ -44,6 +44,7 @@ use ort::session::Session;use ort::value::Tensor;use crate::DownloadProgress;+use crate::tts::ReadError;/// The bundle id, as the settings panel and a request name it.pub const BUNDLE_ID: &str = "chatterbox-turbo-q8";@@ -406,7 +407,7 @@ impl Engine {reference: &[f32],cancel: &AtomicBool,progress: &mut dyn FnMut(f32),- ) -> Result<Vec<f32>, String> {+ ) -> Result<Vec<f32>, ReadError> {let started = std::time::Instant::now();log::info!("chatterbox: reading {} characters in the voice of a {:.1}s recording",@@ -420,7 +421,7 @@ impl Engine {let text = punc_norm(text);let chunks = chunks(&text);if chunks.is_empty() {- return Err("nothing to say: the text is empty".to_owned());+ return Err("nothing to say: the text is empty".into());}log::info!("chatterbox: {} chunk(s) to read", chunks.len());let expected: usize = chunks.iter().map(|chunk| expected_tokens(chunk)).sum();@@ -474,14 +475,15 @@ impl Engine {/// The speech encoder over the recording, brought to the loudness the/// model was trained at.- fn hear(&self, reference: &[f32]) -> Result<Voice, String> {+ fn hear(&self, reference: &[f32]) -> Result<Voice, ReadError> {let seconds = reference.len() as f32 / SAMPLE_RATE as f32;if seconds < MIN_REFERENCE_SECONDS {return Err(format!("the recording is {seconds:.1}s of sound and Chatterbox needs at least \{MIN_REFERENCE_SECONDS:.0}s of clear speech to take a voice from - pick a longer \sample, or start it earlier"- ));+ )+ .into());}let (reference, gain_db) = norm_loudness(reference);log::info!("chatterbox: the recording brought to {TARGET_LUFS} LUFS ({gain_db:+.1} dB)");@@ -492,7 +494,7 @@ impl Engine {let audio = tensor_f32(vec![1, reference.len()], reference)?;let outputs = encoder.run(vec![feed("audio_values", audio)])- .map_err(|error| format!("chatterbox speech encoder: {error}"))?;+ .map_err(|error| ReadError::Runtime(format!("chatterbox speech encoder: {error}")))?;let voice = Voice {features: take_f32(&outputs, "audio_features")?,prompt: take_i64(&outputs, "audio_tokens")?,@@ -517,7 +519,7 @@ impl Engine {voice: &Voice,cancel: &AtomicBool,progress: &mut dyn FnMut(usize),- ) -> Result<Vec<i64>, String> {+ ) -> Result<Vec<i64>, ReadError> {// With the post-processor's two `<|endoftext|>` on the end: they are// what the embedding graph turns into the start of speech, and a// text without them is read wrong from its last word on - see the@@ -548,16 +550,17 @@ impl Engine {// text's own terminators becoming the start of speech inside the// embedding graph, so nothing is appended here; every later step// reads the one token just written, with the cache.- let embed_ids =- |embed: &mut Session, ids: &[i64]| -> Result<(Vec<usize>, Vec<f32>), String> {- let outputs = embed- .run(vec![feed(- "input_ids",- tensor_i64(vec![1, ids.len()], ids.to_vec())?,- )])- .map_err(|error| format!("chatterbox embedding: {error}"))?;- take_f32(&outputs, "inputs_embeds")- };+ let embed_ids = |embed: &mut Session,+ ids: &[i64]|+ -> Result<(Vec<usize>, Vec<f32>), ReadError> {+ let outputs = embed+ .run(vec![feed(+ "input_ids",+ tensor_i64(vec![1, ids.len()], ids.to_vec())?,+ )])+ .map_err(|error| ReadError::Runtime(format!("chatterbox embedding: {error}")))?;+ Ok(take_f32(&outputs, "inputs_embeds")?)+ };let (_, text_embeds) = embed_ids(&mut embed, &text_ids)?;let width = voice.features.0[2];let mut embeds = voice.features.1.clone();@@ -577,7 +580,7 @@ impl Engine {let limit = MAX_NEW_TOKENS.min(expected_tokens(chunk) * RUNAWAY + 64);for step in 0..limit {if cancel.load(Ordering::Relaxed) {- return Err("speech generation cancelled".to_owned());+ return Err("speech generation cancelled".into());}let total_len = prompt_len + step;let mut inputs: Vec<Fed> = Vec::with_capacity(3 + LAYERS * 2);@@ -605,9 +608,9 @@ impl Engine {inputs.push(feed(format!("past_key_values.{layer}.{kind}"), value));}}- let mut outputs = model- .run(inputs)- .map_err(|error| format!("chatterbox language model: {error}"))?;+ let mut outputs = model.run(inputs).map_err(|error| {+ ReadError::Runtime(format!("chatterbox language model: {error}"))+ })?;let (logits_shape, logits) = take_f32(&outputs, "logits")?;let vocab = logits_shape[2];let last = &logits[(logits_shape[1] - 1) * vocab..];@@ -653,7 +656,7 @@ impl Engine {/// The decoder over the recording's tokens, the written ones, and a/// beat of silence: sound.- fn decode(&self, voice: &Voice, tokens: &[i64]) -> Result<Vec<f32>, String> {+ fn decode(&self, voice: &Voice, tokens: &[i64]) -> Result<Vec<f32>, ReadError> {let all = with_prompt(&voice.prompt, tokens);let mut decoder = self.decoder@@ -671,7 +674,7 @@ impl Engine {tensor_f32(voice.spectrum.0.clone(), voice.spectrum.1.clone())?,),])- .map_err(|error| format!("chatterbox decoder: {error}"))?;+ .map_err(|error| ReadError::Runtime(format!("chatterbox decoder: {error}")))?;let (_, wave) = take_f32(&outputs, "waveform")?;Ok(wave)}diff --git a/src/crates/concat-speech/src/tts.rs b/src/crates/concat-speech/src/tts.rsindex cb916109..e64c90c6 100644--- a/src/crates/concat-speech/src/tts.rs+++ b/src/crates/concat-speech/src/tts.rs@@ -416,6 +416,96 @@ enum Engine {struct CachedEngine {model_id: String,tts: Engine,+ /// Whether it was loaded for the accelerator rather than the CPU.+ accelerated: bool,+}++/// Why a read failed, as far as the accelerator is concerned: the runtime+/// failed running one of the networks, which the accelerator may be to+/// blame for and the CPU may well run - DirectML refuses some of+/// Chatterbox's quantised attention (#291) - or something else, which the+/// CPU would stop at too: the text, the recording, a cancel.+#[derive(Clone, Debug, PartialEq, Eq)]+pub enum ReadError {+ /// The runtime failed running a network.+ Runtime(String),+ /// Anything else.+ Other(String),+}++impl From<String> for ReadError {+ fn from(message: String) -> Self {+ ReadError::Other(message)+ }+}++impl From<&str> for ReadError {+ fn from(message: &str) -> Self {+ ReadError::Other(message.to_owned())+ }+}++impl std::fmt::Display for ReadError {+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {+ match self {+ ReadError::Runtime(message) | ReadError::Other(message) => f.write_str(message),+ }+ }+}++/// The accelerator the runtime is asked for here, by the name its log+/// lines use.+#[cfg_attr(not(feature = "chatterbox"), allow(dead_code))]+fn accelerator() -> &'static str {+ if cfg!(target_os = "macos") {+ "CoreML"+ } else if cfg!(target_os = "windows") {+ "DirectML"+ } else {+ "the accelerator"+ }+}++/// `read` on `engine`, and when an engine loaded for the accelerator fails+/// it in the runtime, `read` once more on the engine `on_cpu` loads for the+/// CPU. A model that loads for the accelerator can still be refused there a+/// network at a time, mid-read, and loading alone never found that out+/// (#291); slower is better than no voice. Any other failure is handed+/// back as it is, and nothing is loaded.+///+/// The CPU engine comes back beside the result, whether or not the second+/// read went through, for the caller to keep in place of the one that+/// failed: an accelerator that refused a network once will refuse it again.+#[cfg_attr(not(feature = "chatterbox"), allow(dead_code))]+fn read_or_on_cpu<E, T>(+ engine: &E,+ accelerated: bool,+ model: &str,+ mut read: impl FnMut(&E) -> Result<T, ReadError>,+ on_cpu: impl FnOnce() -> Result<E, String>,+) -> (Result<T, String>, Option<E>) {+ match read(engine) {+ Ok(done) => (Ok(done), None),+ Err(ReadError::Runtime(error)) if accelerated => {+ log::warn!(+ "tts: {model} failed on {} ({error}); loading it for the CPU and reading again",+ accelerator()+ );+ match on_cpu() {+ Ok(cpu) => {+ let again = read(&cpu).map_err(|error| error.to_string());+ (again, Some(cpu))+ }+ Err(load) => (+ Err(format!(+ "{error}; and it would not load for the CPU either: {load}"+ )),+ None,+ ),+ }+ }+ Err(error) => (Err(error.to_string()), None),+ }}/// The one synthesis and the one download that can run at a time, plus the@@ -430,6 +520,10 @@ pub struct Speech {gate: Arc<SingleFlight>,downloads: Arc<SingleFlight>,engine: Mutex<Option<CachedEngine>>,+ /// The models the accelerator failed to run this session: loaded for+ /// the CPU from then on, rather than tried there and failed again each+ /// time one is loaded.+ on_cpu: Mutex<std::collections::HashSet<String>>,}impl Default for Speech {@@ -445,6 +539,7 @@ impl Speech {gate: Arc::new(SingleFlight::new()),downloads: Arc::new(SingleFlight::new()),engine: Mutex::new(None),+ on_cpu: Mutex::new(std::collections::HashSet::new()),}}@@ -721,14 +816,20 @@ impl Speech {// The accelerator where there is one, and the CPU when a model// will not load for it: slower is better than not at all, and// nobody should have to find a setting to get the voice back.- let tts = match load(crate::accelerated()) {- Ok(tts) => tts,- Err(error) if crate::accelerated() => {+ // A model the accelerator failed to run this session goes+ // straight to the CPU.+ let refused = self+ .on_cpu+ .lock()+ .is_ok_and(|models| models.contains(&request.model_id));+ let (tts, accelerated) = match load(crate::accelerated() && !refused) {+ Ok(tts) => (tts, crate::accelerated() && !refused),+ Err(error) if crate::accelerated() && !refused => {log::warn!("tts: {} would not load on the accelerator ({error}); loading it for the CPU",request.model_id);- load(false)?+ (load(false)?, false)}Err(error) => return Err(error),};@@ -740,9 +841,15 @@ impl Speech {*engine = Some(CachedEngine {model_id: request.model_id.clone(),tts,+ accelerated,});}- let tts = &engine.as_ref().expect("engine cached above").tts;+ let cached = engine.as_ref().expect("engine cached above");+ let (tts, accelerated) = (&cached.tts, cached.accelerated);+ // Sherpa's families are not read again on the CPU: their runtime+ // reports no failure to tell one by.+ #[cfg(not(feature = "chatterbox"))]+ let _ = accelerated;// Wall-clock millis plus process id: unique enough for files created// by one human clicking a button, and stable for the media bin name.@@ -768,9 +875,30 @@ impl Speech {Engine::Chatterbox(chatterbox) => {let samples = reference_samples(&recordings, request)?;let mut progress = progress;- let spoken = chatterbox.speak(text, &samples, &cancel, &mut |fraction| {- progress(fraction);- })?;+ let (spoken, on_cpu) = read_or_on_cpu(+ chatterbox.as_ref(),+ accelerated,+ &request.model_id,+ |chatterbox| {+ chatterbox.speak(text, &samples, &cancel, &mut |fraction| {+ progress(fraction);+ })+ },+ || crate::chatterbox::Engine::load(&dir, false),+ );+ // The CPU engine stays for the session, and the model is+ // loaded for the CPU from now on.+ if let Some(cpu) = on_cpu {+ if let Ok(mut models) = self.on_cpu.lock() {+ models.insert(request.model_id.clone());+ }+ *engine = Some(CachedEngine {+ model_id: request.model_id.clone(),+ tts: Engine::Chatterbox(Box::new(cpu)),+ accelerated: false,+ });+ }+ let spoken = spoken?;if spoken.is_empty() {return Err("the engine produced no audio for this text".to_owned());}@@ -1086,6 +1214,73 @@ fn load_kokoro(dir: &Path, accelerated: bool) -> Result<OfflineTts, String> {mod tests {use super::*;+ /// A read the accelerator's runtime fails is read again once on the+ /// CPU, and the CPU engine comes back to be kept: DirectML refused+ /// Chatterbox's attention mid-read, after it had loaded (#291).+ #[test]+ fn a_read_the_accelerator_fails_is_read_again_on_the_cpu() {+ let read = |engine: &&str| match *engine {+ "gpu" => Err(ReadError::Runtime(+ "MultiHeadAttention: E_INVALIDARG".to_owned(),+ )),+ _ => Ok(format!("read on the {engine}")),+ };+ let (result, kept) = read_or_on_cpu(&"gpu", true, "chatterbox", read, || Ok("cpu"));+ assert_eq!(result.as_deref(), Ok("read on the cpu"));+ assert_eq!(kept, Some("cpu"));++ // Read once on the CPU, however it goes.+ let mut reads = 0;+ let (result, kept) = read_or_on_cpu(+ &"gpu",+ true,+ "chatterbox",+ |_: &&str| {+ reads += 1;+ Err::<(), _>(ReadError::Runtime("still refused".to_owned()))+ },+ || Ok("cpu"),+ );+ assert_eq!(reads, 2);+ assert_eq!(result, Err("still refused".to_owned()));+ assert_eq!(kept, Some("cpu"), "the CPU engine is kept all the same");++ // A model that will not load for the CPU says both.+ let (result, kept) = read_or_on_cpu(&"gpu", true, "chatterbox", read, || {+ Err("no memory".to_owned())+ });+ let error = result.expect_err("nothing read");+ assert!(+ error.contains("E_INVALIDARG") && error.contains("no memory"),+ "{error}"+ );+ assert!(kept.is_none());+ }++ /// Nothing is loaded again for a failure the CPU would meet too, nor+ /// for an engine on the CPU already.+ #[test]+ fn only_the_accelerators_failures_are_read_again() {+ let never = || -> Result<&str, String> { panic!("loaded for the CPU") };+ let (result, kept) = read_or_on_cpu(+ &"gpu",+ true,+ "chatterbox",+ |_: &&str| Err::<(), _>(ReadError::from("speech generation cancelled")),+ never,+ );+ assert_eq!(result, Err("speech generation cancelled".to_owned()));+ assert!(kept.is_none());+ let (result, _) = read_or_on_cpu(+ &"cpu",+ false,+ "chatterbox",+ |_: &&str| Err::<(), _>(ReadError::Runtime("refused".to_owned())),+ never,+ );+ assert_eq!(result, Err("refused".to_owned()));+ }+#[test]fn a_voice_belongs_to_one_family_and_a_model_says_which_it_reads_with() {assert_eq!(family_of("kokoro-int8-multi-lang-v1_0"), Family::Kokoro);