Skip to content

The truly free, open-source, local-first, cross-platform CapCut replacement. Scriptable through a JSON-RPC API & MCP.

fix(speech): a voice the GPU cannot read is read again on the CPU (#291)

Jareere5d8197c400b
diff --git a/src/crates/concat-speech/src/chatterbox.rs b/src/crates/concat-speech/src/chatterbox.rs
index f66c7b5b..35cdd6a1 100644
--- a/src/crates/concat-speech/src/chatterbox.rs
+++ b/src/crates/concat-speech/src/chatterbox.rs
@@ -44,6 +44,7 @@ use ort::session::Session;
use ort::value::Tensor;
use crate::DownloadProgress;
+use crate::tts::ReadError;
/// The bundle id, as the settings panel and a request name it.
pub const BUNDLE_ID: &str = "chatterbox-turbo-q8";
@@ -406,7 +407,7 @@ impl Engine {
reference: &[f32],
cancel: &AtomicBool,
progress: &mut dyn FnMut(f32),
- ) -> Result<Vec<f32>, String> {
+ ) -> Result<Vec<f32>, ReadError> {
let started = std::time::Instant::now();
log::info!(
"chatterbox: reading {} characters in the voice of a {:.1}s recording",
@@ -420,7 +421,7 @@ impl Engine {
let text = punc_norm(text);
let chunks = chunks(&text);
if chunks.is_empty() {
- return Err("nothing to say: the text is empty".to_owned());
+ return Err("nothing to say: the text is empty".into());
}
log::info!("chatterbox: {} chunk(s) to read", chunks.len());
let expected: usize = chunks.iter().map(|chunk| expected_tokens(chunk)).sum();
@@ -474,14 +475,15 @@ impl Engine {
/// The speech encoder over the recording, brought to the loudness the
/// model was trained at.
- fn hear(&self, reference: &[f32]) -> Result<Voice, String> {
+ fn hear(&self, reference: &[f32]) -> Result<Voice, ReadError> {
let seconds = reference.len() as f32 / SAMPLE_RATE as f32;
if seconds < MIN_REFERENCE_SECONDS {
return Err(format!(
"the recording is {seconds:.1}s of sound and Chatterbox needs at least \
{MIN_REFERENCE_SECONDS:.0}s of clear speech to take a voice from - pick a longer \
sample, or start it earlier"
- ));
+ )
+ .into());
}
let (reference, gain_db) = norm_loudness(reference);
log::info!("chatterbox: the recording brought to {TARGET_LUFS} LUFS ({gain_db:+.1} dB)");
@@ -492,7 +494,7 @@ impl Engine {
let audio = tensor_f32(vec![1, reference.len()], reference)?;
let outputs = encoder
.run(vec![feed("audio_values", audio)])
- .map_err(|error| format!("chatterbox speech encoder: {error}"))?;
+ .map_err(|error| ReadError::Runtime(format!("chatterbox speech encoder: {error}")))?;
let voice = Voice {
features: take_f32(&outputs, "audio_features")?,
prompt: take_i64(&outputs, "audio_tokens")?,
@@ -517,7 +519,7 @@ impl Engine {
voice: &Voice,
cancel: &AtomicBool,
progress: &mut dyn FnMut(usize),
- ) -> Result<Vec<i64>, String> {
+ ) -> Result<Vec<i64>, ReadError> {
// With the post-processor's two `<|endoftext|>` on the end: they are
// what the embedding graph turns into the start of speech, and a
// text without them is read wrong from its last word on - see the
@@ -548,16 +550,17 @@ impl Engine {
// text's own terminators becoming the start of speech inside the
// embedding graph, so nothing is appended here; every later step
// reads the one token just written, with the cache.
- let embed_ids =
- |embed: &mut Session, ids: &[i64]| -> Result<(Vec<usize>, Vec<f32>), String> {
- let outputs = embed
- .run(vec![feed(
- "input_ids",
- tensor_i64(vec![1, ids.len()], ids.to_vec())?,
- )])
- .map_err(|error| format!("chatterbox embedding: {error}"))?;
- take_f32(&outputs, "inputs_embeds")
- };
+ let embed_ids = |embed: &mut Session,
+ ids: &[i64]|
+ -> Result<(Vec<usize>, Vec<f32>), ReadError> {
+ let outputs = embed
+ .run(vec![feed(
+ "input_ids",
+ tensor_i64(vec![1, ids.len()], ids.to_vec())?,
+ )])
+ .map_err(|error| ReadError::Runtime(format!("chatterbox embedding: {error}")))?;
+ Ok(take_f32(&outputs, "inputs_embeds")?)
+ };
let (_, text_embeds) = embed_ids(&mut embed, &text_ids)?;
let width = voice.features.0[2];
let mut embeds = voice.features.1.clone();
@@ -577,7 +580,7 @@ impl Engine {
let limit = MAX_NEW_TOKENS.min(expected_tokens(chunk) * RUNAWAY + 64);
for step in 0..limit {
if cancel.load(Ordering::Relaxed) {
- return Err("speech generation cancelled".to_owned());
+ return Err("speech generation cancelled".into());
}
let total_len = prompt_len + step;
let mut inputs: Vec<Fed> = Vec::with_capacity(3 + LAYERS * 2);
@@ -605,9 +608,9 @@ impl Engine {
inputs.push(feed(format!("past_key_values.{layer}.{kind}"), value));
}
}
- let mut outputs = model
- .run(inputs)
- .map_err(|error| format!("chatterbox language model: {error}"))?;
+ let mut outputs = model.run(inputs).map_err(|error| {
+ ReadError::Runtime(format!("chatterbox language model: {error}"))
+ })?;
let (logits_shape, logits) = take_f32(&outputs, "logits")?;
let vocab = logits_shape[2];
let last = &logits[(logits_shape[1] - 1) * vocab..];
@@ -653,7 +656,7 @@ impl Engine {
/// The decoder over the recording's tokens, the written ones, and a
/// beat of silence: sound.
- fn decode(&self, voice: &Voice, tokens: &[i64]) -> Result<Vec<f32>, String> {
+ fn decode(&self, voice: &Voice, tokens: &[i64]) -> Result<Vec<f32>, ReadError> {
let all = with_prompt(&voice.prompt, tokens);
let mut decoder = self
.decoder
@@ -671,7 +674,7 @@ impl Engine {
tensor_f32(voice.spectrum.0.clone(), voice.spectrum.1.clone())?,
),
])
- .map_err(|error| format!("chatterbox decoder: {error}"))?;
+ .map_err(|error| ReadError::Runtime(format!("chatterbox decoder: {error}")))?;
let (_, wave) = take_f32(&outputs, "waveform")?;
Ok(wave)
}
diff --git a/src/crates/concat-speech/src/tts.rs b/src/crates/concat-speech/src/tts.rs
index cb916109..e64c90c6 100644
--- a/src/crates/concat-speech/src/tts.rs
+++ b/src/crates/concat-speech/src/tts.rs
@@ -416,6 +416,96 @@ enum Engine {
struct CachedEngine {
model_id: String,
tts: Engine,
+ /// Whether it was loaded for the accelerator rather than the CPU.
+ accelerated: bool,
+}
+
+/// Why a read failed, as far as the accelerator is concerned: the runtime
+/// failed running one of the networks, which the accelerator may be to
+/// blame for and the CPU may well run - DirectML refuses some of
+/// Chatterbox's quantised attention (#291) - or something else, which the
+/// CPU would stop at too: the text, the recording, a cancel.
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub enum ReadError {
+ /// The runtime failed running a network.
+ Runtime(String),
+ /// Anything else.
+ Other(String),
+}
+
+impl From<String> for ReadError {
+ fn from(message: String) -> Self {
+ ReadError::Other(message)
+ }
+}
+
+impl From<&str> for ReadError {
+ fn from(message: &str) -> Self {
+ ReadError::Other(message.to_owned())
+ }
+}
+
+impl std::fmt::Display for ReadError {
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ match self {
+ ReadError::Runtime(message) | ReadError::Other(message) => f.write_str(message),
+ }
+ }
+}
+
+/// The accelerator the runtime is asked for here, by the name its log
+/// lines use.
+#[cfg_attr(not(feature = "chatterbox"), allow(dead_code))]
+fn accelerator() -> &'static str {
+ if cfg!(target_os = "macos") {
+ "CoreML"
+ } else if cfg!(target_os = "windows") {
+ "DirectML"
+ } else {
+ "the accelerator"
+ }
+}
+
+/// `read` on `engine`, and when an engine loaded for the accelerator fails
+/// it in the runtime, `read` once more on the engine `on_cpu` loads for the
+/// CPU. A model that loads for the accelerator can still be refused there a
+/// network at a time, mid-read, and loading alone never found that out
+/// (#291); slower is better than no voice. Any other failure is handed
+/// back as it is, and nothing is loaded.
+///
+/// The CPU engine comes back beside the result, whether or not the second
+/// read went through, for the caller to keep in place of the one that
+/// failed: an accelerator that refused a network once will refuse it again.
+#[cfg_attr(not(feature = "chatterbox"), allow(dead_code))]
+fn read_or_on_cpu<E, T>(
+ engine: &E,
+ accelerated: bool,
+ model: &str,
+ mut read: impl FnMut(&E) -> Result<T, ReadError>,
+ on_cpu: impl FnOnce() -> Result<E, String>,
+) -> (Result<T, String>, Option<E>) {
+ match read(engine) {
+ Ok(done) => (Ok(done), None),
+ Err(ReadError::Runtime(error)) if accelerated => {
+ log::warn!(
+ "tts: {model} failed on {} ({error}); loading it for the CPU and reading again",
+ accelerator()
+ );
+ match on_cpu() {
+ Ok(cpu) => {
+ let again = read(&cpu).map_err(|error| error.to_string());
+ (again, Some(cpu))
+ }
+ Err(load) => (
+ Err(format!(
+ "{error}; and it would not load for the CPU either: {load}"
+ )),
+ None,
+ ),
+ }
+ }
+ Err(error) => (Err(error.to_string()), None),
+ }
}
/// The one synthesis and the one download that can run at a time, plus the
@@ -430,6 +520,10 @@ pub struct Speech {
gate: Arc<SingleFlight>,
downloads: Arc<SingleFlight>,
engine: Mutex<Option<CachedEngine>>,
+ /// The models the accelerator failed to run this session: loaded for
+ /// the CPU from then on, rather than tried there and failed again each
+ /// time one is loaded.
+ on_cpu: Mutex<std::collections::HashSet<String>>,
}
impl Default for Speech {
@@ -445,6 +539,7 @@ impl Speech {
gate: Arc::new(SingleFlight::new()),
downloads: Arc::new(SingleFlight::new()),
engine: Mutex::new(None),
+ on_cpu: Mutex::new(std::collections::HashSet::new()),
}
}
@@ -721,14 +816,20 @@ impl Speech {
// The accelerator where there is one, and the CPU when a model
// will not load for it: slower is better than not at all, and
// nobody should have to find a setting to get the voice back.
- let tts = match load(crate::accelerated()) {
- Ok(tts) => tts,
- Err(error) if crate::accelerated() => {
+ // A model the accelerator failed to run this session goes
+ // straight to the CPU.
+ let refused = self
+ .on_cpu
+ .lock()
+ .is_ok_and(|models| models.contains(&request.model_id));
+ let (tts, accelerated) = match load(crate::accelerated() && !refused) {
+ Ok(tts) => (tts, crate::accelerated() && !refused),
+ Err(error) if crate::accelerated() && !refused => {
log::warn!(
"tts: {} would not load on the accelerator ({error}); loading it for the CPU",
request.model_id
);
- load(false)?
+ (load(false)?, false)
}
Err(error) => return Err(error),
};
@@ -740,9 +841,15 @@ impl Speech {
*engine = Some(CachedEngine {
model_id: request.model_id.clone(),
tts,
+ accelerated,
});
}
- let tts = &engine.as_ref().expect("engine cached above").tts;
+ let cached = engine.as_ref().expect("engine cached above");
+ let (tts, accelerated) = (&cached.tts, cached.accelerated);
+ // Sherpa's families are not read again on the CPU: their runtime
+ // reports no failure to tell one by.
+ #[cfg(not(feature = "chatterbox"))]
+ let _ = accelerated;
// Wall-clock millis plus process id: unique enough for files created
// by one human clicking a button, and stable for the media bin name.
@@ -768,9 +875,30 @@ impl Speech {
Engine::Chatterbox(chatterbox) => {
let samples = reference_samples(&recordings, request)?;
let mut progress = progress;
- let spoken = chatterbox.speak(text, &samples, &cancel, &mut |fraction| {
- progress(fraction);
- })?;
+ let (spoken, on_cpu) = read_or_on_cpu(
+ chatterbox.as_ref(),
+ accelerated,
+ &request.model_id,
+ |chatterbox| {
+ chatterbox.speak(text, &samples, &cancel, &mut |fraction| {
+ progress(fraction);
+ })
+ },
+ || crate::chatterbox::Engine::load(&dir, false),
+ );
+ // The CPU engine stays for the session, and the model is
+ // loaded for the CPU from now on.
+ if let Some(cpu) = on_cpu {
+ if let Ok(mut models) = self.on_cpu.lock() {
+ models.insert(request.model_id.clone());
+ }
+ *engine = Some(CachedEngine {
+ model_id: request.model_id.clone(),
+ tts: Engine::Chatterbox(Box::new(cpu)),
+ accelerated: false,
+ });
+ }
+ let spoken = spoken?;
if spoken.is_empty() {
return Err("the engine produced no audio for this text".to_owned());
}
@@ -1086,6 +1214,73 @@ fn load_kokoro(dir: &Path, accelerated: bool) -> Result<OfflineTts, String> {
mod tests {
use super::*;
+ /// A read the accelerator's runtime fails is read again once on the
+ /// CPU, and the CPU engine comes back to be kept: DirectML refused
+ /// Chatterbox's attention mid-read, after it had loaded (#291).
+ #[test]
+ fn a_read_the_accelerator_fails_is_read_again_on_the_cpu() {
+ let read = |engine: &&str| match *engine {
+ "gpu" => Err(ReadError::Runtime(
+ "MultiHeadAttention: E_INVALIDARG".to_owned(),
+ )),
+ _ => Ok(format!("read on the {engine}")),
+ };
+ let (result, kept) = read_or_on_cpu(&"gpu", true, "chatterbox", read, || Ok("cpu"));
+ assert_eq!(result.as_deref(), Ok("read on the cpu"));
+ assert_eq!(kept, Some("cpu"));
+
+ // Read once on the CPU, however it goes.
+ let mut reads = 0;
+ let (result, kept) = read_or_on_cpu(
+ &"gpu",
+ true,
+ "chatterbox",
+ |_: &&str| {
+ reads += 1;
+ Err::<(), _>(ReadError::Runtime("still refused".to_owned()))
+ },
+ || Ok("cpu"),
+ );
+ assert_eq!(reads, 2);
+ assert_eq!(result, Err("still refused".to_owned()));
+ assert_eq!(kept, Some("cpu"), "the CPU engine is kept all the same");
+
+ // A model that will not load for the CPU says both.
+ let (result, kept) = read_or_on_cpu(&"gpu", true, "chatterbox", read, || {
+ Err("no memory".to_owned())
+ });
+ let error = result.expect_err("nothing read");
+ assert!(
+ error.contains("E_INVALIDARG") && error.contains("no memory"),
+ "{error}"
+ );
+ assert!(kept.is_none());
+ }
+
+ /// Nothing is loaded again for a failure the CPU would meet too, nor
+ /// for an engine on the CPU already.
+ #[test]
+ fn only_the_accelerators_failures_are_read_again() {
+ let never = || -> Result<&str, String> { panic!("loaded for the CPU") };
+ let (result, kept) = read_or_on_cpu(
+ &"gpu",
+ true,
+ "chatterbox",
+ |_: &&str| Err::<(), _>(ReadError::from("speech generation cancelled")),
+ never,
+ );
+ assert_eq!(result, Err("speech generation cancelled".to_owned()));
+ assert!(kept.is_none());
+ let (result, _) = read_or_on_cpu(
+ &"cpu",
+ false,
+ "chatterbox",
+ |_: &&str| Err::<(), _>(ReadError::Runtime("refused".to_owned())),
+ never,
+ );
+ assert_eq!(result, Err("refused".to_owned()));
+ }
+
#[test]
fn a_voice_belongs_to_one_family_and_a_model_says_which_it_reads_with() {
assert_eq!(family_of("kokoro-int8-multi-lang-v1_0"), Family::Kokoro);