The truly free, open-source, local-first, cross-platform CapCut replacement. Scriptable through a JSON-RPC API & MCP.
Rust ffmpegfree-video-editormodel-context-protocolnon-linear-editoroffline-firstopen-source-video-editorself-hostedvideo-editingvideo-editing-automationvideo-editing-softwarevideo-editing-toolsvideo-editorvoice-cloningyoutube-automationyoutube-shorts
fix(speech): Chatterbox reads the text the way its export expects
Jareerc7d3a9371c11
diff --git a/src/Cargo.lock b/src/Cargo.lockindex e6c2ea34..3d572c66 100644--- a/src/Cargo.lock+++ b/src/Cargo.lock@@ -1426,6 +1426,7 @@ dependencies = ["bzip2 0.6.1","concat-host","concat-media",+ "log","ort","serde","serde_json",diff --git a/src/crates/concat-speech/Cargo.toml b/src/crates/concat-speech/Cargo.tomlindex 05379172..abf9b863 100644--- a/src/crates/concat-speech/Cargo.toml+++ b/src/crates/concat-speech/Cargo.toml@@ -29,6 +29,11 @@ bzip2 = "0.6"ort = { workspace = true, optional = true }tokenizers = { version = "0.23", optional = true, default-features = false, features = ["fancy-regex"] }+# What a run says about itself: which model loaded, how long a read took,+# where a loop ran out. Through the facade, so the window's log file and a+# terminal both get it; see concat-host's logs.rs.+log = { version = "0.4", features = ["std"] }+# Transcription: whisper.cpp in-process. Built from source by cmake at build# time; on macOS with Metal so the encoder runs on the GPU.whisper-rs = "0.16"diff --git a/src/crates/concat-speech/src/chatterbox.rs b/src/crates/concat-speech/src/chatterbox.rsindex d28ffbd1..ed7ef3a7 100644--- a/src/crates/concat-speech/src/chatterbox.rs+++ b/src/crates/concat-speech/src/chatterbox.rs@@ -18,10 +18,23 @@//! file lands in a `.part` and is renamed whole, so a torn download is//! never mistaken for a model.//!-//! Every voice is a recording: there are no speakers built in. The sheet-//! offers the selected clip's own. Resemble's Python stamps its output-//! with a watermark; the ONNX path carries no such step, and this one-//! adds none.+//! Every voice is a recording: there are no speakers built in. Resemble's+//! Python stamps its output with a watermark; the ONNX path carries no+//! such step, and this one adds none.+//!+//! The one thing the export is particular about, and the thing that turns+//! a read into moaning when it is got wrong: the embedding graph splits+//! whatever ids it is given into two runs. Everything but the last two goes+//! through the *text* table; the last two go through the *speech* table,+//! with `<|endoftext|>` (50256) swapped for the start-of-speech token+//! (6561) on the way. The tokenizer's post-processor appends two+//! `<|endoftext|>` to every text, so the whole of its output - terminators+//! included - is embedded in one call, and the graph itself turns the two+//! terminators into two start-of-speech marks. A single written token,+//! embedded on its own, has no first run and is speech. Nothing else may be+//! appended: a start token added by hand is a third mark the model never+//! saw in training, and a text encoded without its terminators sends its+//! last word through the speech table as noise.use std::path::{Path, PathBuf};use std::sync::Mutex;@@ -39,8 +52,13 @@ pub const BUNDLE_ID: &str = "chatterbox-turbo-q8";pub const SAMPLE_RATE: u32 = 24_000;/// Speech tokens a second: how long a token is in sound.const TOKENS_PER_SECOND: f32 = 25.0;-/// The token the language model starts writing after.+/// The token the language model starts writing after. Never fed by hand:+/// the embedding graph makes it from the text's `<|endoftext|>` pair.const START_SPEECH_TOKEN: i64 = 6561;+/// GPT-2's `<|endoftext|>`, which the tokenizer's post-processor appends+/// twice to every text and the embedding graph reads as the start of+/// speech.+const TEXT_END_TOKEN: i64 = 50256;/// The token that means it has finished.const STOP_SPEECH_TOKEN: i64 = 6562;/// A token of silence, three of which end every utterance.@@ -298,16 +316,23 @@ impl Engine {if !installed(dir) {return Err("the Chatterbox bundle is incomplete - re-download it".to_owned());}+ let started = std::time::Instant::now();+ log::info!("chatterbox: loading the bundle in {}", dir.display());let tokenizer = tokenizers::Tokenizer::from_file(dir.join("tokenizer.json")).map_err(|error| format!("chatterbox tokenizer: {error}"))?;let onnx = dir.join("onnx");- Ok(Engine {+ let engine = Engine {tokenizer,speech_encoder: Mutex::new(session(&onnx.join("speech_encoder_quantized.onnx"))?),embed_tokens: Mutex::new(session(&onnx.join("embed_tokens_quantized.onnx"))?),language_model: Mutex::new(session(&onnx.join("language_model_quantized.onnx"))?),decoder: Mutex::new(session(&onnx.join("conditional_decoder_quantized.onnx"))?),- })+ };+ log::info!(+ "chatterbox: four networks loaded in {:.1}s",+ started.elapsed().as_secs_f32()+ );+ Ok(engine)}/// Reads `text` in the voice of `reference`, mono samples at@@ -321,11 +346,18 @@ impl Engine {cancel: &AtomicBool,progress: &mut dyn FnMut(f32),) -> Result<Vec<f32>, String> {+ let started = std::time::Instant::now();+ log::info!(+ "chatterbox: reading {} characters in the voice of a {:.1}s recording",+ text.chars().count(),+ reference.len() as f32 / SAMPLE_RATE as f32+ );let voice = self.hear(reference)?;let chunks = chunks(text);if chunks.is_empty() {return Err("nothing to say: the text is empty".to_owned());}+ log::info!("chatterbox: {} chunk(s) to read", chunks.len());let expected: usize = chunks.iter().map(|chunk| expected_tokens(chunk)).sum();let mut written = 0usize;let mut samples = Vec::new();@@ -336,13 +368,41 @@ impl Engine {(SAMPLE_RATE as f32 * GAP_SECONDS) as usize,));}+ log::debug!(+ "chatterbox: chunk {}/{}: {:?}",+ index + 1,+ chunks.len(),+ chunk.chars().take(80).collect::<String>()+ );+ let writing = std::time::Instant::now();let tokens = self.write(chunk, &voice, cancel, &mut |count| {progress(((written + count) as f32 / expected.max(1) as f32).min(0.95));})?;+ log::info!(+ "chatterbox: chunk {}/{} wrote {} speech tokens ({:.1}s of speech) in {:.1}s",+ index + 1,+ chunks.len(),+ tokens.len(),+ tokens.len() as f32 / TOKENS_PER_SECOND,+ writing.elapsed().as_secs_f32()+ );written += tokens.len();+ let decoding = std::time::Instant::now();let wave = self.decode(&voice, &tokens)?;+ log::info!(+ "chatterbox: chunk {}/{} decoded to {:.2}s of audio in {:.1}s",+ index + 1,+ chunks.len(),+ wave.len() as f32 / SAMPLE_RATE as f32,+ decoding.elapsed().as_secs_f32()+ );samples.extend(wave);}+ log::info!(+ "chatterbox: {:.2}s of audio in {:.1}s",+ samples.len() as f32 / SAMPLE_RATE as f32,+ started.elapsed().as_secs_f32()+ );progress(1.0);Ok(samples)}@@ -360,12 +420,20 @@ impl Engine {let outputs = encoder.run(vec![feed("audio_values", audio)]).map_err(|error| format!("chatterbox speech encoder: {error}"))?;- Ok(Voice {+ let voice = Voice {features: take_f32(&outputs, "audio_features")?,prompt: take_i64(&outputs, "audio_tokens")?,embedding: take_f32(&outputs, "speaker_embeddings")?,spectrum: take_f32(&outputs, "speaker_features")?,- })+ };+ log::debug!(+ "chatterbox: heard the voice: features {:?}, {} prompt tokens, embedding {:?}, spectrum {:?}",+ voice.features.0,+ voice.prompt.len(),+ voice.embedding.0,+ voice.spectrum.0+ );+ Ok(voice)}/// The language model over one chunk: the speech tokens it writes,@@ -377,14 +445,23 @@ impl Engine {cancel: &AtomicBool,progress: &mut dyn FnMut(usize),) -> Result<Vec<i64>, String> {+ // With the post-processor's two `<|endoftext|>` on the end: they are+ // what the embedding graph turns into the start of speech, and a+ // text without them is read wrong from its last word on - see the+ // module's note.let encoding = self.tokenizer- .encode(chunk, false)+ .encode(chunk, true).map_err(|error| format!("chatterbox tokenizer: {error}"))?;let text_ids: Vec<i64> = encoding.get_ids().iter().map(|&id| i64::from(id)).collect();- if text_ids.is_empty() {+ if text_ids.iter().all(|&id| id == TEXT_END_TOKEN) {return Ok(Vec::new());}+ log::debug!(+ "chatterbox: {} text tokens, ending {:?}",+ text_ids.len(),+ &text_ids[text_ids.len().saturating_sub(3)..]+ );let mut embed = self.embed_tokens.lock()@@ -394,8 +471,10 @@ impl Engine {.lock().map_err(|_| "chatterbox: model poisoned")?;- // The first step reads the voice, the text and the start token- // together; every later step reads one token with the cache.+ // The first step reads the voice and the whole text together - the+ // text's own terminators becoming the start of speech inside the+ // embedding graph, so nothing is appended here; every later step+ // reads the one token just written, with the cache.let embed_ids =|embed: &mut Session, ids: &[i64]| -> Result<(Vec<usize>, Vec<f32>), String> {let outputs = embed@@ -407,11 +486,9 @@ impl Engine {take_f32(&outputs, "inputs_embeds")};let (_, text_embeds) = embed_ids(&mut embed, &text_ids)?;- let (_, start_embed) = embed_ids(&mut embed, &[START_SPEECH_TOKEN])?;let width = voice.features.0[2];let mut embeds = voice.features.1.clone();embeds.extend_from_slice(&text_embeds);- embeds.extend_from_slice(&start_embed);let mut seq_len = embeds.len() / width;let mut total_len = seq_len;let mut position = 0usize;@@ -466,8 +543,15 @@ impl Engine {generated.push(next);progress(step + 1);if next == STOP_SPEECH_TOKEN {+ log::debug!("chatterbox: stop token after {} speech tokens", step);break;}+ if step + 1 == MAX_NEW_TOKENS {+ log::warn!(+ "chatterbox: no stop token in {MAX_NEW_TOKENS} tokens - the read ran out; \+ the recording or the text may be one the model cannot follow"+ );+ }position += seq_len;let (_, next_embed) = embed_ids(&mut embed, &[next])?;embeds = next_embed;@@ -625,8 +709,7 @@ pub fn bundle_dir(models: &Path) -> PathBuf {mod tests {use super::*;- const UPSTREAM: &str =- "https://huggingface.co/ResembleAI/chatterbox-turbo-ONNX/resolve/main/";+ const UPSTREAM: &str = "https://huggingface.co/ResembleAI/chatterbox-turbo-ONNX/resolve/main/";#[test]fn the_table_names_nine_distinct_files_with_the_bundle_in_front() {diff --git a/src/crates/concat-speech/src/tts.rs b/src/crates/concat-speech/src/tts.rsindex 07482430..02dd0480 100644--- a/src/crates/concat-speech/src/tts.rs+++ b/src/crates/concat-speech/src/tts.rs@@ -616,12 +616,31 @@ impl Speech {) -> Result<SpeakResult, String> {let job = self.gate.begin("speech generation")?;let cancel = job.cancel_handle();+ let started = std::time::Instant::now();let text = request.text.trim();if text.is_empty() {return Err("nothing to say: the text is empty".to_owned());}let family = family_of(&request.model_id);+ log::info!(+ "tts: {} reading {} characters with voice {} ({}), speed {:.2}, pauses {:?}, steps {:?}{}",+ request.model_id,+ text.chars().count(),+ request.voice,+ voice_name(request.voice).unwrap_or("?"),+ request.speed,+ request.pauses,+ request.steps,+ request+ .reference+ .as_ref()+ .map(|reference| format!(+ ", voice from {} at {:.1}s",+ reference.path, reference.start+ ))+ .unwrap_or_default()+ );match voice_family(request.voice) {Some(owner) if owner == family => {}Some(_) => {@@ -672,6 +691,8 @@ impl Speech {if engine.as_ref().map(|cached| cached.model_id.as_str()) != Some(request.model_id.as_str()){// Load before overwriting: a failed load keeps the old engine.+ let loading = std::time::Instant::now();+ log::info!("tts: loading {} from {}", request.model_id, dir.display());let tts = match family {Family::Kokoro => Engine::Sherpa(load_kokoro(&dir)?),Family::Pocket => Engine::Sherpa(load_pocket(&dir)?),@@ -684,6 +705,11 @@ impl Speech {return Err("Chatterbox is not part of this build".to_owned());}};+ log::info!(+ "tts: {} loaded in {:.1}s",+ request.model_id,+ loading.elapsed().as_secs_f32()+ );*engine = Some(CachedEngine {model_id: request.model_id.clone(),tts,@@ -724,9 +750,15 @@ impl Speech {let rate = crate::chatterbox::SAMPLE_RATE;std::fs::write(&file, crate::chatterbox::wav_bytes(&spoken, rate)).map_err(|error| format!("could not write {}: {error}", file.display()))?;+ let duration = spoken.len() as f64 / f64::from(rate);+ log::info!(+ "tts: wrote {} ({duration:.2}s) in {:.1}s",+ file.display(),+ started.elapsed().as_secs_f32()+ );return Ok(SpeakResult {path: file.to_string_lossy().into_owned(),- duration: spoken.len() as f64 / f64::from(rate),+ duration,});}Engine::Sherpa(tts) => tts,@@ -800,6 +832,12 @@ impl Speech {}let duration = audio.samples().len() as f64 / f64::from(audio.sample_rate().max(1));+ log::info!(+ "tts: wrote {} ({duration:.2}s at {} Hz) in {:.1}s",+ file.display(),+ audio.sample_rate(),+ started.elapsed().as_secs_f32()+ );Ok(SpeakResult {path: file.to_string_lossy().into_owned(),duration,@@ -882,6 +920,17 @@ fn reference_samples(dir: &Path, request: &SpeakRequest) -> Result<Vec<f32>, Strpath.display()));}+ let peak = samples+ .iter()+ .fold(0.0f32, |peak, sample| peak.max(sample.abs()));+ log::info!(+ "tts: voice from {} at {start:.1}s: {:.1}s of sound, peak {peak:.2}",+ path.display(),+ samples.len() as f32 / REFERENCE_RATE as f32+ );+ if peak < 0.01 {+ log::warn!("tts: the voice recording is near silent - the read will be in nobody's voice");+ }Ok(samples)}