The truly free, open-source, local-first, cross-platform CapCut replacement. Scriptable through a JSON-RPC API & MCP.
Rust ffmpegfree-video-editormodel-context-protocolnon-linear-editoroffline-firstopen-source-video-editorself-hostedvideo-editingvideo-editing-automationvideo-editing-softwarevideo-editing-toolsvideo-editorvoice-cloningyoutube-automationyoutube-shorts
feat(speech): run the voices on the Mac's GPU, from Settings › Speech
Jareer8bc152b3761b
diff --git a/src/crates/concat-speech/Cargo.toml b/src/crates/concat-speech/Cargo.tomlindex abf9b863..330d6132 100644--- a/src/crates/concat-speech/Cargo.toml+++ b/src/crates/concat-speech/Cargo.toml@@ -68,6 +68,10 @@ sherpa-onnx = { version = "1.13", default-features = false, features = ["shared"[target.'cfg(target_os = "macos")'.dependencies]whisper-rs = { version = "0.16", features = ["metal"] }+# The CoreML provider, so Chatterbox can run on the Mac's own GPU and Neural+# Engine when Settings › Speech asks for it. Pulls pyke's CoreML build of the+# runtime in place of the plain one.+ort = { workspace = true, optional = true, features = ["coreml"] }[features]chatterbox = ["dep:ort", "dep:tokenizers"]diff --git a/src/crates/concat-speech/src/chatterbox.rs b/src/crates/concat-speech/src/chatterbox.rsindex 5e555ee3..c38658e0 100644--- a/src/crates/concat-speech/src/chatterbox.rs+++ b/src/crates/concat-speech/src/chatterbox.rs@@ -271,15 +271,37 @@ struct Voice {spectrum: (Vec<usize>, Vec<f32>),}-fn session(path: &Path) -> Result<Session, String> {+/// One network, opened for the CPU or, when asked and where the build has+/// it, for CoreML: the Mac's GPU and Neural Engine. CoreML takes the parts+/// of a graph it knows and hands the rest back to the CPU, and a graph it+/// cannot take at all still runs - slower, not broken - so asking for it+/// is safe; the log says which it got.+fn session(path: &Path, accelerated: bool) -> Result<Session, String> {let threads = std::thread::available_parallelism().map(|count| count.get()).unwrap_or(1).clamp(1, 8);- Session::builder()+ let mut builder = Session::builder().map_err(|error| format!("onnx runtime: {error}"))?.with_intra_threads(threads)- .map_err(|error| format!("onnx runtime: {error}"))?+ .map_err(|error| format!("onnx runtime: {error}"))?;+ #[cfg(target_os = "macos")]+ if accelerated {+ use ort::ep::coreml::{ComputeUnits, CoreML};+ builder = builder+ .with_execution_providers([CoreML::default()+ .with_compute_units(ComputeUnits::All)+ .with_subgraphs(true)+ .build()])+ .map_err(|error| format!("onnx runtime: coreml: {error}"))?;+ log::info!(+ "chatterbox: {} asked to run on CoreML",+ path.file_name().unwrap_or_default().to_string_lossy()+ );+ }+ #[cfg(not(target_os = "macos"))]+ let _ = accelerated;+ builder.commit_from_file(path).map_err(|error| format!("{}: {error}", path.display()))}@@ -333,22 +355,27 @@ fn take_i64(outputs: &ort::session::SessionOutputs<'_>, name: &str) -> Result<Ve}impl Engine {- /// Loads the bundle in `dir`.- pub fn load(dir: &Path) -> Result<Engine, String> {+ /// Loads the bundle in `dir`, for the accelerator or the CPU.+ pub fn load(dir: &Path, accelerated: bool) -> Result<Engine, String> {if !installed(dir) {return Err("the Chatterbox bundle is incomplete - re-download it".to_owned());}let started = std::time::Instant::now();- log::info!("chatterbox: loading the bundle in {}", dir.display());+ log::info!(+ "chatterbox: loading the bundle in {} for the {}",+ dir.display(),+ if accelerated { "accelerator" } else { "CPU" }+ );let tokenizer = tokenizers::Tokenizer::from_file(dir.join("tokenizer.json")).map_err(|error| format!("chatterbox tokenizer: {error}"))?;let onnx = dir.join("onnx");+ let open = |name: &str| session(&onnx.join(name), accelerated);let engine = Engine {tokenizer,- speech_encoder: Mutex::new(session(&onnx.join("speech_encoder_quantized.onnx"))?),- embed_tokens: Mutex::new(session(&onnx.join("embed_tokens_quantized.onnx"))?),- language_model: Mutex::new(session(&onnx.join("language_model_quantized.onnx"))?),- decoder: Mutex::new(session(&onnx.join("conditional_decoder_quantized.onnx"))?),+ speech_encoder: Mutex::new(open("speech_encoder_quantized.onnx")?),+ embed_tokens: Mutex::new(open("embed_tokens_quantized.onnx")?),+ language_model: Mutex::new(open("language_model_quantized.onnx")?),+ decoder: Mutex::new(open("conditional_decoder_quantized.onnx")?),};log::info!("chatterbox: four networks loaded in {:.1}s",diff --git a/src/crates/concat-speech/src/lib.rs b/src/crates/concat-speech/src/lib.rsindex ff84a031..56727632 100644--- a/src/crates/concat-speech/src/lib.rs+++ b/src/crates/concat-speech/src/lib.rs@@ -25,6 +25,31 @@ pub mod tts;pub use transcribe::Transcriber;pub use tts::Speech;+/// Whether the voices run on the machine's own accelerator where the build+/// has one, or on the CPU. Process-wide, the way `concat_media`'s hardware+/// decode preference is: Settings › Speech sets it, and an engine reads it+/// as it loads - one already loaded the other way is loaded again on the+/// next read. Off until asked, because the accelerator is a bet: CoreML+/// runs the parts of a network it knows and hands the rest back to the+/// CPU, and which parts those are is the model's business.+static ACCELERATED: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false);++/// Asks the engines to run on the accelerator, or not to.+pub fn set_accelerated(on: bool) {+ ACCELERATED.store(on, std::sync::atomic::Ordering::Relaxed);+}++/// Whether the engines are asked to run on the accelerator.+pub fn accelerated() -> bool {+ ACCELERATED.load(std::sync::atomic::Ordering::Relaxed)+}++/// Whether this build has an accelerator to offer at all: CoreML, on a+/// Mac. Elsewhere the switch is not shown, since it would do nothing.+pub const fn acceleration_offered() -> bool {+ cfg!(target_os = "macos")+}+/// Progress for one model download.#[derive(Clone, Debug, PartialEq, Eq)]pub struct DownloadProgress {diff --git a/src/crates/concat-speech/src/tts.rs b/src/crates/concat-speech/src/tts.rsindex e45e3c67..0b849442 100644--- a/src/crates/concat-speech/src/tts.rs+++ b/src/crates/concat-speech/src/tts.rs@@ -413,6 +413,9 @@ enum Engine {struct CachedEngine {model_id: String,+ /// Loaded for the accelerator, or for the CPU; a change of mind loads+ /// the model again.+ accelerated: bool,tts: Engine,}@@ -690,18 +693,26 @@ impl Speech {.engine.lock().map_err(|_| "speech state poisoned".to_owned())?;- if engine.as_ref().map(|cached| cached.model_id.as_str()) != Some(request.model_id.as_str())- {+ let accelerated = crate::accelerated();+ let stale = engine.as_ref().is_none_or(|cached| {+ cached.model_id != request.model_id || cached.accelerated != accelerated+ });+ if stale {// Load before overwriting: a failed load keeps the old engine.let loading = std::time::Instant::now();- log::info!("tts: loading {} from {}", request.model_id, dir.display());+ log::info!(+ "tts: loading {} from {} for the {}",+ request.model_id,+ dir.display(),+ if accelerated { "accelerator" } else { "CPU" }+ );let tts = match family {- Family::Kokoro => Engine::Sherpa(load_kokoro(&dir)?),- Family::Pocket => Engine::Sherpa(load_pocket(&dir)?),+ Family::Kokoro => Engine::Sherpa(load_kokoro(&dir, accelerated)?),+ Family::Pocket => Engine::Sherpa(load_pocket(&dir, accelerated)?),#[cfg(feature = "chatterbox")]- Family::Chatterbox => {- Engine::Chatterbox(Box::new(crate::chatterbox::Engine::load(&dir)?))- }+ Family::Chatterbox => Engine::Chatterbox(Box::new(+ crate::chatterbox::Engine::load(&dir, accelerated)?,+ )),#[cfg(not(feature = "chatterbox"))]Family::Chatterbox => {return Err("Chatterbox is not part of this build".to_owned());@@ -714,6 +725,7 @@ impl Speech {);*engine = Some(CachedEngine {model_id: request.model_id.clone(),+ accelerated,tts,});}@@ -959,7 +971,7 @@ fn onnx_starting(dir: &Path, prefix: &str) -> Option<String> {/// Builds the engine for a Pocket TTS bundle: the five networks and the/// two tables the sherpa-onnx example names.-fn load_pocket(dir: &Path) -> Result<OfflineTts, String> {+fn load_pocket(dir: &Path, accelerated: bool) -> Result<OfflineTts, String> {let part = |prefix: &str| {onnx_starting(dir, prefix).ok_or_else(|| format!("the model folder has no {prefix} network - re-download it"))@@ -985,6 +997,7 @@ fn load_pocket(dir: &Path) -> Result<OfflineTts, String> {voice_embedding_cache_capacity: 8,},num_threads: threads(),+ provider: Some(provider(accelerated).to_owned()),..Default::default()},..Default::default()@@ -993,6 +1006,18 @@ fn load_pocket(dir: &Path) -> Result<OfflineTts, String> {.ok_or_else(|| "the speech engine failed to load - try re-downloading the model".to_owned())}+/// The ONNX Runtime provider sherpa is asked for: CoreML on a Mac that+/// wants the accelerator, the CPU otherwise. sherpa falls back to the CPU+/// itself, with a line in the log, where its runtime was built without+/// the one asked for.+fn provider(accelerated: bool) -> &'static str {+ if accelerated && cfg!(target_os = "macos") {+ "coreml"+ } else {+ "cpu"+ }+}+/// The threads the engine may use: every core up to eight.fn threads() -> i32 {std::thread::available_parallelism()@@ -1004,7 +1029,7 @@ fn threads() -> i32 {////// The lexicon list mirrors the official sherpa-onnx invocation for this/// bundle: US English and Chinese, joined by commas, each only if present.-fn load_kokoro(dir: &Path) -> Result<OfflineTts, String> {+fn load_kokoro(dir: &Path, accelerated: bool) -> Result<OfflineTts, String> {let network = onnx_file(dir).ok_or_else(|| "the model folder has no .onnx network - re-download it".to_owned())?;let existing = |name: &str| {@@ -1034,6 +1059,7 @@ fn load_kokoro(dir: &Path) -> Result<OfflineTts, String> {..Default::default()},num_threads: threads(),+ provider: Some(provider(accelerated).to_owned()),..Default::default()},..Default::default()diff --git a/src/crates/concat/locales/en.json b/src/crates/concat/locales/en.jsonindex 29f341c6..d4bba998 100644--- a/src/crates/concat/locales/en.json+++ b/src/crates/concat/locales/en.json@@ -587,6 +587,7 @@"Off, the keys do nothing.": "Off, the keys do nothing.","Oil Paint": "Oil Paint","On, deleting a clip pulls what follows it on the same track into the gap, and trimming an edge moves the rest of the track with it. Off, a delete leaves a gap and a trim moves only the edge. Shift-Delete closes the gap either way.": "On, deleting a clip pulls what follows it on the same track into the gap, and trimming an edge moves the rest of the track with it. Off, a delete leaves a gap and a trim moves only the edge. Shift-Delete closes the gap either way.",+ "On, the voices run on this Mac's GPU and Neural Engine through CoreML wherever the model allows, and on the CPU where it does not. Off, everything runs on the CPU. Takes effect on the next read; a model already loaded is loaded again.": "On, the voices run on this Mac's GPU and Neural Engine through CoreML wherever the model allows, and on the CPU where it does not. Off, everything runs on the CPU. Takes effect on the next read; a model already loaded is loaded again.","On, video decodes on this machine's own hardware, which is what makes 4K play on a laptop. A file the hardware will not take decodes in software as before.": "On, video decodes on this machine's own hardware, which is what makes 4K play on a laptop. A file the hardware will not take decodes in software as before.","One repeat, fading.": "One repeat, fading.","One title per phrase spoken, cut to the words' own timing, on the first lane with room.": "One title per phrase spoken, cut to the words' own timing, on the first lane with room.",@@ -713,6 +714,7 @@"Rotation": "Rotation","Rotational vortex twist around a focal point.": "Rotational vortex twist around a focal point.","Row height": "Row height",+ "Run voices on the GPU": "Run voices on the GPU","Runs locally": "Runs locally","Sage": "Sage","Sample": "Sample",diff --git a/src/crates/concat/src/lib.rs b/src/crates/concat/src/lib.rsindex 93e9ac86..e7152985 100644--- a/src/crates/concat/src/lib.rs+++ b/src/crates/concat/src/lib.rs@@ -1195,6 +1195,9 @@ pub fn run() -> Result<(), slint::PlatformError> {app.on_settings_hardware_decode_changed(on_window!(|state, on: bool| {state.handle(Msg::Settings(SettingsMsg::HardwareDecodeChanged(on)));}));+ app.on_settings_speech_accelerated_changed(on_window!(|state, on: bool| {+ state.handle(Msg::Settings(SettingsMsg::SpeechAcceleratedChanged(on)));+ }));app.on_settings_download_source_changed(on_window!(|state, index: i32| {state.handle(Msg::Settings(SettingsMsg::DownloadSourceChanged(index)));}));diff --git a/src/crates/concat/src/panes/settings.rs b/src/crates/concat/src/panes/settings.rsindex b6700853..78b2e3c7 100644--- a/src/crates/concat/src/panes/settings.rs+++ b/src/crates/concat/src/panes/settings.rs@@ -43,6 +43,8 @@ pub enum SettingsMsg {/// The magnetic timeline switch; the tray's button is the same fact.MagneticChanged(bool),HardwareDecodeChanged(bool),+ /// The voices run on the accelerator.+ SpeechAcceleratedChanged(bool),DownloadSourceChanged(i32),DownloadBaseEdited(String),ServerEnabledChanged(bool),@@ -139,6 +141,8 @@ pub struct SettingsPane {pub custom_context_actions: bool,/// Video decodes on the platform's hardware.pub hardware_decode: bool,+ /// The voices run on the machine's accelerator.+ pub speech_accelerated: bool,/// Index into `SourcePreference::ALL`: where model downloads look first.pub download_source: usize,/// The base URL of a custom download source.@@ -169,6 +173,8 @@ impl SettingsPane {self.custom_context_actions = studio.prefs.custom_context_actions;self.hardware_decode = studio.prefs.hardware_decode_on();concat_media::set_hardware_decode(self.hardware_decode);+ self.speech_accelerated = studio.prefs.speech_accelerated;+ concat_speech::set_accelerated(self.speech_accelerated);self.download_source = Self::download_source(studio).0;self.download_base = studio.prefs.download_base.clone().unwrap_or_default();Self::apply_download_source(studio);@@ -222,6 +228,14 @@ impl SettingsPane {studio.prefs.magnetic = on;studio.prefs.save(&studio.host.dirs);}+ SettingsMsg::SpeechAcceleratedChanged(on) => {+ self.speech_accelerated = on;+ studio.prefs.speech_accelerated = on;+ studio.prefs.save(&studio.host.dirs);+ // An engine already loaded the other way is loaded again on+ // the next read; nothing running is disturbed.+ concat_speech::set_accelerated(on);+ }SettingsMsg::HardwareDecodeChanged(on) => {self.hardware_decode = on;studio.prefs.hardware_decode = Some(on);@@ -559,6 +573,8 @@ impl SettingsPane {custom_context_actions: self.custom_context_actions,magnetic: studio.prefs.magnetic,hardware_decode: self.hardware_decode,+ speech_accelerated: self.speech_accelerated,+ speech_acceleration_offered: concat_speech::acceleration_offered(),hardware_decode_offered: concat_media::HwDevice::platform_default().is_some_and(concat_media::HwDevice::linked),download_source: self.download_source as i32,diff --git a/src/crates/concat/src/prefs.rs b/src/crates/concat/src/prefs.rsindex 1c3b6347..42fe3c90 100644--- a/src/crates/concat/src/prefs.rs+++ b/src/crates/concat/src/prefs.rs@@ -61,6 +61,11 @@ pub struct Preferences {/// where the hardware path has been exercised (macOS and iOS) and off/// elsewhere; see `Preferences::hardware_decode_on`.pub hardware_decode: Option<bool>,+ /// The voices run on the machine's own accelerator - CoreML on a Mac -+ /// rather than the CPU. Off by default: what the accelerator takes of+ /// a network is the network's business, and the CPU is the answer that+ /// is always right.+ pub speech_accelerated: bool,/// Where model downloads look first: a `SourcePreference` by name./// Absent is automatic.pub download_source: Option<String>,diff --git a/src/crates/concat/ui/app.slint b/src/crates/concat/ui/app.slintindex bfb8f18a..8fe3d301 100644--- a/src/crates/concat/ui/app.slint+++ b/src/crates/concat/ui/app.slint@@ -209,6 +209,7 @@ export component App inherits Window {callback settings-custom-context-actions-changed(on: bool);callback settings-magnetic-changed(on: bool);callback settings-hardware-decode-changed(on: bool);+ callback settings-speech-accelerated-changed(on: bool);callback settings-download-source-changed(index: int);callback settings-download-base-edited(text: string);callback settings-server-enabled-changed(on: bool);@@ -548,6 +549,7 @@ export component App inherits Window {custom-context-actions-changed(on) => { root.settings-custom-context-actions-changed(on); }magnetic-changed(on) => { root.settings-magnetic-changed(on); }hardware-decode-changed(on) => { root.settings-hardware-decode-changed(on); }+ speech-accelerated-changed(on) => { root.settings-speech-accelerated-changed(on); }download-source-changed(index) => { root.settings-download-source-changed(index); }download-base-edited(text) => { root.settings-download-base-edited(text); }server-enabled-changed(on) => { root.settings-server-enabled-changed(on); }diff --git a/src/crates/concat/ui/dialogs/settings.slint b/src/crates/concat/ui/dialogs/settings.slintindex 7b2fe677..6a0735db 100644--- a/src/crates/concat/ui/dialogs/settings.slint+++ b/src/crates/concat/ui/dialogs/settings.slint@@ -83,6 +83,11 @@ export struct SettingsData {/// this build and machine have hardware to decode on; off, the/// switch is not shownhardware-decode-offered: bool,+ /// the voices run on the machine's accelerator rather than the CPU+ speech-accelerated: bool,+ /// this build has an accelerator to offer the voices; off, the switch+ /// is not shown+ speech-acceleration-offered: bool,/// where model downloads look first: automatic, Concat's mirror,/// hf-mirror.com, the original source, a custom URLdownload-source: int,@@ -450,6 +455,7 @@ export component SettingsDialog inherits Modal {callback custom-context-actions-changed(on: bool);callback magnetic-changed(on: bool);callback hardware-decode-changed(on: bool);+ callback speech-accelerated-changed(on: bool);callback download-source-changed(index: int);callback download-base-edited(text: string);callback server-enabled-changed(on: bool);@@ -892,16 +898,48 @@ export component SettingsDialog inherits Modal {}}- if root.data.tab == 3: DialogSection {- first: true;- title: I18n.t("Voices");+ if root.data.tab == 3: VerticalLayout {+ DialogSection {+ first: true;+ title: I18n.t("Voices");++ for model in root.voices: ModelRow {+ model: model;+ activated => { root.model-activated(model.id); }+ download => { root.model-download(model.id); }+ cancel => { root.model-cancel(model.id); }+ remove => { root.model-remove(model.id); }+ }+ }++ // The machine's own accelerator, where the build has one.+ // A bet rather than a default: CoreML takes the parts of a+ // network it knows and hands the rest to the CPU, so how+ // much faster a read gets is the model's business.+ if root.data.speech-acceleration-offered: DialogSection {+ title: I18n.t("Engine");+ hint: I18n.t("On, the voices run on this Mac's GPU and Neural Engine through CoreML wherever the model allows, and on the CPU where it does not. Off, everything runs on the CPU. Takes effect on the next read; a model already loaded is loaded again.");++ HorizontalLayout {+ spacing: 6px;- for model in root.voices: ModelRow {- model: model;- activated => { root.model-activated(model.id); }- download => { root.model-download(model.id); }- cancel => { root.model-cancel(model.id); }- remove => { root.model-remove(model.id); }+ Text {+ text: I18n.t("Run voices on the GPU");+ color: Theme.fg;+ font-size: Theme.fs;+ vertical-alignment: center;+ }++ Rectangle { horizontal-stretch: 1; }++ VerticalLayout {+ alignment: center;+ Switch {+ checked: root.data.speech-accelerated;+ toggled(on) => { root.speech-accelerated-changed(on); }+ }+ }+ }}}