Skip to content

The truly free, open-source, local-first, cross-platform CapCut replacement. Scriptable through a JSON-RPC API & MCP.

feat(speech): Chatterbox runs on the GPU on Windows, through DirectML

Jareer27dad0b32098
diff --git a/src/crates/concat-speech/Cargo.toml b/src/crates/concat-speech/Cargo.toml
index 330d6132..f04cc400 100644
--- a/src/crates/concat-speech/Cargo.toml
+++ b/src/crates/concat-speech/Cargo.toml
@@ -73,6 +73,11 @@ whisper-rs = { version = "0.16", features = ["metal"] }
# runtime in place of the plain one.
ort = { workspace = true, optional = true, features = ["coreml"] }
+# DirectML, so Chatterbox runs on any Direct3D 12 GPU; the same runtime
+# build concat-vision's cutouts use.
+[target.'cfg(target_os = "windows")'.dependencies]
+ort = { workspace = true, optional = true, features = ["directml"] }
+
[features]
chatterbox = ["dep:ort", "dep:tokenizers"]
diff --git a/src/crates/concat-speech/src/chatterbox.rs b/src/crates/concat-speech/src/chatterbox.rs
index 576a1aa6..f66c7b5b 100644
--- a/src/crates/concat-speech/src/chatterbox.rs
+++ b/src/crates/concat-speech/src/chatterbox.rs
@@ -299,7 +299,19 @@ fn session(path: &Path, accelerated: bool) -> Result<Session, String> {
path.file_name().unwrap_or_default().to_string_lossy()
);
}
- #[cfg(not(target_os = "macos"))]
+ // DirectML, the GPU of any vendor through Direct3D 12. The runtime is
+ // built with it on Windows already, for the cutout models (concat-vision).
+ #[cfg(target_os = "windows")]
+ if accelerated {
+ builder = builder
+ .with_execution_providers([ort::ep::DirectML::default().build()])
+ .map_err(|error| format!("onnx runtime: directml: {error}"))?;
+ log::info!(
+ "chatterbox: {} asked to run on DirectML",
+ path.file_name().unwrap_or_default().to_string_lossy()
+ );
+ }
+ #[cfg(not(any(target_os = "macos", target_os = "windows")))]
let _ = accelerated;
builder
.commit_from_file(path)
diff --git a/src/crates/concat-speech/src/lib.rs b/src/crates/concat-speech/src/lib.rs
index 9157831f..83cd34b3 100644
--- a/src/crates/concat-speech/src/lib.rs
+++ b/src/crates/concat-speech/src/lib.rs
@@ -26,7 +26,10 @@ pub use transcribe::Transcriber;
pub use tts::Speech;
/// Whether the voices ask for the machine's own accelerator: CoreML, the
-/// Mac's GPU and Neural Engine. Not a setting. Measured on an M5 against
+/// Mac's GPU and Neural Engine, and on Windows DirectML for Chatterbox -
+/// Kokoro and Pocket run through sherpa's own runtime, which k2-fsa builds
+/// for the CPU there, so they stay on it. Linux has no accelerator in the
+/// runtime the app ships. Not a setting. Measured on an M5 against
/// the CPU, a warm read took 10.3 s against 14.8 for Kokoro (and came out
/// sample for sample the same), 5.6 against 6.5 for Pocket and 48.6
/// against 60.6 for Chatterbox. The one cost is Chatterbox's load, about
@@ -34,7 +37,7 @@ pub use tts::Speech;
/// run since the engine stays loaded. A model that will not load for CoreML
/// at all loads for the CPU instead (tts.rs).
pub const fn accelerated() -> bool {
- cfg!(target_os = "macos")
+ cfg!(any(target_os = "macos", target_os = "windows"))
}
/// Progress for one model download.