The open-source AI voice studio. Clone, dictate, create.
feat(backend): run Chatterbox multilingual on MLX for Apple Silicon
Charles Hassead5d64c0a3d2
diff --git a/backend/backends/__init__.py b/backend/backends/__init__.pyindex e3cfe049..9c539da0 100644--- a/backend/backends/__init__.py+++ b/backend/backends/__init__.py@@ -290,10 +290,17 @@ def _get_qwen_custom_voice_configs() -> list[ModelConfig]:def _get_non_qwen_tts_configs() -> list[ModelConfig]:- """Return model configs for non-Qwen TTS engines.+ """Return model configs for non-Qwen TTS engines."""+ # Chatterbox multilingual follows the same backend-aware split as Qwen: the MLX+ # backend loads pre-converted weights, so the download must match the backend that+ # will consume it.+ if get_backend_type() == "mlx":+ chatterbox_repo = "mlx-community/chatterbox-multilingual-v3"+ chatterbox_size_mb = 2600+ else:+ chatterbox_repo = "ResembleAI/chatterbox"+ chatterbox_size_mb = 3200- These are static — no backend-type branching needed.- """return [ModelConfig(model_name="luxtts",@@ -307,8 +314,8 @@ def _get_non_qwen_tts_configs() -> list[ModelConfig]:model_name="chatterbox-tts",display_name="Chatterbox TTS (Multilingual)",engine="chatterbox",- hf_repo_id="ResembleAI/chatterbox",- size_mb=3200,+ hf_repo_id=chatterbox_repo,+ size_mb=chatterbox_size_mb,needs_trim=True,languages=["zh",@@ -707,9 +714,16 @@ def get_tts_backend_for_engine(engine: str) -> TTSBackend:backend = LuxTTSBackend()elif engine == "chatterbox":- from .chatterbox_backend import ChatterboxTTSBackend+ # Same split the qwen engine already makes: on Apple Silicon the MLX/Metal+ # port renders 7-9x faster than the CPU-pinned PyTorch path.+ if get_backend_type() == "mlx":+ from .chatterbox_mlx_backend import ChatterboxMLXTTSBackend++ backend = ChatterboxMLXTTSBackend()+ else:+ from .chatterbox_backend import ChatterboxTTSBackend- backend = ChatterboxTTSBackend()+ backend = ChatterboxTTSBackend()elif engine == "chatterbox_turbo":from .chatterbox_turbo_backend import ChatterboxTurboTTSBackenddiff --git a/backend/backends/chatterbox_mlx_backend.py b/backend/backends/chatterbox_mlx_backend.pynew file mode 100644index 00000000..57a88224--- /dev/null+++ b/backend/backends/chatterbox_mlx_backend.py@@ -0,0 +1,179 @@+"""+Chatterbox multilingual TTS backend, MLX (Metal) flavour.++Same zero-shot voice cloning and same 23 languages as ``chatterbox_backend``, but running+on the Apple Silicon GPU through mlx-audio instead of PyTorch on the CPU.++The PyTorch path is pinned to the CPU on macOS (see ``chatterbox_backend``), which costs+roughly 4x realtime. This backend uses the pre-converted weights published as+``mlx-community/chatterbox-multilingual-v3`` and renders the same sentences at about 0.5x+realtime on an M4 Max with a cloned pt-BR profile.++This mirrors the split the qwen engine already makes between ``mlx_backend`` and+``pytorch_backend``: MLX where it is available, PyTorch everywhere else.+"""++import asyncio+import logging+from pathlib import Path+from typing import ClassVar++import numpy as np++from .base import (+ combine_voice_prompts as _combine_voice_prompts,+ is_model_cached,+ model_load_progress,+)++logger = logging.getLogger(__name__)++CHATTERBOX_MLX_HF_REPO = "mlx-community/chatterbox-multilingual-v3"++# Files that must be present for the MLX multilingual model+_MLX_WEIGHT_FILES = ["model.safetensors", "config.json", "tokenizer.json"]+++class ChatterboxMLXTTSBackend:+ """Chatterbox Multilingual TTS backend for voice cloning, on MLX/Metal."""++ def __init__(self):+ self.model = None+ self.model_size = "default"+ self._model_load_lock = asyncio.Lock()++ def is_loaded(self) -> bool:+ return self.model is not None++ def _get_model_path(self, model_size: str = "default") -> str:+ return CHATTERBOX_MLX_HF_REPO++ def _is_model_cached(self, model_size: str = "default") -> bool:+ return is_model_cached(CHATTERBOX_MLX_HF_REPO, required_files=_MLX_WEIGHT_FILES)++ async def load_model(self, model_size: str = "default") -> None:+ """Load the Chatterbox multilingual MLX model."""+ if self.model is not None:+ return+ async with self._model_load_lock:+ if self.model is not None:+ return+ await asyncio.to_thread(self._load_model_sync)++ def _load_model_sync(self):+ """Synchronous model loading."""+ is_cached = self._is_model_cached()++ with model_load_progress("chatterbox-tts", is_cached):+ from huggingface_hub import snapshot_download # lazy: heavy import+ from mlx_audio.tts.models.chatterbox.chatterbox import Model # lazy: heavy import++ logger.info("Loading Chatterbox Multilingual TTS on MLX (Metal)...")+ ckpt_dir = snapshot_download(CHATTERBOX_MLX_HF_REPO)+ self.model = Model.from_pretrained(ckpt_dir)++ logger.info("Chatterbox Multilingual TTS (MLX) loaded successfully")++ def unload_model(self) -> None:+ """Unload model to free memory."""+ if self.model is None:+ return++ del self.model+ self.model = None+ try:+ import mlx.core as mx # lazy: heavy import++ mx.clear_cache()+ except Exception:+ logger.debug("mlx cache not cleared", exc_info=True)+ logger.info("Chatterbox (MLX) unloaded")++ async def create_voice_prompt(+ self,+ audio_path: str,+ reference_text: str,+ use_cache: bool = True,+ ) -> tuple[dict, bool]:+ """+ Create voice prompt from reference audio.++ Chatterbox conditions on the reference audio at generation time, so the prompt+ just stores the file path.+ """+ voice_prompt = {+ "ref_audio": str(audio_path),+ "ref_text": reference_text,+ }+ return voice_prompt, False++ async def combine_voice_prompts(+ self,+ audio_paths: list[str],+ reference_texts: list[str],+ ) -> tuple[np.ndarray, str]:+ return await _combine_voice_prompts(audio_paths, reference_texts)++ # The MLX port carries its own sampling defaults, validated by ear against the PyTorch+ # output on a cloned profile. The PyTorch tuning (repetition_penalty=2.0) is not+ # transferable: the two implementations weight it differently.+ _DEFAULTS: ClassVar[dict] = {+ "exaggeration": 0.1,+ "cfg_weight": 0.5,+ "temperature": 0.8,+ "repetition_penalty": 1.2,+ }++ async def generate(+ self,+ text: str,+ voice_prompt: dict,+ language: str = "en",+ seed: int | None = None,+ instruct: str | None = None,+ ) -> tuple[np.ndarray, int]:+ """+ Generate audio using Chatterbox Multilingual TTS on MLX.++ Args:+ text: Text to synthesize+ voice_prompt: Dict with ref_audio path+ language: BCP-47 language code+ seed: Random seed for reproducibility+ instruct: Unused (protocol compatibility)++ Returns:+ Tuple of (audio_array, sample_rate)+ """+ await self.load_model()++ ref_audio = voice_prompt.get("ref_audio")+ if ref_audio and not Path(ref_audio).exists():+ logger.warning(f"Reference audio not found: {ref_audio}")+ ref_audio = None++ def _generate_sync():+ import mlx.core as mx # lazy: heavy import++ if seed is not None:+ mx.random.seed(seed)++ logger.info(f"[Chatterbox/MLX] Generating: lang={language}")++ # mlx-audio yields GenerationResult chunks; the whole clip is their concatenation.+ chunks = [+ np.asarray(result.audio).squeeze()+ for result in self.model.generate(+ text,+ ref_audio=ref_audio,+ lang_code=language,+ verbose=False,+ **self._DEFAULTS,+ )+ ]+ audio = np.concatenate(chunks).astype(np.float32) if chunks else np.zeros(0, dtype=np.float32)++ sample_rate = getattr(self.model, "sr", None) or 24000+ return audio, int(sample_rate)++ return await asyncio.to_thread(_generate_sync)