The open-source AI voice studio. Clone, dictate, create.
fix: patch S3Tokenizer.log_mel_spectrogram for float64→float32 cast
James Pinebfd7b815a596
diff --git a/backend/backends/chatterbox_backend.py b/backend/backends/chatterbox_backend.pyindex 826d7c05..a440f998 100644--- a/backend/backends/chatterbox_backend.py+++ b/backend/backends/chatterbox_backend.py@@ -178,20 +178,32 @@ class ChatterboxTTSBackend:progress_manager.mark_complete(model_name)task_manager.complete_download(model_name)- # Monkey-patch VoiceEncoder.forward to cast input to float32.- # The upstream melspectrogram returns float64 numpy arrays when- # hp.normalized_mels is False (the default). pack() preserves- # the dtype, so double tensors hit float32 LSTM weights →- # "expected m1 and m2 to have the same dtype: float != double".- _ve = self.model.ve- _orig_ve_forward = _ve.forward.__func__ if hasattr(_ve.forward, '__func__') else _ve.forward-+ # Patch float64 → float32 dtype mismatches in upstream chatterbox.+ # librosa.load returns float64 numpy; multiple upstream code paths+ # convert it to a torch tensor via torch.from_numpy() without+ # casting, then matmul it against float32 model weights.import types- def _f32_forward(self_ve, mels):+ # Patch S3Tokenizer (used by s3gen.tokenizer)+ _tokzr = self.model.s3gen.tokenizer+ _orig_log_mel = _tokzr.log_mel_spectrogram.__func__++ def _f32_log_mel(self_tokzr, audio, padding=0):+ import torch as _torch+ if _torch.is_tensor(audio):+ audio = audio.float()+ return _orig_log_mel(self_tokzr, audio, padding)++ _tokzr.log_mel_spectrogram = types.MethodType(_f32_log_mel, _tokzr)++ # Patch VoiceEncoder+ _ve = self.model.ve+ _orig_ve_forward = _ve.forward.__func__++ def _f32_ve_forward(self_ve, mels):return _orig_ve_forward(self_ve, mels.float())- _ve.forward = types.MethodType(_f32_forward, _ve)+ _ve.forward = types.MethodType(_f32_ve_forward, _ve)logger.info("Chatterbox Multilingual TTS loaded successfully")diff --git a/backend/backends/chatterbox_turbo_backend.py b/backend/backends/chatterbox_turbo_backend.pyindex 6c5bf842..3ce54f38 100644--- a/backend/backends/chatterbox_turbo_backend.py+++ b/backend/backends/chatterbox_turbo_backend.py@@ -178,20 +178,38 @@ class ChatterboxTurboTTSBackend:progress_manager.mark_complete(model_name)task_manager.complete_download(model_name)- # Monkey-patch VoiceEncoder.forward to cast input to float32.- # The upstream melspectrogram returns float64 numpy arrays when- # hp.normalized_mels is False (the default). pack() preserves- # the dtype, so double tensors hit float32 LSTM weights →- # "expected m1 and m2 to have the same dtype: float != double".- _ve = self.model.ve- _orig_ve_forward = _ve.forward.__func__ if hasattr(_ve.forward, '__func__') else _ve.forward-+ # Patch float64 → float32 dtype mismatches in upstream chatterbox.+ # librosa.load returns float64 numpy; multiple upstream code paths+ # convert it to a torch tensor via torch.from_numpy() without+ # casting, then matmul it against float32 model weights.+ # We patch the two known entry points:+ #+ # 1. S3Tokenizer.log_mel_spectrogram — the audio tensor from+ # librosa hits _mel_filters (float32) in a matmul.+ # 2. VoiceEncoder.forward — float64 mel spectrograms hit the+ # float32 LSTM weights.import types- def _f32_forward(self_ve, mels):+ # Patch S3Tokenizer (used by s3gen.tokenizer)+ _tokzr = self.model.s3gen.tokenizer+ _orig_log_mel = _tokzr.log_mel_spectrogram.__func__++ def _f32_log_mel(self_tokzr, audio, padding=0):+ import torch as _torch+ if _torch.is_tensor(audio):+ audio = audio.float()+ return _orig_log_mel(self_tokzr, audio, padding)++ _tokzr.log_mel_spectrogram = types.MethodType(_f32_log_mel, _tokzr)++ # Patch VoiceEncoder+ _ve = self.model.ve+ _orig_ve_forward = _ve.forward.__func__++ def _f32_ve_forward(self_ve, mels):return _orig_ve_forward(self_ve, mels.float())- _ve.forward = types.MethodType(_f32_forward, _ve)+ _ve.forward = types.MethodType(_f32_ve_forward, _ve)logger.info("Chatterbox Turbo TTS loaded successfully")