SOTA Open Source TTS
Fix vq alignment
Lengyuef578dc35722e
diff --git a/fish_speech/configs/vq_diffusion.yaml b/fish_speech/configs/vq_diffusion.yamlindex 99dbbe6..2d6cb4c 100644--- a/fish_speech/configs/vq_diffusion.yaml+++ b/fish_speech/configs/vq_diffusion.yaml@@ -50,6 +50,12 @@ model:hop_length: ${hop_length}speaker_use_feats: true+ downsample:+ _target_: fish_speech.models.vq_diffusion.lit_module.ConvDownSample+ dims: [128, 512, 128]+ kernel_sizes: [3, 3]+ strides: [2, 2]+text_encoder:_target_: fish_speech.models.vqgan.modules.encoders.TextEncoderin_channels: 128@@ -104,7 +110,7 @@ model:_target_: fish_speech.models.vqgan.spectrogram.LogMelSpectrogramsample_rate: 32000n_fft: 2048- hop_length: 1280+ hop_length: 320win_length: 2048n_mels: 128diff --git a/fish_speech/models/vq_diffusion/bigvgan/bigvgan.py b/fish_speech/models/vq_diffusion/bigvgan/bigvgan.pyindex 71ab337..7a3a7cb 100644--- a/fish_speech/models/vq_diffusion/bigvgan/bigvgan.py+++ b/fish_speech/models/vq_diffusion/bigvgan/bigvgan.py@@ -366,9 +366,7 @@ class BigVGAN(nn.Module):@torch.no_grad()def decode(self, mel):- mel = F.pad(mel, (0, 10), "reflect")y = self.model(mel)- y = y[:, :, : -self.h.hop_size * 10]return y@torch.no_grad()diff --git a/fish_speech/models/vq_diffusion/lit_module.py b/fish_speech/models/vq_diffusion/lit_module.pyindex 37d14f6..1890c04 100644--- a/fish_speech/models/vq_diffusion/lit_module.py+++ b/fish_speech/models/vq_diffusion/lit_module.py@@ -1,7 +1,8 @@import itertools-from typing import Any, Callable+from typing import Any, Callable, Optionalimport lightning as L+import numpy as npimport torchimport torch.nn.functional as Fimport wandb@@ -11,7 +12,6 @@ from lightning.pytorch.loggers import TensorBoardLogger, WandbLoggerfrom matplotlib import pyplot as pltfrom torch import nnfrom tqdm import tqdm-from transformers import HubertModelfrom fish_speech.models.vq_diffusion.convnext_1d import ConvNext1DModelfrom fish_speech.models.vqgan.modules.encoders import (@@ -22,6 +22,57 @@ from fish_speech.models.vqgan.modules.encoders import (from fish_speech.models.vqgan.utils import plot_mel, sequence_mask+class ConvDownSample(nn.Module):+ def __init__(+ self,+ dims: list,+ kernel_sizes: list,+ strides: list,+ ):+ super().__init__()++ self.dims = dims+ self.kernel_sizes = kernel_sizes+ self.strides = strides+ self.total_strides = np.prod(self.strides)++ self.convs = nn.ModuleList(+ [+ nn.ModuleList(+ [+ nn.Conv1d(+ in_channels=self.dims[i],+ out_channels=self.dims[i + 1],+ kernel_size=self.kernel_sizes[i],+ stride=self.strides[i],+ padding=(self.kernel_sizes[i] - 1) // 2,+ ),+ nn.LayerNorm(self.dims[i + 1], elementwise_affine=True),+ nn.GELU(),+ ]+ )+ for i in range(len(self.dims) - 1)+ ]+ )++ self.apply(self.init_weights)++ def init_weights(self, m):+ if isinstance(m, nn.Conv1d):+ nn.init.normal_(m.weight, std=0.02)+ elif isinstance(m, nn.LayerNorm):+ nn.init.ones_(m.weight)+ nn.init.zeros_(m.bias)++ def forward(self, x):+ for conv, norm, act in self.convs:+ x = conv(x)+ x = norm(x.mT).mT+ x = act(x)++ return x++class VQDiffusion(L.LightningModule):def __init__(self,@@ -37,6 +88,7 @@ class VQDiffusion(L.LightningModule):hop_length: int = 640,sample_rate: int = 32000,speaker_use_feats: bool = False,+ downsample: Optional[nn.Module] = None,):super().__init__()@@ -55,6 +107,7 @@ class VQDiffusion(L.LightningModule):self.speaker_encoder = speaker_encoderself.text_encoder = text_encoderself.denoiser = denoiser+ self.downsample = downsampleself.vocoder = vocoderself.hop_length = hop_length@@ -100,8 +153,18 @@ class VQDiffusion(L.LightningModule):audios, sample_rate=self.sampling_rate)+ if self.downsample is not None:+ features = self.downsample(features)+mel_lengths = audio_lengths // self.hop_length- feature_lengths = audio_lengths // self.hop_length // 2+ feature_lengths = (+ audio_lengths+ / self.sampling_rate+ * self.feature_mel_transform.sample_rate+ / self.feature_mel_transform.hop_length+ / (self.downsample.total_strides if self.downsample is not None else 1)+ ).long()+feature_masks = torch.unsqueeze(sequence_mask(feature_lengths, features.shape[2]), 1).to(gt_mels.dtype)@@ -181,8 +244,18 @@ class VQDiffusion(L.LightningModule):gt_mels = self.mel_transform(audios, sample_rate=self.sampling_rate)features = self.feature_mel_transform(audios, sample_rate=self.sampling_rate)+ if self.downsample is not None:+ features = self.downsample(features)+mel_lengths = audio_lengths // self.hop_length- feature_lengths = audio_lengths // self.hop_length // 2+ feature_lengths = (+ audio_lengths+ / self.sampling_rate+ * self.feature_mel_transform.sample_rate+ / self.feature_mel_transform.hop_length+ / (self.downsample.total_strides if self.downsample is not None else 1)+ ).long()+feature_masks = torch.unsqueeze(sequence_mask(feature_lengths, features.shape[2]), 1).to(gt_mels.dtype)@@ -222,6 +295,7 @@ class VQDiffusion(L.LightningModule):).prev_samplesampled_mels = self.denormalize_mels(sampled_mels)+ sampled_mels = sampled_mels * mel_maskswith torch.autocast(device_type=sampled_mels.device.type, enabled=False):# Run vocoder on fp32