SOTA Open Source TTS
Add lfq
Lengyuedf304ac4384f
diff --git a/fish_speech/configs/hubert_vq_diffusion.yaml b/fish_speech/configs/hubert_vq_diffusion.yamlindex a814a2b..b96a860 100644--- a/fish_speech/configs/hubert_vq_diffusion.yaml+++ b/fish_speech/configs/hubert_vq_diffusion.yaml@@ -13,7 +13,7 @@ trainer:gradient_clip_algorithm: 'norm'precision: 16-mixedmax_steps: 1_000_000- val_check_interval: 1000+ val_check_interval: 5000sample_rate: 44100hop_length: 512@@ -67,7 +67,7 @@ model:_target_: fish_speech.models.vqgan.modules.encoders.VQEncoderin_channels: 128vq_channels: 128- codebook_size: 4096+ codebook_size: 16384downsample: 1speaker_encoder:diff --git a/fish_speech/models/vq_diffusion/lit_module.py b/fish_speech/models/vq_diffusion/lit_module.pyindex a545589..b20e097 100644--- a/fish_speech/models/vq_diffusion/lit_module.py+++ b/fish_speech/models/vq_diffusion/lit_module.py@@ -218,7 +218,7 @@ class VQDiffusion(L.LightningModule):# Run vocoder on fp32fake_audios = self.vocoder.decode(sampled_mels.float())- mel_loss = F.l1_loss(gt_mels, sampled_mels)+ mel_loss = F.l1_loss(gt_mels * mel_masks, sampled_mels * mel_masks)self.log("val/mel_loss",mel_loss,diff --git a/fish_speech/models/vqgan/modules/encoders.py b/fish_speech/models/vqgan/modules/encoders.pyindex 5bc8934..ef5a6d2 100644--- a/fish_speech/models/vqgan/modules/encoders.py+++ b/fish_speech/models/vqgan/modules/encoders.py@@ -3,7 +3,7 @@ from typing import Optionalimport torchimport torch.nn as nnimport torch.nn.functional as F-from vector_quantize_pytorch import VectorQuantize+from vector_quantize_pytorch import LFQ, VectorQuantizefrom fish_speech.models.vqgan.modules.modules import WNfrom fish_speech.models.vqgan.modules.transformer import (@@ -234,12 +234,12 @@ class VQEncoder(nn.Module):):super().__init__()- self.vq = VectorQuantize(+ self.vq = LFQ(dim=vq_channels,codebook_size=codebook_size,- threshold_ema_dead_code=2,- kmeans_init=False,- channel_last=False,+ # threshold_ema_dead_code=2,+ # kmeans_init=False,+ # channel_last=False,)self.downsample = downsampleself.conv_in = nn.Conv1d(@@ -286,8 +286,8 @@ class VQEncoder(nn.Module):x_mask = F.pad(x_mask, (0, self.downsample - x_len % self.downsample))x = self.conv_in(x)- q, _, loss = self.vq(x)- x = self.conv_out(q) * x_mask+ q, _, loss = self.vq(x.mT)+ x = self.conv_out(q.mT) * x_maskx = x[:, :, :x_len]return x, loss