SOTA Open Source TTS
Try fix infinity
Lengyuefb68b0271655
diff --git a/fish_speech/configs/vqgan_pretrain.yaml b/fish_speech/configs/vqgan_pretrain.yamlindex 86eb33f..7e4cbbb 100644--- a/fish_speech/configs/vqgan_pretrain.yaml+++ b/fish_speech/configs/vqgan_pretrain.yaml@@ -7,7 +7,7 @@ project: vq_reflow_shallow_group_fsq_8x1024_wavenet# Lightning Trainertrainer:accelerator: gpu- devices: auto+ devices: 1precision: bf16-mixedmax_steps: 1_000_000val_check_interval: 1000@@ -38,7 +38,7 @@ data:train_dataset: ${train_dataset}val_dataset: ${val_dataset}num_workers: 4- batch_size: 32+ batch_size: 128val_batch_size: 4# Model Configuration@@ -52,9 +52,8 @@ model:freeze_encoder: false# Reflow configs- reflow_use_shallow: truereflow_inference_steps: 10- reflow_inference_start_t: 0.5+ reflow_inference_start_t: 0.0encoder:_target_: fish_speech.models.vqgan.modules.wavenet.WaveNetdiff --git a/fish_speech/models/vqgan/lit_module.py b/fish_speech/models/vqgan/lit_module.pyindex 1975aea..b4fb59e 100644--- a/fish_speech/models/vqgan/lit_module.py+++ b/fish_speech/models/vqgan/lit_module.py@@ -30,7 +30,6 @@ class VQGAN(L.LightningModule):weight_mel: float = 1.0,sampling_rate: int = 44100,freeze_encoder: bool = False,- reflow_use_shallow: bool = False,reflow_inference_steps: int = 10,reflow_inference_start_t: float = 0.5,):@@ -61,7 +60,6 @@ class VQGAN(L.LightningModule):self.spec_min = -12self.spec_max = 3self.sampling_rate = sampling_rate- self.reflow_use_shallow = reflow_use_shallowself.reflow_inference_steps = reflow_inference_stepsself.reflow_inference_start_t = reflow_inference_start_t@@ -133,18 +131,12 @@ class VQGAN(L.LightningModule):# Reflow, given x_1_aux, we want to reconstruct x_1x_1 = self.norm_spec(gt_mels)-- if self.reflow_use_shallow:- # Detach the gradient of the generated mel- x_1_aux = self.norm_spec(gen_mel.detach())- else:- x_1_aux = x_1-- t = torch.rand(gt_mels.shape[0], device=gt_mels.device)+ t = torch.rand(gt_mels.shape[0], device=gt_mels.device, dtype=torch.float32)+ t = torch.clamp(t, 1e-6, 1 - 1e-6) # Avoid 0 and 1x_0 = torch.randn_like(x_1)# X_t = t * X_1 + (1 - t) * X_0- x_t = x_0 + t[:, None, None] * (x_1_aux - x_0)+ x_t = x_0 + t[:, None, None] * (x_1 - x_0)v_pred = self.reflow(x_t,@@ -153,13 +145,21 @@ class VQGAN(L.LightningModule):)# Log L2 loss with- weights = 0.398942 / t / (1 - t) * torch.exp(-0.5 * torch.log(t / (1 - t)) ** 2)- loss_reflow = weights[:, None, None] * F.mse_loss(- x_1 - x_0, v_pred, reduction="none"- )- loss_reflow = (loss_reflow * mel_masks_float_conv).mean(- dim=1- ).sum() / mel_masks_float_conv.sum()+ with torch.autocast(device_type=gt_mels.device.type, dtype=torch.float32):+ weights = (+ 0.398942 / t / (1 - t) * torch.exp(-0.5 * torch.log(t / (1 - t)) ** 2)+ )+ assert (+ torch.isnan(weights).any() == False+ and torch.isinf(weights).any() == False+ ), "Found NaN or Inf in weights."++ loss_reflow = weights[:, None, None] * F.mse_loss(+ x_1 - x_0, v_pred, reduction="none"+ )+ loss_reflow = (loss_reflow * mel_masks_float_conv).mean(+ dim=1+ ).sum() / mel_masks_float_conv.sum()# Total lossloss = (@@ -239,7 +239,7 @@ class VQGAN(L.LightningModule):)# Reflow inference- t_start = self.reflow_inference_start_t if self.reflow_use_shallow else 0.0+ t_start = 0.0x_1 = self.norm_spec(gen_aux_mels)x_0 = torch.randn_like(x_1)