Skip to content

SOTA Open Source TTS

remove unused configs

Lengyue49f84853c463
diff --git a/fish_speech/configs/llama_finetune.yaml b/fish_speech/configs/llama_finetune.yaml
deleted file mode 100644
index 052764a..0000000
--- a/fish_speech/configs/llama_finetune.yaml
+++ /dev/null
@@ -1,63 +0,0 @@
-defaults:
- - base
- - _self_
-
-project: llama_finetune
-
-# Lightning Trainer
-trainer:
- accumulate_grad_batches: 2
- gradient_clip_val: 1.0
- gradient_clip_algorithm: 'norm'
-
-# Dataset Configuration
-tokenizer:
- _target_: transformers.AutoTokenizer.from_pretrained
- pretrained_model_name_or_path: fishaudio/speech-lm-300m
- revision: text-pretrain-10k
-
-# Dataset Configuration
-train_dataset:
- - _target_: fish_speech.datasets.text.StreamTextDataset
- repo: fishaudio/cn-hubert-25hz-vq
- prefix: 'data/train'
-
-val_dataset:
- _target_: fish_speech.datasets.text.StreamTextDataset
- repo: fishaudio/cn-hubert-25hz-vq
- prefix: 'data/test'
-
-data:
- _target_: fish_speech.datasets.text.TextDataModule
- train_dataset: ${train_dataset}
- val_dataset: ${val_dataset}
- num_workers: 4
- batch_size: 8
- tokenizer: ${tokenizer}
-
-# Model Configuration
-model:
- _target_: fish_speech.models.text2semantic.TextToSemantic
-
- model:
- _target_: transformers.AutoModelForCausalLM.from_pretrained
- pretrained_model_name_or_path: fishaudio/speech-lm-300m
- revision: text-pretrain-10k
-
- optimizer:
- _target_: torch.optim.AdamW
- _partial_: true
- lr: 1e-4
- weight_decay: 0.1
- betas: [0.9, 0.95]
- eps: 1e-5
-
- lr_scheduler:
- _target_: torch.optim.lr_scheduler.LambdaLR
- _partial_: true
- lr_lambda:
- _target_: fish_speech.scheduler.get_cosine_schedule_with_warmup_lr_lambda
- _partial_: true
- num_warmup_steps: 2000
- num_training_steps: ${trainer.max_steps}
- final_lr_ratio: 0.1
diff --git a/fish_speech/configs/llama_pretrain.yaml b/fish_speech/configs/llama_pretrain.yaml
deleted file mode 100644
index b0f5df1..0000000
--- a/fish_speech/configs/llama_pretrain.yaml
+++ /dev/null
@@ -1,73 +0,0 @@
-defaults:
- - base
- - _self_
-
-project: llama_pretrain
-
-# Say we want a 3 trillion seen token schedule
-# 3e12 / 1024 / 512 / 8 = 715255
-# But we use a 100k steps schedule here to save time
-# This is a 400 billion seen token schedule:
-# 1024 * 512 * 8 * 100000 = 419_430_400_000
-
-# Lightning Trainer
-trainer:
- accumulate_grad_batches: 64
- gradient_clip_val: 1.0
- gradient_clip_algorithm: 'norm'
- num_nodes: 1
- limit_val_batches: 100 # 100 batches for validation
-
-# Dataset Configuration
-tokenizer:
- _target_: transformers.AutoTokenizer.from_pretrained
- pretrained_model_name_or_path: fishaudio/speech-lm-300m
- revision: init
-
-# Dataset Configuration
-dataset:
- _target_: fish_speech.datasets.text.InterleaveDataset
- datasets:
- - _target_: fish_speech.datasets.text.StreamTextDataset
- prefix: 'en/'
- - _target_: fish_speech.datasets.text.StreamTextDataset
- prefix: 'zh/'
- - _target_: fish_speech.datasets.text.StreamTextDataset
- prefix: 'ja/'
- probabilities: [0.4, 0.3, 0.3]
- seed: 42
-
-data:
- _target_: fish_speech.datasets.text.TextDataModule
- train_dataset: ${dataset}
- val_dataset: ${dataset}
- num_workers: 4
- batch_size: 8
- tokenizer: ${tokenizer}
-
-# Model Configuration
-model:
- _target_: fish_speech.models.text2semantic.TextToSemantic
-
- model:
- _target_: transformers.AutoModelForCausalLM.from_pretrained
- pretrained_model_name_or_path: fishaudio/speech-lm-300m
- revision: init
-
- optimizer:
- _target_: torch.optim.AdamW
- _partial_: true
- lr: 3e-4
- weight_decay: 0.1
- betas: [0.9, 0.95]
- eps: 1e-5
-
- lr_scheduler:
- _target_: torch.optim.lr_scheduler.LambdaLR
- _partial_: true
- lr_lambda:
- _target_: fish_speech.scheduler.get_cosine_schedule_with_warmup_lr_lambda
- _partial_: true
- num_warmup_steps: 2000
- num_training_steps: ${trainer.max_steps}
- final_lr_ratio: 0.1
diff --git a/fish_speech/configs/text2semantic.yaml b/fish_speech/configs/text2semantic_finetune.yaml
similarity index 87%
rename from fish_speech/configs/text2semantic.yaml
rename to fish_speech/configs/text2semantic_finetune.yaml
index a16ae2a..3a03702 100644
--- a/fish_speech/configs/text2semantic.yaml
+++ b/fish_speech/configs/text2semantic_finetune.yaml
@@ -2,15 +2,15 @@ defaults:
- base
- _self_
-project: text2semantic_400m
-max_length: 1024
+project: text2semantic_400m_multi
+max_length: 4096
# Lightning Trainer
trainer:
accumulate_grad_batches: 2
gradient_clip_val: 1.0
gradient_clip_algorithm: 'norm'
- max_steps: 1_000_000
+ max_steps: 10000
precision: bf16-true
limit_val_batches: 10
@@ -55,7 +55,8 @@ model:
dim: 1024
rope_base: 10000
norm_eps: 1e-5
- num_codebooks: 0 # single codebook
+ num_codebooks: 4 # single codebook
+ codebook_size: 168 # codebook size 160 + 2 special tokens
optimizer:
_target_: torch.optim.AdamW
@@ -74,3 +75,8 @@ model:
num_warmup_steps: 2000
num_training_steps: ${trainer.max_steps}
final_lr_ratio: 0.1
+
+# Callbacks
+callbacks:
+ model_checkpoint:
+ every_n_train_steps: 1000
diff --git a/fish_speech/configs/text2semantic_multi.yaml b/fish_speech/configs/text2semantic_pretrain.yaml
similarity index 100%
rename from fish_speech/configs/text2semantic_multi.yaml
rename to fish_speech/configs/text2semantic_pretrain.yaml
diff --git a/fish_speech/configs/vq_naive.yaml b/fish_speech/configs/vq_naive.yaml
deleted file mode 100644
index b23485d..0000000
--- a/fish_speech/configs/vq_naive.yaml
+++ /dev/null
@@ -1,145 +0,0 @@
-defaults:
- - base
- - _self_
-
-project: vq_naive
-
-# Lightning Trainer
-trainer:
- accelerator: gpu
- devices: 1
- strategy: ddp_find_unused_parameters_true
- gradient_clip_val: 1.0
- gradient_clip_algorithm: 'norm'
- precision: bf16-mixed
- max_steps: 100_000
- val_check_interval: 5000
-
-sample_rate: 22050
-hop_length: 256
-num_mels: 80
-n_fft: 1024
-win_length: 1024
-segment_size: 512
-
-# Dataset Configuration
-train_dataset:
- _target_: fish_speech.datasets.vqgan.VQGANDataset
- filelist: data/filelist.split.train
- sample_rate: ${sample_rate}
- hop_length: ${hop_length}
- slice_frames: ${segment_size}
-
-val_dataset:
- _target_: fish_speech.datasets.vqgan.VQGANDataset
- filelist: data/filelist.split.valid
- sample_rate: ${sample_rate}
- hop_length: ${hop_length}
-
-data:
- _target_: fish_speech.datasets.vqgan.VQGANDataModule
- train_dataset: ${train_dataset}
- val_dataset: ${val_dataset}
- num_workers: 8
- batch_size: 128
- val_batch_size: 16
-
-# Model Configuration
-model:
- _target_: fish_speech.models.vqgan.VQNaive
- sample_rate: ${sample_rate}
- hop_length: ${hop_length}
-
- downsample:
- _target_: fish_speech.models.vq_diffusion.lit_module.ConvDownSample
- dims: ["${num_mels}", 512, 256]
- kernel_sizes: [3, 3]
- strides: [2, 2]
-
- mel_encoder:
- _target_: fish_speech.models.vqgan.modules.encoders.TextEncoder
- in_channels: 256
- out_channels: 256
- hidden_channels: 192
- hidden_channels_ffn: 768
- n_heads: 2
- n_layers: 6
- kernel_size: 1
- dropout: 0.1
- use_vae: false
-
- vq_encoder:
- _target_: fish_speech.models.vqgan.modules.encoders.VQEncoder
- in_channels: 256
- vq_channels: 256
- codebook_size: 4096
- downsample: 1
-
- speaker_encoder:
- _target_: fish_speech.models.vqgan.modules.encoders.SpeakerEncoder
- in_channels: ${num_mels}
- hidden_channels: 192
- out_channels: 256
- num_heads: 2
- num_layers: 4
- p_dropout: 0.1
-
- decoder:
- _target_: fish_speech.models.vqgan.modules.encoders.TextEncoder
- in_channels: 256
- out_channels: ${num_mels}
- hidden_channels: 192
- hidden_channels_ffn: 768
- n_heads: 2
- n_layers: 8
- kernel_size: 1
- use_vae: false
- dropout: 0.1
- gin_channels: 256
- speaker_cond_layer: 4
-
- mel_transform:
- _target_: fish_speech.models.vqgan.spectrogram.LogMelSpectrogram
- sample_rate: ${sample_rate}
- n_fft: ${n_fft}
- hop_length: ${hop_length}
- win_length: ${win_length}
- n_mels: ${num_mels}
- f_min: 0
- f_max: 8000
-
- vocoder:
- _target_: fish_speech.models.vqgan.modules.decoder.Generator
- initial_channel: ${num_mels}
- resblock: "1"
- resblock_kernel_sizes: [3, 7, 11]
- resblock_dilation_sizes: [[1, 3, 5], [1, 3, 5], [1, 3, 5]]
- upsample_rates: [8, 8, 2, 2]
- upsample_initial_channel: 512
- upsample_kernel_sizes: [16, 16, 4, 4]
- ckpt_path: "checkpoints/hifigan-v1-universal-22050/g_02500000"
-
- optimizer:
- _target_: torch.optim.AdamW
- _partial_: true
- lr: 1e-4
- betas: [0.9, 0.999]
- eps: 1e-5
-
- lr_scheduler:
- _target_: torch.optim.lr_scheduler.LambdaLR
- _partial_: true
- lr_lambda:
- _target_: fish_speech.scheduler.get_cosine_schedule_with_warmup_lr_lambda
- _partial_: true
- num_warmup_steps: 1000
- num_training_steps: ${trainer.max_steps}
- final_lr_ratio: 0.05
-
-callbacks:
- grad_norm_monitor:
- sub_module:
- - mel_encoder
- - vq_encoder
- - speaker_encoder
- - decoder
diff --git a/fish_speech/configs/vq_naive_40hz.yaml b/fish_speech/configs/vq_naive_40hz.yaml
deleted file mode 100644
index f1f6812..0000000
--- a/fish_speech/configs/vq_naive_40hz.yaml
+++ /dev/null
@@ -1,145 +0,0 @@
-defaults:
- - base
- - _self_
-
-project: vq_naive_40hz
-
-# Lightning Trainer
-trainer:
- accelerator: gpu
- devices: 4
- strategy: ddp_find_unused_parameters_true
- gradient_clip_val: 1.0
- gradient_clip_algorithm: 'norm'
- precision: bf16-mixed
- max_steps: 1_000_000
- val_check_interval: 5000
-
-sample_rate: 22050
-hop_length: 256
-num_mels: 80
-n_fft: 1024
-win_length: 1024
-segment_size: 512
-
-# Dataset Configuration
-train_dataset:
- _target_: fish_speech.datasets.vqgan.VQGANDataset
- filelist: data/filelist.split.train
- sample_rate: ${sample_rate}
- hop_length: ${hop_length}
- slice_frames: ${segment_size}
-
-val_dataset:
- _target_: fish_speech.datasets.vqgan.VQGANDataset
- filelist: data/filelist.split.valid
- sample_rate: ${sample_rate}
- hop_length: ${hop_length}
-
-data:
- _target_: fish_speech.datasets.vqgan.VQGANDataModule
- train_dataset: ${train_dataset}
- val_dataset: ${val_dataset}
- num_workers: 4
- batch_size: 32
- val_batch_size: 16
-
-# Model Configuration
-model:
- _target_: fish_speech.models.vqgan.VQNaive
- sample_rate: ${sample_rate}
- hop_length: ${hop_length}
-
- downsample:
- _target_: fish_speech.models.vq_diffusion.lit_module.ConvDownSample
- dims: ["${num_mels}", 256]
- kernel_sizes: [3]
- strides: [2]
-
- mel_encoder:
- _target_: fish_speech.models.vqgan.modules.encoders.TextEncoder
- in_channels: 256
- out_channels: 256
- hidden_channels: 192
- hidden_channels_ffn: 768
- n_heads: 2
- n_layers: 6
- kernel_size: 1
- dropout: 0.1
- use_vae: false
-
- vq_encoder:
- _target_: fish_speech.models.vqgan.modules.encoders.VQEncoder
- in_channels: 256
- vq_channels: 256
- codebook_size: 4096
- downsample: 1
-
- speaker_encoder:
- _target_: fish_speech.models.vqgan.modules.encoders.SpeakerEncoder
- in_channels: ${num_mels}
- hidden_channels: 192
- out_channels: 256
- num_heads: 2
- num_layers: 4
- p_dropout: 0.1
-
- decoder:
- _target_: fish_speech.models.vqgan.modules.encoders.TextEncoder
- in_channels: 256
- out_channels: ${num_mels}
- hidden_channels: 192
- hidden_channels_ffn: 768
- n_heads: 2
- n_layers: 8
- kernel_size: 1
- use_vae: false
- dropout: 0.1
- gin_channels: 256
- speaker_cond_layer: 4
-
- mel_transform:
- _target_: fish_speech.models.vqgan.spectrogram.LogMelSpectrogram
- sample_rate: ${sample_rate}
- n_fft: ${n_fft}
- hop_length: ${hop_length}
- win_length: ${win_length}
- n_mels: ${num_mels}
- f_min: 0
- f_max: 8000
-
- vocoder:
- _target_: fish_speech.models.vqgan.modules.decoder.Generator
- initial_channel: ${num_mels}
- resblock: "1"
- resblock_kernel_sizes: [3, 7, 11]
- resblock_dilation_sizes: [[1, 3, 5], [1, 3, 5], [1, 3, 5]]
- upsample_rates: [8, 8, 2, 2]
- upsample_initial_channel: 512
- upsample_kernel_sizes: [16, 16, 4, 4]
- ckpt_path: "checkpoints/hifigan-v1-universal-22050/g_02500000"
-
- optimizer:
- _target_: torch.optim.AdamW
- _partial_: true
- lr: 1e-4
- betas: [0.9, 0.999]
- eps: 1e-5
-
- lr_scheduler:
- _target_: torch.optim.lr_scheduler.LambdaLR
- _partial_: true
- lr_lambda:
- _target_: fish_speech.scheduler.get_cosine_schedule_with_warmup_lr_lambda
- _partial_: true
- num_warmup_steps: 1000
- num_training_steps: ${trainer.max_steps}
- final_lr_ratio: 0.05
-
-callbacks:
- grad_norm_monitor:
- sub_module:
- - mel_encoder
- - vq_encoder
- - speaker_encoder
- - decoder
diff --git a/fish_speech/configs/vqgan_single.yaml b/fish_speech/configs/vqgan_single.yaml
deleted file mode 100644
index 7e22736..0000000
--- a/fish_speech/configs/vqgan_single.yaml
+++ /dev/null
@@ -1,134 +0,0 @@
-defaults:
- - base
- - _self_
-
-project: vqgan_single
-
-# Lightning Trainer
-trainer:
- accelerator: gpu
- devices: 4
- strategy: ddp_find_unused_parameters_true
- precision: 32
- max_steps: 1_000_000
- val_check_interval: 5000
-
-sample_rate: 22050
-hop_length: 256
-num_mels: 80
-n_fft: 1024
-win_length: 1024
-segment_size: 256
-
-# Dataset Configuration
-train_dataset:
- _target_: fish_speech.datasets.vqgan.VQGANDataset
- filelist: data/filelist.split.train
- sample_rate: ${sample_rate}
- hop_length: ${hop_length}
- slice_frames: ${segment_size}
-
-val_dataset:
- _target_: fish_speech.datasets.vqgan.VQGANDataset
- filelist: data/filelist.split.valid
- sample_rate: ${sample_rate}
- hop_length: ${hop_length}
-
-data:
- _target_: fish_speech.datasets.vqgan.VQGANDataModule
- train_dataset: ${train_dataset}
- val_dataset: ${val_dataset}
- num_workers: 4
- batch_size: 32
- val_batch_size: 4
-
-# Model Configuration
-model:
- _target_: fish_speech.models.vqgan.VQGAN
- sample_rate: ${sample_rate}
- hop_length: ${hop_length}
- segment_size: 8192
- freeze_hifigan: false
-
- downsample:
- _target_: fish_speech.models.vqgan.modules.encoders.ConvDownSampler
- dims: ["${num_mels}", 512, 256]
- kernel_sizes: [3, 3]
- strides: [2, 2]
-
- mel_encoder:
- _target_: fish_speech.models.vqgan.modules.modules.WN
- hidden_channels: 256
- kernel_size: 3
- dilation_rate: 2
- n_layers: 6
-
- vq_encoder:
- _target_: fish_speech.models.vqgan.modules.encoders.VQEncoder
- in_channels: 256
- vq_channels: 256
- codebook_size: 4096
- codebook_groups: 1
- downsample: 1
-
- speaker_encoder:
- _target_: fish_speech.models.vqgan.modules.encoders.SpeakerEncoder
- in_channels: ${num_mels}
- hidden_channels: 256
- out_channels: 512
- num_layers: 6
-
- decoder:
- _target_: fish_speech.models.vqgan.modules.modules.WN
- hidden_channels: 256
- out_channels: ${num_mels}
- kernel_size: 3
- dilation_rate: 2
- n_layers: 6
- gin_channels: 512
-
- generator:
- _target_: fish_speech.models.vqgan.modules.decoder.Generator
- initial_channel: ${num_mels}
- resblock: "1"
- resblock_kernel_sizes: [3, 7, 11]
- resblock_dilation_sizes: [[1, 3, 5], [1, 3, 5], [1, 3, 5]]
- upsample_rates: [8, 8, 2, 2]
- upsample_initial_channel: 512
- upsample_kernel_sizes: [16, 16, 4, 4]
- ckpt_path: "checkpoints/hifigan-v1-universal-22050/g_02500000"
-
- discriminator:
- _target_: fish_speech.models.vqgan.modules.discriminator.EnsembleDiscriminator
- ckpt_path: checkpoints/hifigan-v1-universal-22050/do_02500000
-
- mel_transform:
- _target_: fish_speech.models.vqgan.spectrogram.LogMelSpectrogram
- sample_rate: ${sample_rate}
- n_fft: ${n_fft}
- hop_length: ${hop_length}
- win_length: ${win_length}
- n_mels: ${num_mels}
- f_min: 0
- f_max: 8000
-
- optimizer:
- _target_: torch.optim.AdamW
- _partial_: true
- lr: 2e-4
- betas: [0.8, 0.99]
- eps: 1e-5
-
- lr_scheduler:
- _target_: torch.optim.lr_scheduler.ExponentialLR
- _partial_: true
- gamma: 0.999999 # Estimated base on LibriTTS dataset
-
-callbacks:
- grad_norm_monitor:
- sub_module:
- - generator
- - discriminator
- - mel_encoder
- - vq_encoder
- - decoder
diff --git a/fish_speech/configs/vqgan_single_2x.yaml b/fish_speech/configs/vqgan_single_2x.yaml
deleted file mode 100644
index 36d59b4..0000000
--- a/fish_speech/configs/vqgan_single_2x.yaml
+++ /dev/null
@@ -1,134 +0,0 @@
-defaults:
- - base
- - _self_
-
-project: vqgan_single_2x
-
-# Lightning Trainer
-trainer:
- accelerator: gpu
- devices: 4
- strategy: ddp_find_unused_parameters_true
- precision: 32
- max_steps: 1_000_000
- val_check_interval: 5000
-
-sample_rate: 22050
-hop_length: 256
-num_mels: 80
-n_fft: 1024
-win_length: 1024
-segment_size: 256
-
-# Dataset Configuration
-train_dataset:
- _target_: fish_speech.datasets.vqgan.VQGANDataset
- filelist: data/filelist.split.train
- sample_rate: ${sample_rate}
- hop_length: ${hop_length}
- slice_frames: ${segment_size}
-
-val_dataset:
- _target_: fish_speech.datasets.vqgan.VQGANDataset
- filelist: data/filelist.split.valid
- sample_rate: ${sample_rate}
- hop_length: ${hop_length}
-
-data:
- _target_: fish_speech.datasets.vqgan.VQGANDataModule
- train_dataset: ${train_dataset}
- val_dataset: ${val_dataset}
- num_workers: 4
- batch_size: 32
- val_batch_size: 4
-
-# Model Configuration
-model:
- _target_: fish_speech.models.vqgan.VQGAN
- sample_rate: ${sample_rate}
- hop_length: ${hop_length}
- segment_size: 8192
- freeze_hifigan: false
-
- downsample:
- _target_: fish_speech.models.vqgan.modules.encoders.ConvDownSampler
- dims: ["${num_mels}", 512, 384]
- kernel_sizes: [3, 3]
- strides: [2, 2]
-
- mel_encoder:
- _target_: fish_speech.models.vqgan.modules.modules.WN
- hidden_channels: 384
- kernel_size: 3
- dilation_rate: 2
- n_layers: 12
-
- vq_encoder:
- _target_: fish_speech.models.vqgan.modules.encoders.VQEncoder
- in_channels: 384
- vq_channels: 384
- codebook_size: 4096
- codebook_groups: 1
- downsample: 1
-
- speaker_encoder:
- _target_: fish_speech.models.vqgan.modules.encoders.SpeakerEncoder
- in_channels: ${num_mels}
- hidden_channels: 384
- out_channels: 512
- num_layers: 12
-
- decoder:
- _target_: fish_speech.models.vqgan.modules.modules.WN
- hidden_channels: 384
- out_channels: ${num_mels}
- kernel_size: 3
- dilation_rate: 2
- n_layers: 12
- gin_channels: 512
-
- generator:
- _target_: fish_speech.models.vqgan.modules.decoder.Generator
- initial_channel: ${num_mels}
- resblock: "1"
- resblock_kernel_sizes: [3, 7, 11]
- resblock_dilation_sizes: [[1, 3, 5], [1, 3, 5], [1, 3, 5]]
- upsample_rates: [8, 8, 2, 2]
- upsample_initial_channel: 512
- upsample_kernel_sizes: [16, 16, 4, 4]
- ckpt_path: "checkpoints/hifigan-v1-universal-22050/g_02500000"
-
- discriminator:
- _target_: fish_speech.models.vqgan.modules.discriminator.EnsembleDiscriminator
- ckpt_path: checkpoints/hifigan-v1-universal-22050/do_02500000
-
- mel_transform:
- _target_: fish_speech.models.vqgan.spectrogram.LogMelSpectrogram
- sample_rate: ${sample_rate}
- n_fft: ${n_fft}
- hop_length: ${hop_length}
- win_length: ${win_length}
- n_mels: ${num_mels}
- f_min: 0
- f_max: 8000
-
- optimizer:
- _target_: torch.optim.AdamW
- _partial_: true
- lr: 2e-4
- betas: [0.8, 0.99]
- eps: 1e-5
-
- lr_scheduler:
- _target_: torch.optim.lr_scheduler.ExponentialLR
- _partial_: true
- gamma: 0.999999 # Estimated base on LibriTTS dataset
-
-callbacks:
- grad_norm_monitor:
- sub_module:
- - generator
- - discriminator
- - mel_encoder
- - vq_encoder
- - decoder