SOTA Open Source TTS
Optimize data loading & gradient accumulate
Lengyue900fc9dec986
diff --git a/speech_lm/configs/pretrain.yaml b/speech_lm/configs/pretrain.yamlindex 6f39b48..2dfcd88 100644--- a/speech_lm/configs/pretrain.yaml+++ b/speech_lm/configs/pretrain.yaml@@ -48,10 +48,7 @@ dataloader:batch_size: ${schedule.micro_batch_size}num_workers: 4collate_fn:- _target_: transformers.DataCollatorWithPadding- tokenizer: ${tokenizer}- max_length: ${schedule.max_length}- padding: max_length+ _target_: transformers.DefaultDataCollatoroptimizer:_target_: torch.optim.AdamWdiff --git a/speech_lm/train.py b/speech_lm/train.pyindex 16a3091..1683459 100644--- a/speech_lm/train.py+++ b/speech_lm/train.py@@ -1,4 +1,3 @@-import loggingfrom pathlib import Pathimport hydra@@ -40,9 +39,11 @@ def train(while global_step < cfg.schedule.max_steps:for batch in dataloader:+ # Accumulate gradientsis_accumulating = (accumulate_steps % cfg.schedule.gradient_accumulation_steps != 0)+ accumulate_steps += 1# Train one stepwith fabric.no_backward_sync(model, enabled=is_accumulating):@@ -50,7 +51,6 @@ def train(fabric.backward(loss)if is_accumulating:- accumulate_steps += 1continue# Perform gradient clipping