SOTA Open Source TTS
Fix logger in config
Lengyue026d419e9af7
diff --git a/speech_lm/configs/pretrain.yaml b/speech_lm/configs/pretrain.yamlindex 15e3794..6f39b48 100644--- a/speech_lm/configs/pretrain.yaml+++ b/speech_lm/configs/pretrain.yaml@@ -13,10 +13,10 @@ trainer:devices: autoprecision: bf16-mixedloggers:- - _target_: pytorch_lightning.loggers.TensorBoardLogger- save_dir: ${paths.run_dir}- name: tensorboard- version: null+ _target_: pytorch_lightning.loggers.TensorBoardLogger+ save_dir: ${paths.run_dir}+ name: tensorboard+ version: nullmodel:_target_: transformers.AutoModelForCausalLM.from_pretrained@@ -32,9 +32,12 @@ tokenizer:# 3e12 / 1024 / 512 / 8 = 715255schedule:max_length: 1024- batch_size: 16+ batch_size: 512+ micro_batch_size: 2max_steps: 715255- save_every: 2000+ save_interval: 2000+ gradient_accumulation_steps: "${eval: ${schedule.batch_size} // ${schedule.micro_batch_size}}"+ clip_grad_norm: 1.0dataloader:_target_: torch.utils.data.DataLoader@@ -42,7 +45,7 @@ dataloader:_target_: speech_lm.dataset.build_datasettokenizer: ${tokenizer}max_length: ${schedule.max_length}- batch_size: ${schedule.batch_size}+ batch_size: ${schedule.micro_batch_size}num_workers: 4collate_fn:_target_: transformers.DataCollatorWithPaddingdiff --git a/speech_lm/train.py b/speech_lm/train.pyindex 1f881be..16a3091 100644--- a/speech_lm/train.py+++ b/speech_lm/train.py@@ -35,25 +35,47 @@ def train():bar = tqdm(total=cfg.schedule.max_steps, desc="Training")bar.update(global_step)+ accumulate_steps = 0+ optimizer.zero_grad()while global_step < cfg.schedule.max_steps:for batch in dataloader:- # Train loop- optimizer.zero_grad()- loss = model(**batch).loss- fabric.backward(loss)+ is_accumulating = (+ accumulate_steps % cfg.schedule.gradient_accumulation_steps != 0+ )++ # Train one step+ with fabric.no_backward_sync(model, enabled=is_accumulating):+ loss = model(**batch).loss+ fabric.backward(loss)++ if is_accumulating:+ accumulate_steps += 1+ continue++ # Perform gradient clipping+ grad_norm = fabric.clip_gradients(+ model, optimizer, max_norm=cfg.schedule.clip_grad_norm, norm_type=2.0+ )++ # Updateoptimizer.step()+ optimizer.zero_grad()scheduler.step()- fabric.log_dict({- "train/loss": loss,- "train/lr": optimizer.param_groups[0]["lr"],- }, step=global_step)+ fabric.log_dict(+ {+ "train/loss": loss,+ "train/lr": optimizer.param_groups[0]["lr"],+ "train/grad_norm": grad_norm,+ },+ step=global_step,+ )global_step += 1bar.update(1)- if global_step % cfg.schedule.save_steps == 0:+ if global_step % cfg.schedule.save_interval == 0:fabric.save(Path(cfg.paths.checkpoint_dir) / f"step_{global_step}.ckpt",{