Open-Sora: Democratizing Efficient Video Production for All
Python
added warmup lr scheduler (#121)
Frank Leeea96cdb5e4ef
diff --git a/opensora/utils/lr_scheduler.py b/opensora/utils/lr_scheduler.pynew file mode 100644index 0000000..e0f75f5--- /dev/null+++ b/opensora/utils/lr_scheduler.py@@ -0,0 +1,22 @@+from torch.optim.lr_scheduler import _LRScheduler+++class LinearWarmupLR(_LRScheduler):+ """Linearly warmup learning rate and then linearly decay.++ Args:+ optimizer (:class:`torch.optim.Optimizer`): Wrapped optimizer.+ warmup_steps (int, optional): Number of warmup steps, defaults to 0+ last_step (int, optional): The index of last step, defaults to -1. When last_step=-1,+ the schedule is started from the beginning or When last_step=-1, sets initial lr as lr.+ """++ def __init__(self, optimizer, warmup_steps: int = 0, last_epoch: int = -1):+ self.warmup_steps = warmup_steps+ super().__init__(optimizer, last_epoch=last_epoch)++ def get_lr(self):+ if self.last_epoch < self.warmup_steps:+ return [(self.last_epoch + 1) / (self.warmup_steps + 1) * lr for lr in self.base_lrs]+ else:+ return self.base_lrsdiff --git a/scripts/train.py b/scripts/train.pyindex 08681dc..73d1fbd 100644--- a/scripts/train.py+++ b/scripts/train.py@@ -17,6 +17,7 @@ from opensora.acceleration.checkpoint import set_grad_checkpointfrom opensora.acceleration.parallel_states import get_data_parallel_groupfrom opensora.datasets.dataloader import prepare_dataloaderfrom opensora.registry import DATASETS, MODELS, SCHEDULERS, build_module+from opensora.utils.lr_scheduler import LinearWarmupLRfrom opensora.utils.ckpt_utils import load, model_gathering, model_sharding, record_model_param_shape, savefrom opensora.utils.config_utils import define_experiment_workspace, parse_configs, save_training_configfrom opensora.utils.misc import (@@ -169,7 +170,13 @@ def main():weight_decay=cfg.get("weight_decay", 0),eps=cfg.get("adam_eps", 1e-8),)- lr_scheduler = None++ warmup_steps = cfg.get("warmup_steps", None)++ if warmup_steps is None:+ lr_scheduler = None+ else:+ lr_scheduler = LinearWarmupLR(optimizer, warmup_steps=cfg.get("warmup_steps"))# == additional preparation ==if cfg.get("grad_checkpoint", False):@@ -288,6 +295,10 @@ def main():booster.backward(loss=loss, optimizer=optimizer)optimizer.step()optimizer.zero_grad()++ # update learning rate+ if lr_scheduler is not None:+ lr_scheduler.step()coordinator.block_all()timer_list.append(backward_t)@@ -323,6 +334,7 @@ def main():"loss": loss.item(),"avg_loss": avg_loss,"acc_step": acc_step,+ "lr": optimizer.param_groups[0]["lr"],"move_data_time": move_data_t.elapsed_time,"encode_time": encode_t.elapsed_time,"mask_time": mask_t.elapsed_time,diff --git a/tests/test_lr_scheduler.py b/tests/test_lr_scheduler.pynew file mode 100644index 0000000..23b5127--- /dev/null+++ b/tests/test_lr_scheduler.py@@ -0,0 +1,30 @@+import torch+from torch.optim import Adam+from torchvision.models import resnet50++from opensora.utils.lr_scheduler import LinearWarmupLR+++def test_lr_scheduler():+ model = resnet50().cuda()+ optimizer = Adam(model.parameters(), lr=0.01)+ scheduler = LinearWarmupLR(optimizer, warmup_steps=10)+ current_lr = scheduler.get_lr()[0]+ data = torch.rand(128, 3, 224, 224).cuda()++ for i in range(100):+ out = model(data)+ out.mean().backward()++ optimizer.step()+ scheduler.step()++ if i >= 10:+ assert scheduler.get_lr()[0] == 0.01+ else:+ assert scheduler.get_lr()[0] > current_lr, f"{scheduler.get_lr()[0]} <= {current_lr}"+ current_lr = scheduler.get_lr()[0]+++if __name__ == "__main__":+ test_lr_scheduler()