Skip to content

Open-Sora: Democratizing Efficient Video Production for All

Python

added warmup lr scheduler (#121)

Frank Leeea96cdb5e4ef
diff --git a/opensora/utils/lr_scheduler.py b/opensora/utils/lr_scheduler.py
new file mode 100644
index 0000000..e0f75f5
--- /dev/null
+++ b/opensora/utils/lr_scheduler.py
@@ -0,0 +1,22 @@
+from torch.optim.lr_scheduler import _LRScheduler
+
+
+class LinearWarmupLR(_LRScheduler):
+ """Linearly warmup learning rate and then linearly decay.
+
+ Args:
+ optimizer (:class:`torch.optim.Optimizer`): Wrapped optimizer.
+ warmup_steps (int, optional): Number of warmup steps, defaults to 0
+ last_step (int, optional): The index of last step, defaults to -1. When last_step=-1,
+ the schedule is started from the beginning or When last_step=-1, sets initial lr as lr.
+ """
+
+ def __init__(self, optimizer, warmup_steps: int = 0, last_epoch: int = -1):
+ self.warmup_steps = warmup_steps
+ super().__init__(optimizer, last_epoch=last_epoch)
+
+ def get_lr(self):
+ if self.last_epoch < self.warmup_steps:
+ return [(self.last_epoch + 1) / (self.warmup_steps + 1) * lr for lr in self.base_lrs]
+ else:
+ return self.base_lrs
diff --git a/scripts/train.py b/scripts/train.py
index 08681dc..73d1fbd 100644
--- a/scripts/train.py
+++ b/scripts/train.py
@@ -17,6 +17,7 @@ from opensora.acceleration.checkpoint import set_grad_checkpoint
from opensora.acceleration.parallel_states import get_data_parallel_group
from opensora.datasets.dataloader import prepare_dataloader
from opensora.registry import DATASETS, MODELS, SCHEDULERS, build_module
+from opensora.utils.lr_scheduler import LinearWarmupLR
from opensora.utils.ckpt_utils import load, model_gathering, model_sharding, record_model_param_shape, save
from opensora.utils.config_utils import define_experiment_workspace, parse_configs, save_training_config
from opensora.utils.misc import (
@@ -169,7 +170,13 @@ def main():
weight_decay=cfg.get("weight_decay", 0),
eps=cfg.get("adam_eps", 1e-8),
)
- lr_scheduler = None
+
+ warmup_steps = cfg.get("warmup_steps", None)
+
+ if warmup_steps is None:
+ lr_scheduler = None
+ else:
+ lr_scheduler = LinearWarmupLR(optimizer, warmup_steps=cfg.get("warmup_steps"))
# == additional preparation ==
if cfg.get("grad_checkpoint", False):
@@ -288,6 +295,10 @@ def main():
booster.backward(loss=loss, optimizer=optimizer)
optimizer.step()
optimizer.zero_grad()
+
+ # update learning rate
+ if lr_scheduler is not None:
+ lr_scheduler.step()
coordinator.block_all()
timer_list.append(backward_t)
@@ -323,6 +334,7 @@ def main():
"loss": loss.item(),
"avg_loss": avg_loss,
"acc_step": acc_step,
+ "lr": optimizer.param_groups[0]["lr"],
"move_data_time": move_data_t.elapsed_time,
"encode_time": encode_t.elapsed_time,
"mask_time": mask_t.elapsed_time,
diff --git a/tests/test_lr_scheduler.py b/tests/test_lr_scheduler.py
new file mode 100644
index 0000000..23b5127
--- /dev/null
+++ b/tests/test_lr_scheduler.py
@@ -0,0 +1,30 @@
+import torch
+from torch.optim import Adam
+from torchvision.models import resnet50
+
+from opensora.utils.lr_scheduler import LinearWarmupLR
+
+
+def test_lr_scheduler():
+ model = resnet50().cuda()
+ optimizer = Adam(model.parameters(), lr=0.01)
+ scheduler = LinearWarmupLR(optimizer, warmup_steps=10)
+ current_lr = scheduler.get_lr()[0]
+ data = torch.rand(128, 3, 224, 224).cuda()
+
+ for i in range(100):
+ out = model(data)
+ out.mean().backward()
+
+ optimizer.step()
+ scheduler.step()
+
+ if i >= 10:
+ assert scheduler.get_lr()[0] == 0.01
+ else:
+ assert scheduler.get_lr()[0] > current_lr, f"{scheduler.get_lr()[0]} <= {current_lr}"
+ current_lr = scheduler.get_lr()[0]
+
+
+if __name__ == "__main__":
+ test_lr_scheduler()