Open-Sora: Democratizing Efficient Video Production for All
Python
Feature/timeout (#50)
Frank Leedb9445464505
diff --git a/.gitignore b/.gitignoreindex 55daee5..8797ae7 100644--- a/.gitignore+++ b/.gitignore@@ -177,3 +177,4 @@ pretrained_models/# Secret fileshostfilegradio_cached_examples/+wandb/diff --git a/scripts/train.py b/scripts/train.pyindex d25c4a4..b097573 100644--- a/scripts/train.py+++ b/scripts/train.py@@ -1,7 +1,7 @@from copy import deepcopy+from datetime import timedeltafrom pprint import pprint-import colossalaiimport torchimport torch.distributed as distimport wandb@@ -9,7 +9,7 @@ from colossalai.booster import Boosterfrom colossalai.booster.plugin import LowLevelZeroPluginfrom colossalai.cluster import DistCoordinatorfrom colossalai.nn.optimizer import HybridAdam-from colossalai.utils import get_current_device+from colossalai.utils import get_current_device, set_seedfrom tqdm import tqdmfrom opensora.acceleration.checkpoint import set_grad_checkpoint@@ -47,7 +47,10 @@ def main():assert cfg.dtype in ["fp16", "bf16"], f"Unknown mixed precision {cfg.dtype}"# 2.1. colossalai init distributed training- colossalai.launch_from_torch({})+ # we set a very large timeout to avoid some processes exit early+ dist.init_process_group(backend="nccl", timeout=timedelta(hours=24))+ torch.cuda.set_device(dist.get_rank() % torch.cuda.device_count())+ set_seed(1024)coordinator = DistCoordinator()device = get_current_device()dtype = to_torch_dtype(cfg.dtype)