Open-Sora: Democratizing Efficient Video Production for All
Python
fixed race condition in exp workspace (#91)
Frank Leefe19568a4972
diff --git a/opensora/utils/config_utils.py b/opensora/utils/config_utils.pyindex ecb9438..af02ec3 100644--- a/opensora/utils/config_utils.py+++ b/opensora/utils/config_utils.py@@ -149,7 +149,7 @@ def parse_configs(training=False):return cfg-def create_experiment_workspace(cfg, get_last_workspace=False):+def define_experiment_workspace(cfg, get_last_workspace=False):"""This function creates a folder for experiment tracking.@@ -169,7 +169,6 @@ def create_experiment_workspace(cfg, get_last_workspace=False):model_name = cfg.model["type"].replace("/", "-")exp_name = f"{experiment_index:03d}-{model_name}"exp_dir = f"{cfg.outputs}/{exp_name}"- os.makedirs(exp_dir, exist_ok=True)return exp_name, exp_dirdiff --git a/scripts/train.py b/scripts/train.pyindex 43137b2..93fc3b6 100644--- a/scripts/train.py+++ b/scripts/train.py@@ -1,10 +1,10 @@+import osfrom copy import deepcopyfrom datetime import timedeltafrom pprint import pprintimport torchimport torch.distributed as dist-import wandbfrom colossalai.booster import Boosterfrom colossalai.booster.plugin import LowLevelZeroPluginfrom colossalai.cluster import DistCoordinator@@ -12,6 +12,7 @@ from colossalai.nn.optimizer import HybridAdamfrom colossalai.utils import get_current_device, set_seedfrom tqdm import tqdm+import wandbfrom opensora.acceleration.checkpoint import set_grad_checkpointfrom opensora.acceleration.parallel_states import (get_data_parallel_group,@@ -23,8 +24,8 @@ from opensora.datasets import prepare_dataloader, prepare_variable_dataloaderfrom opensora.registry import DATASETS, MODELS, SCHEDULERS, build_modulefrom opensora.utils.ckpt_utils import create_logger, load, model_sharding, record_model_param_shape, savefrom opensora.utils.config_utils import (- create_experiment_workspace,create_tensorboard_writer,+ define_experiment_workspace,parse_configs,save_training_config,)@@ -37,8 +38,6 @@ def main():# 1. args & cfg# ======================================================cfg = parse_configs(training=True)- exp_name, exp_dir = create_experiment_workspace(cfg)- save_training_config(cfg._cfg_dict, exp_dir)# ======================================================# 2. runtime variables & colossalai launch@@ -55,7 +54,14 @@ def main():device = get_current_device()dtype = to_torch_dtype(cfg.dtype)- # 2.2. init logger, tensorboard & wandb+ # 2.2. init exp_dir, logger, tensorboard & wandb+ exp_name, exp_dir = define_experiment_workspace(cfg)+ coordinator.block_all()+ if coordinator.is_master():+ os.makedirs(exp_dir, exist_ok=True)+ save_training_config(cfg._cfg_dict, exp_dir)+ coordinator.block_all()+if not coordinator.is_master():logger = create_logger(None)else:diff --git a/scripts/train_vae.py b/scripts/train_vae.pyindex f02bd3d..5398e04 100644--- a/scripts/train_vae.py+++ b/scripts/train_vae.py@@ -5,7 +5,6 @@ from pprint import pprintimport torchimport torch.distributed as dist-import wandbfrom colossalai.booster import Boosterfrom colossalai.booster.plugin import LowLevelZeroPluginfrom colossalai.cluster import DistCoordinator@@ -14,6 +13,7 @@ from colossalai.utils import get_current_device, set_seedfrom einops import rearrangefrom tqdm import tqdm+import wandbfrom opensora.acceleration.checkpoint import set_grad_checkpointfrom opensora.acceleration.parallel_states import get_data_parallel_group, set_data_parallel_groupfrom opensora.datasets import prepare_dataloader@@ -21,8 +21,8 @@ from opensora.models.vae.losses import AdversarialLoss, DiscriminatorLoss, VAELofrom opensora.registry import DATASETS, MODELS, build_modulefrom opensora.utils.ckpt_utils import create_logger, load_json, save_jsonfrom opensora.utils.config_utils import (- create_experiment_workspace,create_tensorboard_writer,+ define_experiment_workspace,parse_configs,save_training_config,)@@ -34,8 +34,6 @@ def main():# 1. args & cfg# ======================================================cfg = parse_configs(training=True)- exp_name, exp_dir = create_experiment_workspace(cfg)- save_training_config(cfg._cfg_dict, exp_dir)# ======================================================# 2. runtime variables & colossalai launch@@ -52,7 +50,14 @@ def main():device = get_current_device()dtype = to_torch_dtype(cfg.dtype)- # 2.2. init logger, tensorboard & wandb+ # 2.2. init exp_dir, logger, tensorboard & wandb+ exp_name, exp_dir = define_experiment_workspace(cfg)+ coordinator.block_all()+ if coordinator.is_master():+ os.makedirs(exp_dir, exist_ok=True)+ save_training_config(cfg._cfg_dict, exp_dir)+ coordinator.block_all()+if not coordinator.is_master():logger = create_logger(None)else: