Open-Sora: Democratizing Efficient Video Production for All
Python
update training script
zhengzangwe9db7b3af134
diff --git a/opensora/utils/ckpt_utils.py b/opensora/utils/ckpt_utils.pyindex 4b97c08..c338e84 100644--- a/opensora/utils/ckpt_utils.py+++ b/opensora/utils/ckpt_utils.py@@ -9,13 +9,10 @@ import torch.distributed as distimport torch.nn as nnfrom colossalai.booster import Boosterfrom colossalai.checkpoint_io import GeneralCheckpointIO-from colossalai.cluster import DistCoordinatorfrom torch.optim import Optimizerfrom torch.optim.lr_scheduler import _LRSchedulerfrom torchvision.datasets.utils import download_url-from opensora.datasets.sampler import VariableVideoBatchSampler-hf_endpoint = os.environ.get("HF_ENDPOINT")if hf_endpoint is None:hf_endpoint = "https://huggingface.co"@@ -160,6 +157,18 @@ def model_sharding(model: torch.nn.Module):param.data = splited_params+def model_gathering(model: torch.nn.Module, model_shape_dict: dict):+ global_rank = dist.get_rank()+ global_size = dist.get_world_size()+ for name, param in model.named_parameters():+ all_params = [torch.empty_like(param.data) for _ in range(global_size)]+ dist.all_gather(all_params, param.data, group=dist.group.WORLD)+ if int(global_rank) == 0:+ all_params = torch.cat(all_params)+ param.data = remove_padding(all_params, model_shape_dict[name]).view(model_shape_dict[name])+ dist.barrier()++def load_json(file_path: str):with open(file_path, "r") as f:return json.load(f)@@ -174,18 +183,6 @@ def remove_padding(tensor: torch.Tensor, original_shape: Tuple) -> torch.Tensor:return tensor[: functools.reduce(operator.mul, original_shape)]-def model_gathering(model: torch.nn.Module, model_shape_dict: dict):- global_rank = dist.get_rank()- global_size = dist.get_world_size()- for name, param in model.named_parameters():- all_params = [torch.empty_like(param.data) for _ in range(global_size)]- dist.all_gather(all_params, param.data, group=dist.group.WORLD)- if int(global_rank) == 0:- all_params = torch.cat(all_params)- param.data = remove_padding(all_params, model_shape_dict[name]).view(model_shape_dict[name])- dist.barrier()--def record_model_param_shape(model: torch.nn.Module) -> dict:param_shape = {}for name, param in model.named_parameters():@@ -193,93 +190,97 @@ def record_model_param_shape(model: torch.nn.Module) -> dict:return param_shape+def load_checkpoint(model, ckpt_path, save_as_pt=False, model_name="model"):+ if ckpt_path.endswith(".pt") or ckpt_path.endswith(".pth"):+ state_dict = find_model(ckpt_path, model=model)+ missing_keys, unexpected_keys = model.load_state_dict(state_dict, strict=False)+ print(f"Missing keys: {missing_keys}")+ print(f"Unexpected keys: {unexpected_keys}")+ elif os.path.isdir(ckpt_path):+ load_from_sharded_state_dict(model, ckpt_path, model_name)+ print(f"Model checkpoint loaded from {ckpt_path}")+ if save_as_pt:+ save_path = os.path.join(ckpt_path, model_name + "_ckpt.pt")+ torch.save(model.state_dict(), save_path)+ print(f"Model checkpoint saved to {save_path}")+ else:+ raise ValueError(f"Invalid checkpoint path: {ckpt_path}")+++# save and load for training++def save(booster: Booster,- model: nn.Module,- ema: nn.Module,- optimizer: Optimizer,- lr_scheduler: _LRScheduler,- epoch: int,- step: int,- global_step: int,- batch_size: int,- coordinator: DistCoordinator,save_dir: str,- shape_dict: dict,+ model: nn.Module = None,+ ema: nn.Module = None,+ optimizer: Optimizer = None,+ lr_scheduler: _LRScheduler = None,sampler=None,+ epoch: int = None,+ step: int = None,+ global_step: int = None,+ batch_size: int = None,):save_dir = os.path.join(save_dir, f"epoch{epoch}-global_step{global_step}")os.makedirs(os.path.join(save_dir, "model"), exist_ok=True)- booster.save_model(model, os.path.join(save_dir, "model"), shard=True)- # ema is not boosted, so we don't need to use booster.save_model- model_gathering(ema, shape_dict)- global_rank = dist.get_rank()- if int(global_rank) == 0:- torch.save(ema.state_dict(), os.path.join(save_dir, "ema.pt"))- model_sharding(ema)-- booster.save_optimizer(optimizer, os.path.join(save_dir, "optimizer"), shard=True, size_per_shard=4096)+ if model is not None:+ booster.save_model(model, os.path.join(save_dir, "model"), shard=True)+ if optimizer is not None:+ booster.save_optimizer(optimizer, os.path.join(save_dir, "optimizer"), shard=True, size_per_shard=4096)if lr_scheduler is not None:booster.save_lr_scheduler(lr_scheduler, os.path.join(save_dir, "lr_scheduler"))- sampler_start_idx = step * batch_size if batch_size is not None else None- running_states = {- "epoch": epoch,- "step": step,- "global_step": global_step,- "sample_start_index": sampler_start_idx,- }- if coordinator.is_master():+ if dist.get_rank() == 0:+ sampler_start_idx = step * batch_size if batch_size is not None else None+ running_states = {+ "epoch": epoch,+ "step": step,+ "global_step": global_step,+ "sample_start_index": sampler_start_idx,+ }save_json(running_states, os.path.join(save_dir, "running_states.json"))++ if ema is not None:+ torch.save(ema.state_dict(), os.path.join(save_dir, "ema.pt"))+if sampler is not None:- if isinstance(sampler, VariableVideoBatchSampler):- torch.save(sampler.state_dict(step), os.path.join(save_dir, "sampler"))- else:- torch.save(sampler.state_dict(), os.path.join(save_dir, "sampler"))+ # only for VariableVideoBatchSampler+ torch.save(sampler.state_dict(step), os.path.join(save_dir, "sampler"))dist.barrier()def load(booster: Booster,- model: nn.Module,- ema: nn.Module,- optimizer: Optimizer,- lr_scheduler: _LRScheduler,load_dir: str,+ model: nn.Module = None,+ ema: nn.Module = None,+ optimizer: Optimizer = None,+ lr_scheduler: _LRScheduler = None,sampler=None,) -> Tuple[int, int, int]:- booster.load_model(model, os.path.join(load_dir, "model"))- # ema is not boosted, so we don't use booster.load_model- ema.load_state_dict(- torch.load(os.path.join(load_dir, "ema.pt"), map_location=torch.device("cpu")),- strict=False,- )- booster.load_optimizer(optimizer, os.path.join(load_dir, "optimizer"))+ assert os.path.exists(load_dir), f"Checkpoint directory {load_dir} does not exist"+ assert os.path.exists(os.path.join(load_dir, "running_states.json")), "running_states.json does not exist"+ running_states = load_json(os.path.join(load_dir, "running_states.json"))+ if model is not None:+ booster.load_model(model, os.path.join(load_dir, "model"))+ if ema is not None:+ # ema is not boosted, so we don't use booster.load_model+ ema.load_state_dict(+ torch.load(os.path.join(load_dir, "ema.pt"), map_location=torch.device("cpu")),+ strict=False,+ )+ if optimizer is not None:+ booster.load_optimizer(optimizer, os.path.join(load_dir, "optimizer"))if lr_scheduler is not None:booster.load_lr_scheduler(lr_scheduler, os.path.join(load_dir, "lr_scheduler"))- running_states = load_json(os.path.join(load_dir, "running_states.json"))if sampler is not None:sampler.load_state_dict(torch.load(os.path.join(load_dir, "sampler")))dist.barrier()+return (running_states["epoch"],running_states["step"],running_states["sample_start_index"],)---def load_checkpoint(model, ckpt_path, save_as_pt=False, model_name="model"):- if ckpt_path.endswith(".pt") or ckpt_path.endswith(".pth"):- state_dict = find_model(ckpt_path, model=model)- missing_keys, unexpected_keys = model.load_state_dict(state_dict, strict=False)- print(f"Missing keys: {missing_keys}")- print(f"Unexpected keys: {unexpected_keys}")- elif os.path.isdir(ckpt_path):- load_from_sharded_state_dict(model, ckpt_path, model_name)- print(f"Model checkpoint loaded from {ckpt_path}")- if save_as_pt:- save_path = os.path.join(ckpt_path, model_name + "_ckpt.pt")- torch.save(model.state_dict(), save_path)- print(f"Model checkpoint saved to {save_path}")- else:- raise ValueError(f"Invalid checkpoint path: {ckpt_path}")diff --git a/scripts/train.py b/scripts/train.pyindex 2795c9c..68c7d33 100644--- a/scripts/train.py+++ b/scripts/train.py@@ -16,7 +16,7 @@ from opensora.acceleration.checkpoint import set_grad_checkpointfrom opensora.acceleration.parallel_states import get_data_parallel_groupfrom opensora.datasets import prepare_dataloader, prepare_variable_dataloaderfrom opensora.registry import DATASETS, MODELS, SCHEDULERS, build_module-from opensora.utils.ckpt_utils import load, model_sharding, record_model_param_shape, save+from opensora.utils.ckpt_utils import load, model_gathering, model_sharding, record_model_param_shape, savefrom opensora.utils.config_utils import (create_tensorboard_writer,define_experiment_workspace,@@ -65,7 +65,7 @@ def main():tb_writer = create_tensorboard_writer(exp_dir)if cfg.get("wandb", False):- wandb.init(project="minisora", name=exp_name, config=cfg.to_dict())+ wandb.init(project="minisora", name=exp_name, config=cfg.to_dict(), dir="./outputs/wandb")# == init ColossalAI booster ==plugin = create_colossalai_plugin(@@ -86,9 +86,9 @@ def main():# == build dataloader ==dataloader_args = dict(dataset=dataset,- batch_size=cfg.batch_size,- num_workers=cfg.num_workers,- seed=cfg.seed,+ batch_size=cfg.get("batch_size", None),+ num_workers=cfg.get("num_workers", 4),+ seed=cfg.get("seed", 1024),shuffle=True,drop_last=True,pin_memory=True,@@ -127,7 +127,7 @@ def main():model.train()model_numel, model_numel_trainable = get_model_numel(model)logger.info(- "Trainable model params: %s, Total model params: %s",+ "[Diffusion] Trainable model params: %s, Total model params: %s",format_numel_str(model_numel_trainable),format_numel_str(model_numel),)@@ -139,7 +139,7 @@ def main():ema.eval()update_ema(ema, model, decay=0, sharded=False)- # == build scheduler ==+ # == setup loss function, build scheduler ==scheduler = build_module(cfg.scheduler, SCHEDULERS)# == setup optimizer ==@@ -191,11 +191,11 @@ def main():logger.info("Loading checkpoint")ret = load(booster,- model,- ema,- optimizer,- lr_scheduler,cfg.load,+ model=model,+ ema=ema,+ optimizer=optimizer,+ lr_scheduler=lr_scheduler,sampler=sampler_to_io,)if not cfg.get("start_from_scratch ", False):@@ -209,6 +209,7 @@ def main():# =======================================================# 6. training loop# =======================================================+ dist.barrier()for epoch in range(start_epoch, cfg_epochs):# == set dataloader to new epoch ==if cfg.dataset.type == DEFAULT_DATASET_NAME:@@ -265,14 +266,14 @@ def main():acc_step += 1# == logging ==- if coordinator.is_master() and (global_step + 1) % cfg.log_every == 0:+ if coordinator.is_master() and (global_step + 1) % cfg.get("log_every", 1) == 0:avg_loss = running_loss / log_step# progress barpbar.set_postfix({"loss": avg_loss, "step": step, "global_step": global_step})# tensorboardtb_writer.add_scalar("loss", loss.item(), global_step)# wandb- if cfg.wandb:+ if cfg.get("wandb", False):wandb.log({"iter": global_step,@@ -284,26 +285,27 @@ def main():step=global_step,)- running_loss = 0+ running_loss = 0.0log_step = 0# == checkpoint saving ==- if cfg.ckpt_every > 0 and (global_step + 1) % cfg.ckpt_every == 0:+ ckpt_every = cfg.get("ckpt_every", 0)+ if ckpt_every > 0 and (global_step + 1) % ckpt_every == 0:+ model_gathering(ema, ema_shape_dict)save(booster,- model,- ema,- optimizer,- lr_scheduler,- epoch,- step + 1,- global_step + 1,- cfg.batch_size,- coordinator,exp_dir,- ema_shape_dict,+ model=model,+ ema=ema,+ optimizer=optimizer,+ lr_scheduler=lr_scheduler,sampler=sampler_to_io,+ epoch=epoch,+ step=step + 1,+ global_step=global_step + 1,+ batch_size=cfg.get("batch_size", None),)+ model_sharding(ema)logger.info("Saved checkpoint at epoch %s step %s global_step %s to %s",epoch,diff --git a/scripts/train_vae.py b/scripts/train_vae.pyindex a878e62..e38ed9d 100644--- a/scripts/train_vae.py+++ b/scripts/train_vae.py@@ -5,6 +5,7 @@ from pprint import pformatimport torchimport torch.distributed as dist+import wandbfrom colossalai.booster import Boosterfrom colossalai.cluster import DistCoordinatorfrom colossalai.nn.optimizer import HybridAdam@@ -12,13 +13,12 @@ from colossalai.utils import get_current_device, set_seedfrom einops import rearrangefrom tqdm import tqdm-import wandbfrom opensora.acceleration.checkpoint import set_grad_checkpointfrom opensora.acceleration.parallel_states import get_data_parallel_groupfrom opensora.datasets import prepare_dataloaderfrom opensora.models.vae.losses import AdversarialLoss, DiscriminatorLoss, VAELossfrom opensora.registry import DATASETS, MODELS, build_module-from opensora.utils.ckpt_utils import load_json, save_json+from opensora.utils.ckpt_utils import load, savefrom opensora.utils.config_utils import (create_tensorboard_writer,define_experiment_workspace,@@ -26,16 +26,16 @@ from opensora.utils.config_utils import (save_training_config,)from opensora.utils.misc import all_reduce_mean, format_numel_str, get_model_numel, to_torch_dtype-from opensora.utils.train_utils import create_colossalai_plugin+from opensora.utils.train_utils import create_colossalai_plugin, create_loggerDEFAULT_DATASET_NAME = "VideoTextDataset"-from opensora.utils.train_utils import create_loggerdef main():# ======================================================- # 1. configs & runtime variables & colossalai launch+ # 1. configs & runtime variables# ======================================================+ # == parse configs ==cfg = parse_configs(training=True)assert torch.cuda.is_available(), "Training currently requires at least one GPU."@@ -47,7 +47,7 @@ def main():# NOTE: A very large timeout is set to avoid some processes exit earlydist.init_process_group(backend="nccl", timeout=timedelta(hours=24))torch.cuda.set_device(dist.get_rank() % torch.cuda.device_count())- set_seed(1024)+ set_seed(cfg.get("seed", 1024))coordinator = DistCoordinator()device = get_current_device()@@ -60,15 +60,14 @@ def main():coordinator.block_all()# == init logger, tensorboard & wandb ==+ logger = create_logger(exp_dir)if coordinator.is_master():- logger = create_logger(exp_dir)- logger.info(f"Experiment directory created at {exp_dir}")+ logger.info("Experiment directory created at %s", exp_dir)logger.info("Training configuration:\n %s", pformat(cfg.to_dict()))+tb_writer = create_tensorboard_writer(exp_dir)if cfg.get("wandb", False):- wandb.init(project="minisora", name=exp_name, config=cfg.to_dict())- else:- logger = create_logger(None)+ wandb.init(project="minisora", name=exp_name, config=cfg.to_dict(), dir="./outputs/wandb")# == init ColossalAI booster ==plugin = create_colossalai_plugin(@@ -82,29 +81,25 @@ def main():# ======================================================# 2. build dataset and dataloader# ======================================================+ # == build dataset ==+ assert cfg.dataset.type == DEFAULT_DATASET_NAME, "Only support VideoTextDataset for vae training"dataset = build_module(cfg.dataset, DATASETS)logger.info("Dataset contains %s samples.", len(dataset))+# == build dataloader ==dataloader_args = dict(dataset=dataset,batch_size=cfg.batch_size,- num_workers=cfg.num_workers,- seed=cfg.seed,+ num_workers=cfg.get("num_workers", 4),+ seed=cfg.get("seed", 1024),shuffle=True,drop_last=True,pin_memory=True,process_group=get_data_parallel_group(),)- if cfg.dataset.type == DEFAULT_DATASET_NAME:- dataloader = prepare_dataloader(**dataloader_args)- total_batch_size = cfg.batch_size * dist.get_world_size() // cfg.get("sp_size", 1)- logger.info("Total batch size: %s", total_batch_size)- else:- dataloader = prepare_variable_dataloader(- bucket_config=cfg.bucket_config,- num_bucket_build_workers=cfg.num_bucket_build_workers,- **dataloader_args,- )+ dataloader = prepare_dataloader(**dataloader_args)+ total_batch_size = cfg.batch_size * dist.get_world_size() // cfg.get("sp_size", 1)+ logger.info("Total batch size: %s", total_batch_size)# ======================================================# 3. build model@@ -114,17 +109,19 @@ def main():model.train()model_numel, model_numel_trainable = get_model_numel(model)logger.info(- "Trainable model params: %s, Total model params: %s",+ "[VAE] Trainable model params: %s, Total model params: %s",format_numel_str(model_numel_trainable),format_numel_str(model_numel),)+# == build discriminator model ==- if cfg.get("discriminator", False) != False:+ use_discriminator = cfg.get("discriminator", None) is not None+ if use_discriminator:discriminator = build_module(cfg.discriminator, MODELS).to(device, dtype)discriminator.train()discriminator_numel, discriminator_numel_trainable = get_model_numel(discriminator)logger.info(- "Trainable model params: %s, Total model params: %s",+ "[Discriminator] Trainable model params: %s, Total model params: %s",format_numel_str(discriminator_numel_trainable),format_numel_str(discriminator_numel),)@@ -138,7 +135,7 @@ def main():dtype=dtype,)- if cfg.get("discriminator", False) != False:+ if use_discriminator:adversarial_loss_fn = AdversarialLoss(discriminator_factor=cfg.get("discriminator_factor", 1),discriminator_start=cfg.get("discriminator_start", -1),@@ -164,7 +161,7 @@ def main():lr_scheduler = None# == setup discriminator optimizer ==- if cfg.get("discriminator", False) != False:+ if use_discriminator:disc_optimizer = HybridAdam(filter(lambda p: p.requires_grad, discriminator.parameters()),adamw_mode=True,@@ -189,7 +186,7 @@ def main():lr_scheduler=lr_scheduler,dataloader=dataloader,)- if cfg.get("discriminator", False) != False:+ if use_discriminator:discriminator, disc_optimizer, _, _, disc_lr_scheduler = booster.boost(model=discriminator,optimizer=disc_optimizer,@@ -197,47 +194,40 @@ def main():)torch.set_default_dtype(torch.float)logger.info("Boosting model for distributed training")- if cfg.dataset.type == DEFAULT_DATASET_NAME:- num_steps_per_epoch = len(dataloader)- else:- num_steps_per_epoch = dataloader.batch_sampler.get_num_batch() // dist.get_world_size()- None if cfg.get("start_from_scratch ", False) else dataloader.batch_sampler+ num_steps_per_epoch = len(dataloader)cfg_epochs = cfg.get("epochs", 1000)logger.info("Training for %s epochs with %s steps per epoch", cfg_epochs, num_steps_per_epoch)# == global variables ==start_epoch = start_step = log_step = sampler_start_idx = acc_step = 0- running_loss = 0.0- running_disc_loss = 0.0+ running_loss = running_disc_loss = 0.0# == resume ==if cfg.get("load", None) is not None:logger.info("Loading checkpoint")- booster.load_model(model, os.path.join(cfg.load, "model"))- booster.load_optimizer(optimizer, os.path.join(cfg.load, "optimizer"))- if cfg.get("discriminator", False) != False and os.path.exists(os.path.join(cfg.load, "discriminator")):+ start_epoch, start_step, sampler_start_idx = load(+ booster,+ cfg.load,+ model=model,+ optimizer=optimizer,+ lr_scheduler=lr_scheduler,+ )+ if use_discriminator and os.path.exists(os.path.join(cfg.load, "discriminator")):booster.load_model(discriminator, os.path.join(cfg.load, "discriminator"))booster.load_optimizer(disc_optimizer, os.path.join(cfg.load, "disc_optimizer"))- running_states = load_json(os.path.join(cfg.load, "running_states.json"))- start_epoch, start_step, sampler_start_idx = (- running_states["epoch"],- running_states["step"],- running_states["sample_start_index"],- )- logger.info("Loaded checkpoint %s at epoch %s step %s", cfg.load, start_epoch, start_step)dist.barrier()+ logger.info("Loaded checkpoint %s at epoch %s step %s", cfg.load, start_epoch, start_step)- if cfg.dataset.type == DEFAULT_DATASET_NAME:- dataloader.sampler.set_start_index(sampler_start_idx)+ dataloader.sampler.set_start_index(sampler_start_idx)# =======================================================# 5. training loop# =======================================================+ dist.barrier()for epoch in range(start_epoch, cfg_epochs):# == set dataloader to new epoch ==- if cfg.dataset.type == DEFAULT_DATASET_NAME:- dataloader.sampler.set_epoch(epoch)+ dataloader.sampler.set_epoch(epoch)dataloader_iter = iter(dataloader)logger.info("Beginning epoch %s...", epoch)@@ -251,10 +241,15 @@ def main():) as pbar:for step, batch in pbar:x = batch["video"].to(device, dtype) # [B, C, T, H, W]- # if random.random() < cfg.get("mixed_image_ratio", 0.0):- # x = x[:, :, :1, :, :]- length = random.randint(1, x.size(2))- x = x[:, :, :length, :, :]++ # == mixed training setting ==+ mixed_strategy = cfg.get("mixed_strategy", None)+ if mixed_strategy == "mixed_video_image":+ if random.random() < cfg.get("mixed_image_ratio", 0.0):+ x = x[:, :, :1, :, :]+ elif mixed_strategy == "mixed_video_random":+ length = random.randint(1, x.size(2))+ x = x[:, :, :length, :, :]# == vae encoding ===x_rec, x_z_rec, z, posterior, x_z = model(x)@@ -264,29 +259,27 @@ def main():disc_loss = torch.tensor(0.0, device=device, dtype=dtype)log_dict = {}- # == real image reconstruction loss computation ==+ # == loss: real image reconstruction ==nll_loss, weighted_nll_loss, weighted_kl_loss = vae_loss_fn(x, x_rec, posterior)log_dict["kl_loss"] = weighted_kl_loss.item()log_dict["nll_loss"] = weighted_nll_loss.item()if cfg.get("use_real_rec_loss", False):vae_loss += weighted_nll_loss + weighted_kl_loss- # == temporal vae reconstruction loss computation ==+ # == loss: temporal vae reconstruction ==_, weighted_z_nll_loss, _ = vae_loss_fn(x_z, x_z_rec, posterior, no_perceptual=True)log_dict["z_nll_loss"] = weighted_z_nll_loss.item()-- # == use z reconstruction loss ==if cfg.get("use_z_rec_loss", False):vae_loss += weighted_z_nll_loss- # == use image loss only ==+ # == loss: image only distillation ==if cfg.get("use_image_identity_loss", False) and x.size(2) == 1:_, image_identity_loss, _ = vae_loss_fn(x_z, z, posterior, no_perceptual=True)vae_loss += image_identity_losslog_dict["image_identity_loss"] = image_identity_loss.item()- # == generator adversarial loss ==- if cfg.get("discriminator", False) != False:+ # == loss: generator adversarial ==+ if use_discriminator:recon_video = rearrange(x_rec, "b c t h w -> (b t) c h w").contiguous()global_step = epoch * num_steps_per_epoch + stepfake_logits = discriminator(recon_video.contiguous())@@ -297,15 +290,18 @@ def main():global_step,is_training=model.training,)+ log_dict["adversarial_loss"] = adversarial_loss.item()vae_loss += adversarial_loss# == generator backward & update ==optimizer.zero_grad()booster.backward(loss=vae_loss, optimizer=optimizer)optimizer.step()+ all_reduce_mean(vae_loss)+ running_loss += vae_loss.item()- # == discriminator adversarial loss ==- if cfg.get("discriminator", False) != False:+ # == loss: discriminator adversarial ==+ if use_discriminator:real_video = rearrange(x, "b c t h w -> (b t) c h w").contiguous()fake_video = rearrange(x_rec, "b c t h w -> (b t) c h w").contiguous()real_logits = discriminator(real_video.contiguous().detach())@@ -316,6 +312,7 @@ def main():global_step,)disc_loss = weighted_d_adversarial_loss+ log_dict["disc_loss"] = disc_loss.item()# == discriminator backward & update ==disc_optimizer.zero_grad()@@ -325,21 +322,21 @@ def main():running_disc_loss += disc_loss.item()# == update log info ==- all_reduce_mean(vae_loss)- running_loss += vae_loss.item()global_step = epoch * num_steps_per_epoch + steplog_step += 1acc_step += 1# == logging ==- if coordinator.is_master() and (global_step + 1) % cfg.log_every == 0:+ if coordinator.is_master() and (global_step + 1) % cfg.get("log_every", 1) == 0:avg_loss = running_loss / log_stepavg_disc_loss = running_disc_loss / log_step+ # progress barpbar.set_postfix({"loss": avg_loss, "disc_loss": avg_disc_loss, "step": step, "global_step": global_step})# tensorboardtb_writer.add_scalar("loss", vae_loss.item(), global_step)+ # wandbif cfg.wandb:wandb.log({@@ -352,33 +349,30 @@ def main():},step=global_step,)- running_loss = 0- running_disc_loss = 0+ running_loss = running_disc_loss = 0.0log_step = 0# == checkpoint saving ==- if cfg.ckpt_every > 0 and (global_step + 1) % cfg.ckpt_every == 0:- save_dir = os.path.join(exp_dir, f"epoch{epoch}-global_step{global_step+1}")- os.makedirs(os.path.join(save_dir, "model"), exist_ok=True) # already handled in booster save_model- booster.save_model(model, os.path.join(save_dir, "model"), shard=True)- booster.save_optimizer(- optimizer, os.path.join(save_dir, "optimizer"), shard=True, size_per_shard=4096+ ckpt_every = cfg.get("ckpt_every", 0)+ if ckpt_every > 0 and (global_step + 1) % ckpt_every == 0:+ save(+ booster,+ exp_dir,+ model=model,+ optimizer=optimizer,+ lr_scheduler=lr_scheduler,+ epoch=epoch,+ step=step + 1,+ global_step=global_step + 1,+ batch_size=cfg.get("batch_size", None),)- if cfg.get("discriminator", False) != False:++ save_dir = os.path.join(exp_dir, f"epoch{epoch}-global_step{global_step+1}")+ if use_discriminator:booster.save_model(discriminator, os.path.join(save_dir, "discriminator"), shard=True)booster.save_optimizer(disc_optimizer, os.path.join(save_dir, "disc_optimizer"), shard=True, size_per_shard=4096)-- running_states = {- "epoch": epoch,- "step": step + 1,- "global_step": global_step + 1,- "sample_start_index": (step + 1) * cfg.batch_size,- }-- if coordinator.is_master():- save_json(running_states, os.path.join(save_dir, "running_states.json"))dist.barrier()logger.info(@@ -389,12 +383,8 @@ def main():exp_dir,)- # the continue epochs are not resumed, so we need to reset the sampler start index and start step- if cfg.dataset.type == DEFAULT_DATASET_NAME:- dataloader.sampler.set_start_index(0)- else:- dataloader.batch_sampler.set_epoch(epoch + 1)- logger.info("Epoch done, recomputing batch sampler")+ # NOTE: the continue epochs are not resumed, so we need to reset the sampler start index and start step+ dataloader.sampler.set_start_index(0)start_step = 0