Open-Sora: Democratizing Efficient Video Production for All
Python
fix dist issue
Shen-Chenhuic7a698e85bf7
diff --cc configs/vae_magvit_v2/train/16x128x128_pixabay_8_GPU_pipeline.pyindex 8405bfa,e3aa372..86f8711--- a/configs/vae_magvit_v2/train/16x128x128_pixabay_8_GPU_pipeline.py+++ b/configs/vae_magvit_v2/train/16x128x128_pixabay_8_GPU_pipeline.py@@@ -84,9 -84,9 +84,11 @@@ magvit uses about # samples (K) * epochepochs = 200log_every = 1- ckpt_every = 1 # 50-ckpt_every = 5++++ckpt_every = 50load = None- batch_size = 4 # 32-batch_size = 4++batch_size = 32++lr = 1e-4grad_clip = 1.0diff --cc scripts/train-vae-v2.pyindex b6cdd25,f386546..4967935--- a/scripts/train-vae-v2.py+++ b/scripts/train-vae-v2.py@@@ -224,14 -235,13 +238,14 @@@ def main()booster.load_lr_scheduler(lr_scheduler, os.path.join(cfg.load, "lr_scheduler"))if disc_lr_scheduler is not None:booster.load_lr_scheduler(disc_lr_scheduler, os.path.join(cfg.load, "disc_lr_scheduler"))- lecam_path = os.path.join(cfg.load, "lecam_state.json")+++ # LeCam EMA for discriminator+ lecam_path = os.path.join(cfg.load, "lecam_states.json")if cfg.lecam_loss_weight is not None and os.path.exists(lecam_path):lecam_state = load_json(lecam_path)lecam_ema_real, lecam_ema_fake = lecam_state["lecam_ema_real"], lecam_state["lecam_ema_fake"]lecam_ema = LeCamEMA(decay=cfg.ema_decay, ema_real=lecam_ema_real, ema_fake=lecam_ema_fake, dtype=dtype, device=device)-- else:- print(f"lecan not loaded, path: {lecam_path}, lecame loss weight {cfg.lecam_loss_weight}")- lecam_ema = LeCamEMA(decay=cfg.ema_decay, dtype=dtype, device=device)++running_states = load_json(os.path.join(cfg.load, "running_states.json"))dist.barrier()start_epoch, start_step, sampler_start_idx = running_states["epoch"], running_states["step"], running_states["sample_start_index"]@@@ -274,9 -284,10 +288,6 @@@disc_time_padding = 0video_contains_first_frame = cfg.video_contains_first_frame---- # lecam_ema_real = torch.tensor(0.0)-- # lecam_ema_fake = torch.tensor(0.0)-for epoch in range(start_epoch, cfg.epochs):dataloader.sampler.set_epoch(epoch)@@@ -405,10 -416,10 +416,7 @@@real_video = real_video if cfg.gradient_penalty_loss_weight is not None else None,)disc_loss = weighted_d_adversarial_loss + lecam_loss + gradient_penalty_loss- if cfg.ema_decay is not None:- # SCH: TODO: is this written properly like this for moving average? e.g. distributed training etc.- # lecam_ema_real = lecam_ema_real * cfg.ema_decay + (1 - cfg.ema_decay) * torch.mean(real_logits.clone().detach())- # lecam_ema_fake = lecam_ema_fake * cfg.ema_decay + (1 - cfg.ema_decay) * torch.mean(fake_logits.clone().detach())+ if cfg.lecam_loss_weight is not None:- # SCH: TODO: is this written properly like this for moving average? e.g. distributed training etc.- # lecam_ema_real = lecam_ema_real * cfg.ema_decay + (1 - cfg.ema_decay) * torch.mean(real_logits.clone().detach())- # lecam_ema_fake = lecam_ema_fake * cfg.ema_decay + (1 - cfg.ema_decay) * torch.mean(fake_logits.clone().detach())ema_real = torch.mean(real_logits.clone().detach()).to(device, dtype)ema_fake = torch.mean(fake_logits.clone().detach()).to(device, dtype)all_reduce_mean(ema_real)@@@ -463,11 -474,15 +471,11 @@@# Save checkpointif cfg.ckpt_every > 0 and (global_step + 1) % cfg.ckpt_every == 0:save_dir = os.path.join(exp_dir, f"epoch{epoch}-global_step{global_step+1}")- os.makedirs(os.path.join(save_dir, "model"), exist_ok=True)+ os.makedirs(os.path.join(save_dir, "model"), exist_ok=True) # already handled in booster save_modelbooster.save_model(vae, os.path.join(save_dir, "model"), shard=True)- print("model saved")booster.save_model(discriminator, os.path.join(save_dir, "discriminator"), shard=True)- print("discriminator saved")booster.save_optimizer(optimizer, os.path.join(save_dir, "optimizer"), shard=True, size_per_shard=4096)- print("optimizer saved")booster.save_optimizer(disc_optimizer, os.path.join(save_dir, "disc_optimizer"), shard=True, size_per_shard=4096)- print("disc opt saved")if lr_scheduler is not None:booster.save_lr_scheduler(lr_scheduler, os.path.join(save_dir, "lr_scheduler"))@@@ -480,17 -504,7 +488,18 @@@"global_step": global_step+1,"sample_start_index": (step+1) * cfg.batch_size,}- save_json(running_states, os.path.join(save_dir, "running_states.json"))++ lecam_ema_real, lecam_ema_fake = lecam_ema.get()+ lecam_state = {+ "lecam_ema_real": lecam_ema_real.item(),+ "lecam_ema_fake": lecam_ema_fake.item(),+ }+ if coordinator.is_master():+ save_json(running_states, os.path.join(save_dir, "running_states.json"))+ if cfg.lecam_loss_weight is not None:+ save_json(lecam_state, os.path.join(save_dir, "lecam_states.json"))+ dist.barrier()++logger.info(f"Saved checkpoint at epoch {epoch} step {step + 1} global_step {global_step + 1} to {exp_dir}")