Open-Sora: Democratizing Efficient Video Production for All
Python
[hotfix] fix sample and update training script (#15)
Hongxin Liue02f6286deae
diff --git a/sample.py b/sample.pyindex 0bb60b5..ffb4f2f 100644--- a/sample.py+++ b/sample.py@@ -19,7 +19,7 @@ from transformers import AutoModel, AutoTokenizer, CLIPTextModelfrom open_sora.diffusion import create_diffusionfrom open_sora.modeling import DiT_models-from open_sora.utils.data import col2video+from open_sora.utils.data import col2video, unnormalize_videodef main(args):@@ -34,16 +34,20 @@ def main(args):.eval())in_channels = vqvae.embedding_dim+ w_h_factor = 4+ t_factor = 2else:# disable VQ-VAE if not provided, just use raw video framesvqvae = Nonein_channels = 3+ w_h_factor = 1+ t_factor = 1text_model = CLIPTextModel.from_pretrained(args.text_model).to(device).eval()tokenizer = AutoTokenizer.from_pretrained(args.text_model)model = DiT_models[args.model](in_channels=in_channels).to(device).eval()patch_size = model.patch_size- # model.load_state_dict(torch.load(args.ckpt))+ model.load_state_dict(torch.load(args.ckpt))diffusion = create_diffusion(str(args.num_sampling_steps))# Create sampling noise:@@ -54,9 +58,9 @@ def main(args):num_frames = args.fps * args.secz = torch.randn(1,- (args.height // patch_size // 4)- * (args.width // patch_size // 4)- * (num_frames // 2),+ (args.height // patch_size // w_h_factor)+ * (args.width // patch_size // w_h_factor)+ * (num_frames // t_factor),in_channels,patch_size,patch_size,@@ -87,7 +91,12 @@ def main(args):samples, _ = samples.chunk(2, dim=0) # Remove null class samplessamples = col2video(samples.squeeze(),- (num_frames // 2, in_channels, args.height // 4, args.width // 4),+ (+ num_frames // t_factor,+ in_channels,+ args.height // w_h_factor,+ args.width // w_h_factor,+ ),)if vqvae is not None:# [T, C, H, W] -> [B, C, T, H, W]@@ -98,6 +107,7 @@ def main(args):else:# [T, C, H, W] -> [T, H, W, C]samples = samples.permute(0, 2, 3, 1)+ samples = unnormalize_video(samples).to(torch.uint8)write_video("sample.mp4", samples.cpu(), args.fps)@@ -105,12 +115,12 @@ def main(args):if __name__ == "__main__":parser = argparse.ArgumentParser()parser.add_argument(- "--model", type=str, choices=list(DiT_models.keys()), default="DiT-S/8"+ "-m", "--model", type=str, choices=list(DiT_models.keys()), default="DiT-S/8")parser.add_argument("--text",type=str,- default="two ladies laughing by seeing some thing another lady throw dresses and keep it back by reverse motion",+ default="a cartoon animals runs through an ice cave in a video game",)parser.add_argument("--cfg-scale", type=float, default=4.0)parser.add_argument("--num-sampling-steps", type=int, default=250)diff --git a/train.py b/train.pyindex 3a353c2..ad2ff74 100644--- a/train.py+++ b/train.py@@ -64,6 +64,18 @@ def all_reduce_mean(tensor: torch.Tensor) -> torch.Tensor:return tensor+def save_checkpoints(booster, model, optimizer, ema, save_path, coordinator):+ os.makedirs(save_path, exist_ok=True)+ booster.save_model(model, os.path.join(save_path, "model"), shard=False)+ booster.save_optimizer(optimizer, os.path.join(save_path, "optimizer"), shard=True)+ if coordinator.is_master():+ ema_state_dict = ema.state_dict()+ for k, v in ema_state_dict.items():+ ema_state_dict[k] = v.cpu()+ torch.save(ema_state_dict, os.path.join(save_path, "ema.pt"))+ dist.barrier()++################################################################################## Training Loop ##################################################################################@@ -89,7 +101,11 @@ def main(args):# Step 3: Create VQ-VAEif len(args.vqvae) > 0:- vqvae = AutoModel.from_pretrained(args.vqvae, trust_remote_code=True).to(get_current_device()).eval()+ vqvae = (+ AutoModel.from_pretrained(args.vqvae, trust_remote_code=True)+ .to(get_current_device())+ .eval()+ )model_kwargs = {"in_channels": vqvae.embedding_dim}else:# disable VQ-VAE if not provided, just use raw video frames@@ -110,7 +126,9 @@ def main(args):model.enable_gradient_checkpointing()# Step 5: create diffusion pipeline- diffusion = create_diffusion(timestep_respacing="") # default: 1000 steps, linear noise schedule+ diffusion = create_diffusion(+ timestep_respacing=""+ ) # default: 1000 steps, linear noise schedule# Step 6: setup optimizer (we used default Adam betas=(0.9, 0.999) and a constant learning rate of 1e-4 in our paper):opt = torch.optim.AdamW(model.parameters(), lr=args.lr, weight_decay=0)@@ -128,6 +146,10 @@ def main(args):# Step 8: setup boostermodel, opt, _, dataloader, _ = booster.boost(model, opt, dataloader=dataloader)+ if args.load_model is not None:+ booster.load_model(model, args.load_model)+ if args.load_optimizer is not None:+ booster.load_optimizer(opt, args.load_optimizer)logger.info(f"Booster init max device memory: {get_accelerator().max_memory_allocated() / 1024 ** 2:.2f} MB",ranks=[0],@@ -154,7 +176,9 @@ def main(args):(video_inputs.shape[0],),device=video_inputs.device,)- loss_dict = diffusion.training_losses(model, video_inputs, t, batch, mask=mask)+ loss_dict = diffusion.training_losses(+ model, video_inputs, t, batch, mask=mask+ )loss = loss_dict["loss"].mean() / args.accumulation_stepstotal_loss.add_(loss.data)booster.backward(loss, opt)@@ -167,7 +191,9 @@ def main(args):all_reduce_mean(total_loss)pbar.set_postfix({"Loss": f"{total_loss.item():.4f}"})if coordinator.is_master():- global_step = (epoch * num_steps_per_epoch) + (step + 1) // args.accumulation_steps+ global_step = (epoch * num_steps_per_epoch) + (+ step + 1+ ) // args.accumulation_stepswriter.add_scalar(tag="Loss",scalar_value=total_loss.item(),@@ -177,22 +203,20 @@ def main(args):total_loss.zero_()# Save DiT checkpoint:- if (args.save_interval > 0 and (step + 1) % (args.save_interval * args.accumulation_steps) == 0) or (- step + 1- ) == len(dataloader):- save_path = os.path.join(args.checkpoint_dir, f"epoch-{epoch}-step-{step}")- os.makedirs(save_path, exist_ok=True)- booster.save_model(model, os.path.join(save_path, "model"), shard=True)- booster.save_optimizer(opt, os.path.join(save_path, "optimizer"), shard=True)- if coordinator.is_master():- ema_state_dict = ema.state_dict()- for k, v in ema_state_dict.items():- ema_state_dict[k] = v.cpu()- torch.save(ema_state_dict, os.path.join(save_path, "ema.pt"))- dist.barrier()+ if args.save_interval > 0 and (+ (step + 1) % (args.save_interval * args.accumulation_steps) == 0+ or (step + 1) == len(dataloader)+ ):+ save_path = os.path.join(+ args.checkpoint_dir, f"epoch-{epoch}-step-{step}"+ )+ save_checkpoints(booster, model, opt, ema, save_path, coordinator)logger.info(f"Saved checkpoint to {save_path}", ranks=[0])get_accelerator().empty_cache()+ final_save_path = os.path.join(args.checkpoint_dir, "final")+ save_checkpoints(booster, model, opt, ema, final_save_path, coordinator)+ logger.info(f"Saved checkpoint to {final_save_path}", ranks=[0])logger.info(f"Training complete, max device memory: {get_accelerator().max_memory_allocated() / 1024 ** 2:.2f} MB",ranks=[0],@@ -202,7 +226,9 @@ def main(args):if __name__ == "__main__":# Default args here will train DiT-XL/2 with the hyperparameters we used in our paper (except training iters).parser = argparse.ArgumentParser()- parser.add_argument("-m", "--model", type=str, choices=list(DiT_models.keys()), default="DiT-S/8")+ parser.add_argument(+ "-m", "--model", type=str, choices=list(DiT_models.keys()), default="DiT-S/8"+ )parser.add_argument("-d", "--dataset", nargs="+", default=[])parser.add_argument("-v", "--video_dir", type=str, required=True)parser.add_argument("-e", "--epochs", type=int, default=10)@@ -214,5 +240,7 @@ if __name__ == "__main__":parser.add_argument("--checkpoint_dir", type=str, default="checkpoints")parser.add_argument("--tensorboard_dir", type=str, default="runs")parser.add_argument("--vqvae", default="hpcai-tech/vqvae")+ parser.add_argument("--load_model", default=None)+ parser.add_argument("--load_optimizer", default=None)args = parser.parse_args()main(args)