Open-Sora: Democratizing Efficient Video Production for All
Python
complete masked training
Zangwei Zheng4e6e17d80030
diff --git a/configs/opensora/inference_long/16x512x512-long.py b/configs/opensora/inference_long/16x512x512-long.pyindex 2bbb766..58c9e7a 100644--- a/configs/opensora/inference_long/16x512x512-long.py+++ b/configs/opensora/inference_long/16x512x512-long.py@@ -37,7 +37,7 @@ prompt = ["In an ornate, historical hall, a massive tidal wave peaks and begins to crash. Two surfers, seizing the moment, skillfully navigate the face of the wave."]-loop = 10+loop = 5condition_frame_length = 4reference_path = ["assets/images/condition/wave.png"]mask_strategy = ["0,0,0,1,0"] # valid when reference_path is not Nonediff --git a/configs/opensora/train/16x512x512-mask.py b/configs/opensora/train/16x512x512-mask.pynew file mode 100644index 0000000..7c8ffc1--- /dev/null+++ b/configs/opensora/train/16x512x512-mask.py@@ -0,0 +1,56 @@+num_frames = 16+frame_interval = 3+image_size = (512, 512)++# Define dataset+root = None+data_path = "CSV_PATH"+use_image_transform = False+num_workers = 4++# Define acceleration+dtype = "bf16"+grad_checkpoint = True+plugin = "zero2"+sp_size = 1++# Define model+model = dict(+ type="STDiT-XL/2",+ space_scale=1.0,+ time_scale=1.0,+ use_x_mask=True,+ from_pretrained=None,+ enable_flashattn=True,+ enable_layernorm_kernel=True,+)+mask_ratios = [0.5, 0.29, 0.07, 0.07, 0.07]+vae = dict(+ type="VideoAutoencoderKL",+ from_pretrained="stabilityai/sd-vae-ft-ema",+ micro_batch_size=128,+)+text_encoder = dict(+ type="t5",+ from_pretrained="DeepFloyd/t5-v1_1-xxl",+ model_max_length=120,+ shardformer=True,+)+scheduler = dict(+ type="iddpm",+ timestep_respacing="",+)++# Others+seed = 42+outputs = "outputs"+wandb = False++epochs = 1000+log_every = 10+ckpt_every = 500+load = None++batch_size = 8+lr = 2e-5+grad_clip = 1.0diff --git a/opensora/schedulers/iddpm/gaussian_diffusion.py b/opensora/schedulers/iddpm/gaussian_diffusion.pyindex 914d4d3..da233c3 100644--- a/opensora/schedulers/iddpm/gaussian_diffusion.py+++ b/opensora/schedulers/iddpm/gaussian_diffusion.py@@ -395,7 +395,6 @@ class GaussianDiffusion:- 'sample': a random sample from the model.- 'pred_xstart': a prediction of x_0."""- model_kwargs['x_mask'] = maskout = self.p_mean_variance(model,x,@@ -673,7 +672,7 @@ class GaussianDiffusion:yield outimg = out["sample"]- def _vb_terms_bpd(self, model, x_start, x_t, t, clip_denoised=True, model_kwargs=None):+ def _vb_terms_bpd(self, model, x_start, x_t, t, clip_denoised=True, model_kwargs=None, mask=None):"""Get a term for the variational lower-bound.The resulting units are bits (rather than nats, as one might expect).@@ -685,20 +684,20 @@ class GaussianDiffusion:true_mean, _, true_log_variance_clipped = self.q_posterior_mean_variance(x_start=x_start, x_t=x_t, t=t)out = self.p_mean_variance(model, x_t, t, clip_denoised=clip_denoised, model_kwargs=model_kwargs)kl = normal_kl(true_mean, true_log_variance_clipped, out["mean"], out["log_variance"])- kl = mean_flat(kl) / np.log(2.0)+ kl = mean_flat(kl, mask=mask) / np.log(2.0)decoder_nll = -discretized_gaussian_log_likelihood(x_start, means=out["mean"], log_scales=0.5 * out["log_variance"])assert decoder_nll.shape == x_start.shape- decoder_nll = mean_flat(decoder_nll) / np.log(2.0)+ decoder_nll = mean_flat(decoder_nll, mask=mask) / np.log(2.0)# At the first timestep return the decoder NLL,# otherwise return KL(q(x_{t-1}|x_t,x_0) || p(x_{t-1}|x_t))output = th.where((t == 0), decoder_nll, kl)return {"output": output, "pred_xstart": out["pred_xstart"]}- def training_losses(self, model, x_start, t, model_kwargs=None, noise=None):+ def training_losses(self, model, x_start, t, model_kwargs=None, noise=None, mask=None):"""Compute training losses for a single timestep.:param model: the model to evaluate loss on.@@ -715,10 +714,13 @@ class GaussianDiffusion:if noise is None:noise = th.randn_like(x_start)x_t = self.q_sample(x_start, t, noise=noise)+ if mask is not None:+ x_t = th.where(mask.unsqueeze(1).unsqueeze(-1).unsqueeze(-1), x_t, x_start)terms = {}if self.loss_type == LossType.KL or self.loss_type == LossType.RESCALED_KL:+ assert mask is None, "mask not supported for KL loss"terms["loss"] = self._vb_terms_bpd(model=model,x_start=x_start,@@ -748,6 +750,7 @@ class GaussianDiffusion:x_t=x_t,t=t,clip_denoised=False,+ mask=mask,)["output"]if self.loss_type == LossType.RESCALED_MSE:# Divide by 1000 for equivalence with initial implementation.@@ -760,7 +763,7 @@ class GaussianDiffusion:ModelMeanType.EPSILON: noise,}[self.model_mean_type]assert model_output.shape == target.shape == x_start.shape- terms["mse"] = mean_flat((target - model_output) ** 2)+ terms["mse"] = mean_flat((target - model_output) ** 2, mask=mask)if "vb" in terms:terms["loss"] = terms["mse"] + terms["vb"]else:diff --git a/opensora/utils/config_utils.py b/opensora/utils/config_utils.pyindex 89c7fe8..350cc8e 100644--- a/opensora/utils/config_utils.py+++ b/opensora/utils/config_utils.py@@ -48,26 +48,27 @@ def merge_args(cfg, args, training=False):cfg.model["from_pretrained"] = args.ckpt_pathargs.ckpt_path = None- if not training:- if args.cfg_scale is not None:- cfg.scheduler["cfg_scale"] = args.cfg_scale- args.cfg_scale = None-for k, v in vars(args).items():if k in cfg and v is not None:cfg[k] = v- if "reference_path" not in cfg:- cfg["reference_path"] = None- if "loop" not in cfg:- cfg["loop"] = 1+ if not training:+ # Inference only+ if "reference_path" not in cfg:+ cfg["reference_path"] = None+ if "loop" not in cfg:+ cfg["loop"] = 1+ if "prompt" not in cfg or cfg["prompt"] is None:+ assert cfg["prompt_path"] is not None, "prompt or prompt_path must be provided"+ cfg["prompt"] = load_prompts(cfg["prompt_path"])+ else:+ # Training only+ if "mask_ratios" not in cfg:+ cfg["mask_ratios"] = None++ # Both training and inferenceif "multi_resolution" not in cfg:cfg["multi_resolution"] = False- if "mask_ratios" not in cfg:- cfg["mask_ratios"] = None- if "prompt" not in cfg or cfg["prompt"] is None:- assert cfg["prompt_path"] is not None, "prompt or prompt_path must be provided"- cfg["prompt"] = load_prompts(cfg["prompt_path"])return cfgdiff --git a/opensora/utils/train_utils.py b/opensora/utils/train_utils.pyindex f846043..4a64aa9 100644--- a/opensora/utils/train_utils.py+++ b/opensora/utils/train_utils.py@@ -1,3 +1,4 @@+import randomfrom collections import OrderedDictimport torch@@ -29,3 +30,45 @@ def update_ema(else:param_data = param.dataema_params[name].mul_(decay).add_(param_data, alpha=1 - decay)+++class MaskGenerator:+ def __init__(self, mask_ratios):+ self.mask_name = ["mask_no", "mask_random", "mask_head", "mask_tail", "mask_head_tail"]+ self.mask_prob = mask_ratios+ print(self.mask_prob)+ self.mask_acc_prob = [sum(self.mask_prob[: i + 1]) for i in range(len(self.mask_prob))]++ def get_mask(self, x):+ mask_type = random.random()+ for i, acc_prob in enumerate(self.mask_acc_prob):+ if mask_type <= acc_prob:+ mask_name = self.mask_name[i]+ break++ mask = torch.ones(x.shape[2], dtype=torch.bool, device=x.device)+ if mask_name == "mask_random":+ random_size = random.randint(1, 4)+ random_pos = random.randint(0, x.shape[2] - random_size)+ mask[random_pos : random_pos + random_size] = 0+ return mask+ elif mask_name == "mask_head":+ random_size = random.randint(1, 4)+ mask[:random_size] = 0+ elif mask_name == "mask_tail":+ random_size = random.randint(1, 4)+ mask[-random_size:] = 0+ elif mask_name == "mask_head_tail":+ random_size = random.randint(1, 4)+ mask[:random_size] = 0+ mask[-random_size:] = 0++ return mask++ def get_masks(self, x):+ masks = []+ for _ in range(len(x)):+ mask = self.get_mask(x)+ masks.append(mask)+ masks = torch.stack(masks, dim=0)+ return masksdiff --git a/scripts/inference_long.py b/scripts/inference_long.pyindex 7f59f6c..4fddaf0 100644--- a/scripts/inference_long.py+++ b/scripts/inference_long.py@@ -176,6 +176,7 @@ def main():j] += f";{loop_i},{len(refs)-1},-{cfg.condition_frame_length},{cfg.condition_frame_length},0"masks = apply_mask_strategy(z, refs_x, mask_strategy, loop_i)+ model_args["x_mask"] = masks# 4.6. diffusion samplingsamples = scheduler.sample(diff --git a/scripts/train.py b/scripts/train.pyindex 9f611b7..1c61d6e 100644--- a/scripts/train.py+++ b/scripts/train.py@@ -28,7 +28,7 @@ from opensora.utils.config_utils import (save_training_config,)from opensora.utils.misc import all_reduce_mean, format_numel_str, get_model_numel, requires_grad, to_torch_dtype-from opensora.utils.train_utils import update_ema+from opensora.utils.train_utils import update_ema, MaskGeneratordef main():@@ -168,6 +168,8 @@ def main():model.train()update_ema(ema, model, decay=0, sharded=False)ema.eval()+ if cfg.mask_ratios is not None:+ mask_generator = MaskGenerator(cfg.mask_ratios)# =======================================================# 5. boost model for distributed training with colossalai@@ -214,15 +216,23 @@ def main():x = batch["video"].to(device, dtype) # [B, C, T, H, W]y = batch["text"]+ # Visual and text encodingwith torch.no_grad():# Prepare visual inputsx = vae.encode(x) # [B, C, T, H/P, W/P]# Prepare text inputsmodel_args = text_encoder.encode(y)++ # Mask+ if cfg.mask_ratios is not None:+ mask = mask_generator.get_masks(x)+ model_args["x_mask"] = mask+ else:+ mask = None# Diffusiont = torch.randint(0, scheduler.num_timesteps, (x.shape[0],), device=device)- loss_dict = scheduler.training_losses(model, x, t, model_args)+ loss_dict = scheduler.training_losses(model, x, t, model_args, mask=mask)# Backward & updateloss = loss_dict["loss"].mean()