Open-Sora: Democratizing Efficient Video Production for All
Python
complete masked inference
Zangwei Zheng9c81cd61bcc9
diff --git a/configs/opensora/inference_long/16x256x256-long.py b/configs/opensora/inference_long/16x512x512-long.pysimilarity index 85%rename from configs/opensora/inference_long/16x256x256-long.pyrename to configs/opensora/inference_long/16x512x512-long.pyindex de3c771..2bbb766 100644--- a/configs/opensora/inference_long/16x256x256-long.py+++ b/configs/opensora/inference_long/16x512x512-long.py@@ -1,15 +1,17 @@+# scripts/inference_long.pynum_frames = 16fps = 24 // 3-image_size = (256, 256)+image_size = (512, 512)# Define modelmodel = dict(type="STDiT-XL/2",- space_scale=0.5,+ space_scale=1.0,time_scale=1.0,+ use_x_mask=True,enable_flashattn=True,enable_layernorm_kernel=True,- from_pretrained="PRETRAINED_MODEL",+ from_pretrained=None,)vae = dict(type="VideoAutoencoderKL",@@ -22,8 +24,8 @@ text_encoder = dict(model_max_length=120,)scheduler = dict(- # type="iddpm",- type="dpm-solver",+ type="iddpm",+ # type="dpm-solver",num_sampling_steps=100,cfg_scale=7.0,)diff --git a/opensora/models/stdit/stdit.py b/opensora/models/stdit/stdit.pyindex 68db681..1485c9e 100644--- a/opensora/models/stdit/stdit.py+++ b/opensora/models/stdit/stdit.py@@ -5,6 +5,7 @@ import torch.nn as nnfrom einops import rearrangefrom timm.models.layers import DropPathfrom timm.models.vision_transformer import Mlp+from timm.models.layers import trunc_normal_from opensora.acceleration.checkpoint import auto_grad_checkpointfrom opensora.acceleration.communications import gather_forward_split_backward, split_forward_gather_backward@@ -137,6 +138,7 @@ class STDiT(nn.Module):space_scale=1.0,time_scale=1.0,freeze=None,+ use_x_mask=False,enable_flashattn=False,enable_layernorm_kernel=False,enable_sequence_parallelism=False,@@ -195,6 +197,11 @@ class STDiT(nn.Module):)self.final_layer = T2IFinalLayer(hidden_size, np.prod(self.patch_size), self.out_channels)+ self.use_x_mask = use_x_mask+ if self.use_x_mask:+ self.x_mask = nn.Parameter(torch.zeros(self.hidden_size))+ trunc_normal_(self.x_mask, std=0.02)+# init modelself.initialize_weights()self.initialize_temporal()@@ -212,7 +219,7 @@ class STDiT(nn.Module):else:self.sp_rank = None- def forward(self, x, timestep, y, mask=None):+ def forward(self, x, timestep, y, mask=None, x_mask=None):"""Forward pass of STDiT.Args:@@ -233,6 +240,10 @@ class STDiT(nn.Module):x = self.x_embedder(x) # [B, N, C]x = rearrange(x, "B (T S) C -> B T S C", T=self.num_temporal, S=self.num_spatial)x = x + self.pos_embed+ if x_mask is not None:+ assert self.use_x_mask+ # x: [B, T, S, C], mask: [B, T], self.x_mask: [C]+ x = x + ~x_mask[:, :, None, None] * self.x_maskx = rearrange(x, "B T S C -> B (T S) C")# shard over the sequence dim if sp is enableddiff --git a/opensora/schedulers/dpms/__init__.py b/opensora/schedulers/dpms/__init__.pyindex 6a1502c..ba2f648 100644--- a/opensora/schedulers/dpms/__init__.py+++ b/opensora/schedulers/dpms/__init__.py@@ -23,6 +23,7 @@ class DMP_SOLVER:additional_args=None,mask=None,):+ assert mask is None, "mask is not supported in dpm-solver"n = len(prompts)model_args = text_encoder.encode(prompts)y = model_args.pop("y")diff --git a/opensora/schedulers/iddpm/__init__.py b/opensora/schedulers/iddpm/__init__.pyindex f1fd49e..3147513 100644--- a/opensora/schedulers/iddpm/__init__.py+++ b/opensora/schedulers/iddpm/__init__.py@@ -87,6 +87,8 @@ def forward_with_cfg(model, x, timestep, y, cfg_scale, **kwargs):# https://github.com/openai/glide-text2im/blob/main/notebooks/text2im.ipynbhalf = x[: len(x) // 2]combined = torch.cat([half, half], dim=0)+ if "x_mask" in kwargs and kwargs["x_mask"] is not None:+ kwargs["x_mask"] = torch.cat([kwargs["x_mask"], kwargs["x_mask"]], dim=0)model_out = model.forward(combined, timestep, y, **kwargs)model_out = model_out["x"] if isinstance(model_out, dict) else model_outeps, rest = model_out[:, :3], model_out[:, 3:]diff --git a/opensora/schedulers/iddpm/gaussian_diffusion.py b/opensora/schedulers/iddpm/gaussian_diffusion.pyindex 173b851..914d4d3 100644--- a/opensora/schedulers/iddpm/gaussian_diffusion.py+++ b/opensora/schedulers/iddpm/gaussian_diffusion.py@@ -395,6 +395,7 @@ class GaussianDiffusion:- 'sample': a random sample from the model.- 'pred_xstart': a prediction of x_0."""+ model_kwargs['x_mask'] = maskout = self.p_mean_variance(model,x,