Skip to content

Open-Sora: Democratizing Efficient Video Production for All

Python

complete masked inference

Zangwei Zheng9c81cd61bcc9
diff --git a/configs/opensora/inference_long/16x256x256-long.py b/configs/opensora/inference_long/16x512x512-long.py
similarity index 85%
rename from configs/opensora/inference_long/16x256x256-long.py
rename to configs/opensora/inference_long/16x512x512-long.py
index de3c771..2bbb766 100644
--- a/configs/opensora/inference_long/16x256x256-long.py
+++ b/configs/opensora/inference_long/16x512x512-long.py
@@ -1,15 +1,17 @@
+# scripts/inference_long.py
num_frames = 16
fps = 24 // 3
-image_size = (256, 256)
+image_size = (512, 512)
# Define model
model = dict(
type="STDiT-XL/2",
- space_scale=0.5,
+ space_scale=1.0,
time_scale=1.0,
+ use_x_mask=True,
enable_flashattn=True,
enable_layernorm_kernel=True,
- from_pretrained="PRETRAINED_MODEL",
+ from_pretrained=None,
)
vae = dict(
type="VideoAutoencoderKL",
@@ -22,8 +24,8 @@ text_encoder = dict(
model_max_length=120,
)
scheduler = dict(
- # type="iddpm",
- type="dpm-solver",
+ type="iddpm",
+ # type="dpm-solver",
num_sampling_steps=100,
cfg_scale=7.0,
)
diff --git a/opensora/models/stdit/stdit.py b/opensora/models/stdit/stdit.py
index 68db681..1485c9e 100644
--- a/opensora/models/stdit/stdit.py
+++ b/opensora/models/stdit/stdit.py
@@ -5,6 +5,7 @@ import torch.nn as nn
from einops import rearrange
from timm.models.layers import DropPath
from timm.models.vision_transformer import Mlp
+from timm.models.layers import trunc_normal_
from opensora.acceleration.checkpoint import auto_grad_checkpoint
from opensora.acceleration.communications import gather_forward_split_backward, split_forward_gather_backward
@@ -137,6 +138,7 @@ class STDiT(nn.Module):
space_scale=1.0,
time_scale=1.0,
freeze=None,
+ use_x_mask=False,
enable_flashattn=False,
enable_layernorm_kernel=False,
enable_sequence_parallelism=False,
@@ -195,6 +197,11 @@ class STDiT(nn.Module):
)
self.final_layer = T2IFinalLayer(hidden_size, np.prod(self.patch_size), self.out_channels)
+ self.use_x_mask = use_x_mask
+ if self.use_x_mask:
+ self.x_mask = nn.Parameter(torch.zeros(self.hidden_size))
+ trunc_normal_(self.x_mask, std=0.02)
+
# init model
self.initialize_weights()
self.initialize_temporal()
@@ -212,7 +219,7 @@ class STDiT(nn.Module):
else:
self.sp_rank = None
- def forward(self, x, timestep, y, mask=None):
+ def forward(self, x, timestep, y, mask=None, x_mask=None):
"""
Forward pass of STDiT.
Args:
@@ -233,6 +240,10 @@ class STDiT(nn.Module):
x = self.x_embedder(x) # [B, N, C]
x = rearrange(x, "B (T S) C -> B T S C", T=self.num_temporal, S=self.num_spatial)
x = x + self.pos_embed
+ if x_mask is not None:
+ assert self.use_x_mask
+ # x: [B, T, S, C], mask: [B, T], self.x_mask: [C]
+ x = x + ~x_mask[:, :, None, None] * self.x_mask
x = rearrange(x, "B T S C -> B (T S) C")
# shard over the sequence dim if sp is enabled
diff --git a/opensora/schedulers/dpms/__init__.py b/opensora/schedulers/dpms/__init__.py
index 6a1502c..ba2f648 100644
--- a/opensora/schedulers/dpms/__init__.py
+++ b/opensora/schedulers/dpms/__init__.py
@@ -23,6 +23,7 @@ class DMP_SOLVER:
additional_args=None,
mask=None,
):
+ assert mask is None, "mask is not supported in dpm-solver"
n = len(prompts)
model_args = text_encoder.encode(prompts)
y = model_args.pop("y")
diff --git a/opensora/schedulers/iddpm/__init__.py b/opensora/schedulers/iddpm/__init__.py
index f1fd49e..3147513 100644
--- a/opensora/schedulers/iddpm/__init__.py
+++ b/opensora/schedulers/iddpm/__init__.py
@@ -87,6 +87,8 @@ def forward_with_cfg(model, x, timestep, y, cfg_scale, **kwargs):
# https://github.com/openai/glide-text2im/blob/main/notebooks/text2im.ipynb
half = x[: len(x) // 2]
combined = torch.cat([half, half], dim=0)
+ if "x_mask" in kwargs and kwargs["x_mask"] is not None:
+ kwargs["x_mask"] = torch.cat([kwargs["x_mask"], kwargs["x_mask"]], dim=0)
model_out = model.forward(combined, timestep, y, **kwargs)
model_out = model_out["x"] if isinstance(model_out, dict) else model_out
eps, rest = model_out[:, :3], model_out[:, 3:]
diff --git a/opensora/schedulers/iddpm/gaussian_diffusion.py b/opensora/schedulers/iddpm/gaussian_diffusion.py
index 173b851..914d4d3 100644
--- a/opensora/schedulers/iddpm/gaussian_diffusion.py
+++ b/opensora/schedulers/iddpm/gaussian_diffusion.py
@@ -395,6 +395,7 @@ class GaussianDiffusion:
- 'sample': a random sample from the model.
- 'pred_xstart': a prediction of x_0.
"""
+ model_kwargs['x_mask'] = mask
out = self.p_mean_variance(
model,
x,