Skip to content

Open-Sora: Democratizing Efficient Video Production for All

Python

complete 3b backbone

zhengzangwf3dcb4d6fb18
diff --git a/assets/texts/t2i_sigma.txt b/assets/texts/t2i_sigma.txt
index 7866969..fa0f0a4 100644
--- a/assets/texts/t2i_sigma.txt
+++ b/assets/texts/t2i_sigma.txt
@@ -1,3 +1,6 @@
+Eiffel Tower was Made up of more than 2 million translucent straws to look like a cloud, with the bell tower at the top of the building, Michel installed huge foam-making machines in the forest to blow huge amounts of unpredictable wet clouds in the building's classic architecture.
+A gorgeously rendered papercraft world of a coral reef, rife with colorful fish and sea creatures.
+Full body shot, a French woman, Photography, French Streets background, backlighting, rim light, Fujifilm.
Close-up photos of models, hazy light and shadow, laser metal hair accessories, soft and beautiful, light gold pupils, white eyelashes, low saturation, real skin details, clear pores and fine lines, light reflection and refraction, ultra-clear, cinematography, award-winning works.
A litter of golden retriever puppies playing in the snow. Their heads pop out of the snow, covered in.
Lego model, future rocket station, intricate details, high resolution, unreal engine, UHD
@@ -5,6 +8,3 @@ One giant, sharp, metal square mirror in the center of the frame, four young peo
Modern luxury contemporary luxury home interiors house, in the style of mimicking ruined materials, ray tracing, haunting houses, and stone, capture the essence of nature, gray and bronze, dynamic outdoor shots.
Over the shoulder game perspective, game screen of Diablo 4, Inside the gorgeous palace is the wet ground, The necromancer knelt before the king, and a horde of skeletons he summoned stood at his side, cinematic light.
A curvy timber house near a sea, designed by Zaha Hadid, represent the image of a cold, modern architecture, at night, white lighting, highly detailed.
-Full body shot, a French woman, Photography, French Streets background, backlighting, rim light, Fujifilm.
-Eiffel Tower was Made up of more than 2 million translucent straws to look like a cloud, with the bell tower at the top of the building, Michel installed huge foam-making machines in the forest to blow huge amounts of unpredictable wet clouds in the building's classic architecture.
-A gorgeously rendered papercraft world of a coral reef, rife with colorful fish and sea creatures.
diff --git a/configs/opensora-v1-2/inference/pixart.py b/configs/opensora-v1-2/inference/pixart.py
index 32f4f46..1baf45d 100644
--- a/configs/opensora-v1-2/inference/pixart.py
+++ b/configs/opensora-v1-2/inference/pixart.py
@@ -1,11 +1,12 @@
num_frames = 1
fps = 1
-image_size = (2560, 1536)
-# image_size = (2048, 2048)
+# image_size = (2560, 1536)
+image_size = (2048, 2048)
model = dict(
- type="PixArtMS-1B/2",
- # from_pretrained="PixArt-Sigma-XL-2-2K-MS.pth",
+ # type="PixArt-1B/2",
+ type="PixArt-XL/2",
+ from_pretrained="PixArt-Sigma-XL-2-2K-MS.pth",
space_scale=4,
no_temporal_pos_emb=True,
enable_flashattn=True,
diff --git a/configs/opensora-v1-2/inference/xl.py b/configs/opensora-v1-2/inference/xl.py
new file mode 100644
index 0000000..00bb21d
--- /dev/null
+++ b/configs/opensora-v1-2/inference/xl.py
@@ -0,0 +1,37 @@
+num_frames = 2
+fps = 1
+image_size = (2560, 1536)
+# image_size = (2048, 2048)
+multi_resolution = "STDiT2"
+
+model = dict(
+ type="STDiT3-XL/2",
+ from_pretrained="PixArt-Sigma-XL-2-2K-MS.pth",
+ enable_flashattn=True,
+ enable_layernorm_kernel=True,
+)
+vae = dict(
+ type="VideoAutoencoderKL",
+ micro_batch_size=4,
+ from_pretrained="PixArt-alpha/pixart_sigma_sdxlvae_T5_diffusers",
+ subfolder="vae",
+ local_files_only=True,
+)
+text_encoder = dict(
+ type="t5",
+ from_pretrained="DeepFloyd/t5-v1_1-xxl",
+ model_max_length=300,
+ local_files_only=True,
+)
+scheduler = dict(
+ type="dpm-solver",
+ num_sampling_steps=14,
+ cfg_scale=4.5,
+)
+dtype = "bf16"
+
+# Others
+batch_size = 1
+seed = 42
+prompt_path = "./assets/texts/t2i_sigma.txt"
+save_dir = "./samples/samples/"
diff --git a/configs/opensora-v1-2/train/3b.py b/configs/opensora-v1-2/train/3b.py
new file mode 100644
index 0000000..d23dfd7
--- /dev/null
+++ b/configs/opensora-v1-2/train/3b.py
@@ -0,0 +1,57 @@
+# Define dataset
+dataset = dict(
+ type="VariableVideoTextDataset",
+ frame_interval=3,
+ transform_name="resize_crop",
+)
+bucket_config = {
+ "144p": {2: (1.0, 1)},
+}
+
+# Define acceleration
+num_workers = 4
+num_bucket_build_workers = 16
+dtype = "bf16"
+grad_checkpoint = True
+plugin = "zero2"
+sp_size = 1
+
+# Define model
+model = dict(
+ type="STDiT3-3B/2",
+ from_pretrained=None,
+ enable_flashattn=True,
+ enable_layernorm_kernel=True,
+)
+vae = dict(
+ type="VideoAutoencoderKL",
+ micro_batch_size=4,
+ from_pretrained="PixArt-alpha/pixart_sigma_sdxlvae_T5_diffusers",
+ subfolder="vae",
+ local_files_only=True,
+)
+text_encoder = dict(
+ type="t5",
+ from_pretrained="DeepFloyd/t5-v1_1-xxl",
+ model_max_length=300,
+ shardformer=True,
+ local_files_only=True,
+)
+scheduler = dict(
+ type="iddpm",
+ timestep_respacing="",
+)
+
+# Others
+seed = 42
+outputs = "outputs"
+wandb = False
+
+epochs = 1000
+log_every = 10
+ckpt_every = 500
+load = None
+
+batch_size = 10 # only for logging
+lr = 2e-5
+grad_clip = 1.0
diff --git a/configs/opensora-v1-2/train/xl.py b/configs/opensora-v1-2/train/xl.py
new file mode 100644
index 0000000..adc379b
--- /dev/null
+++ b/configs/opensora-v1-2/train/xl.py
@@ -0,0 +1,69 @@
+# Define dataset
+dataset = dict(
+ type="VariableVideoTextDataset",
+ frame_interval=3,
+ transform_name="resize_crop",
+)
+bucket_config = {
+ "240p": {16: (1.0, 2)},
+}
+mask_ratios = { # 20%
+ "mask_random": 0.10,
+ "mask_quarter_random": 0.01,
+ "mask_quarter_head": 0.01,
+ "mask_quarter_tail": 0.01,
+ "mask_quarter_head_tail": 0.01,
+ "mask_image_random": 0.01,
+ "mask_image_head": 0.02,
+ "mask_image_tail": 0.01,
+ "mask_image_head_tail": 0.01,
+ "mask_intepolate": 0.01,
+}
+
+# Define acceleration
+num_workers = 4
+num_bucket_build_workers = 16
+dtype = "bf16"
+grad_checkpoint = True
+plugin = "zero2"
+sp_size = 1
+
+# Define model
+model = dict(
+ type="STDiT3-XL/2",
+ from_pretrained=None,
+ enable_flashattn=True,
+ enable_layernorm_kernel=True,
+)
+vae = dict(
+ type="VideoAutoencoderKL",
+ micro_batch_size=4,
+ from_pretrained="PixArt-alpha/pixart_sigma_sdxlvae_T5_diffusers",
+ subfolder="vae",
+ local_files_only=True,
+)
+text_encoder = dict(
+ type="t5",
+ from_pretrained="DeepFloyd/t5-v1_1-xxl",
+ model_max_length=300,
+ shardformer=True,
+ local_files_only=True,
+)
+scheduler = dict(
+ type="iddpm",
+ timestep_respacing="",
+)
+
+# Others
+seed = 42
+outputs = "outputs"
+wandb = False
+
+epochs = 1000
+log_every = 10
+ckpt_every = 500
+load = None
+
+batch_size = 10 # only for logging
+lr = 2e-5
+grad_clip = 1.0
diff --git a/opensora/datasets/datasets.py b/opensora/datasets/datasets.py
index 4c2b316..ca4d70e 100644
--- a/opensora/datasets/datasets.py
+++ b/opensora/datasets/datasets.py
@@ -24,7 +24,7 @@ class VideoTextDataset(torch.utils.data.Dataset):
def __init__(
self,
- data_path,
+ data_path=None,
num_frames=16,
frame_interval=1,
image_size=(256, 256),
@@ -107,10 +107,10 @@ class VideoTextDataset(torch.utils.data.Dataset):
class VariableVideoTextDataset(VideoTextDataset):
def __init__(
self,
- data_path,
+ data_path=None,
num_frames=None,
frame_interval=1,
- image_size=None,
+ image_size=(None, None),
transform_name=None,
):
super().__init__(data_path, num_frames, frame_interval, image_size, transform_name=None)
diff --git a/opensora/models/pixart/pixart.py b/opensora/models/pixart/pixart.py
index 5fec4d0..b5bd9ca 100644
--- a/opensora/models/pixart/pixart.py
+++ b/opensora/models/pixart/pixart.py
@@ -211,6 +211,7 @@ class PixArt(nn.Module):
# embedding
x = self.x_embedder(x) # (B, N, D)
x = rearrange(x, "b (t s) d -> b t s d", t=self.num_temporal, s=self.num_spatial)
+ breakpoint()
x = x + self.pos_embed
if not self.no_temporal_pos_emb:
x = rearrange(x, "b t s d -> b s t d")
@@ -387,17 +388,17 @@ def PixArt_XL_2(from_pretrained=None, **kwargs):
return model
[email protected]_module("PixArtMS-XL/2")
-def PixArtMS_XL_2(from_pretrained=None, **kwargs):
- model = PixArtMS(depth=28, hidden_size=1152, patch_size=(1, 2, 2), num_heads=16, **kwargs)
[email protected]_module("PixArt-1B/2")
+def PixArt_1B_2(from_pretrained=None, **kwargs):
+ model = PixArt(depth=28, hidden_size=1872, patch_size=(1, 2, 2), num_heads=26, **kwargs)
if from_pretrained is not None:
load_checkpoint(model, from_pretrained)
return model
[email protected]_module("PixArtMS-1B/2")
-def PixArtMS_1B_2(from_pretrained=None, **kwargs):
- model = PixArtMS(depth=28, hidden_size=1872, patch_size=(1, 2, 2), num_heads=26, **kwargs)
[email protected]_module("PixArtMS-XL/2")
+def PixArtMS_XL_2(from_pretrained=None, **kwargs):
+ model = PixArtMS(depth=28, hidden_size=1152, patch_size=(1, 2, 2), num_heads=16, **kwargs)
if from_pretrained is not None:
load_checkpoint(model, from_pretrained)
return model
diff --git a/opensora/models/stdit/__init__.py b/opensora/models/stdit/__init__.py
index 605159e..e2ae7b0 100644
--- a/opensora/models/stdit/__init__.py
+++ b/opensora/models/stdit/__init__.py
@@ -1,2 +1,3 @@
from .stdit import STDiT
from .stdit2 import STDiT2
+from .stdit3 import STDiT3
diff --git a/opensora/models/stdit/stdit3.py b/opensora/models/stdit/stdit3.py
index e69de29..85f51e0 100644
--- a/opensora/models/stdit/stdit3.py
+++ b/opensora/models/stdit/stdit3.py
@@ -0,0 +1,368 @@
+import numpy as np
+import torch
+import torch.nn as nn
+from einops import rearrange
+from rotary_embedding_torch import RotaryEmbedding
+from timm.models.layers import DropPath
+from timm.models.vision_transformer import Mlp
+
+from opensora.acceleration.checkpoint import auto_grad_checkpoint
+from opensora.models.layers.blocks import (
+ Attention,
+ CaptionEmbedder,
+ MultiHeadCrossAttention,
+ PatchEmbed3D,
+ PositionEmbedding2D,
+ SizeEmbedder,
+ T2IFinalLayer,
+ TimestepEmbedder,
+ approx_gelu,
+ get_layernorm,
+ t2i_modulate,
+)
+from opensora.registry import MODELS
+from opensora.utils.ckpt_utils import load_checkpoint
+
+
+class STDiT3Block(nn.Module):
+ def __init__(
+ self,
+ hidden_size,
+ num_heads,
+ mlp_ratio=4.0,
+ drop_path=0.0,
+ rope=None,
+ qk_norm=False,
+ temporal=False,
+ enable_flashattn=False,
+ enable_layernorm_kernel=False,
+ enable_sequence_parallelism=False,
+ ):
+ super().__init__()
+ self.temporal = temporal
+ self.hidden_size = hidden_size
+ self.enable_flashattn = enable_flashattn
+ self._enable_sequence_parallelism = enable_sequence_parallelism
+ assert not enable_sequence_parallelism, "Sequence parallelism is not supported in STDiT3Block"
+
+ attn_cls = Attention
+ mha_cls = MultiHeadCrossAttention
+
+ self.norm1 = get_layernorm(hidden_size, eps=1e-6, affine=False, use_kernel=enable_layernorm_kernel)
+ self.attn = attn_cls(
+ hidden_size,
+ num_heads=num_heads,
+ qkv_bias=True,
+ qk_norm=qk_norm,
+ rope=rope,
+ enable_flashattn=enable_flashattn,
+ )
+ self.cross_attn = mha_cls(hidden_size, num_heads)
+ self.norm2 = get_layernorm(hidden_size, eps=1e-6, affine=False, use_kernel=enable_layernorm_kernel)
+ self.mlp = Mlp(
+ in_features=hidden_size, hidden_features=int(hidden_size * mlp_ratio), act_layer=approx_gelu, drop=0
+ )
+ self.drop_path = DropPath(drop_path) if drop_path > 0.0 else nn.Identity()
+ self.scale_shift_table = nn.Parameter(torch.randn(6, hidden_size) / hidden_size**0.5)
+
+ def t_mask_select(self, x_mask, x, masked_x, T, S):
+ # x: [B, (T, S), C]
+ # mased_x: [B, (T, S), C]
+ # x_mask: [B, T]
+ x = rearrange(x, "B (T S) C -> B T S C", T=T, S=S)
+ masked_x = rearrange(masked_x, "B (T S) C -> B T S C", T=T, S=S)
+ x = torch.where(x_mask[:, :, None, None], x, masked_x)
+ x = rearrange(x, "B T S C -> B (T S) C")
+ return x
+
+ def forward(
+ self,
+ x,
+ y,
+ t,
+ mask=None, # text mask
+ x_mask=None, # temporal mask
+ t0=None, # t with timestamp=0
+ T=None, # number of frames
+ S=None, # number of pixel patches
+ ):
+ # prepare modulate parameters
+ B, N, C = x.shape
+ shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = (
+ self.scale_shift_table[None] + t.reshape(B, 6, -1)
+ ).chunk(6, dim=1)
+ if x_mask is not None:
+ shift_msa_zero, scale_msa_zero, gate_msa_zero, shift_mlp_zero, scale_mlp_zero, gate_mlp_zero = (
+ self.scale_shift_table[None] + t0.reshape(B, 6, -1)
+ ).chunk(6, dim=1)
+
+ # modulate (attention)
+ x_m = t2i_modulate(self.norm1(x), shift_msa, scale_msa)
+ if x_mask is not None:
+ x_m_zero = t2i_modulate(self.norm1(x), shift_msa_zero, scale_msa_zero)
+ x_m = self.t_mask_select(x_mask, x_m, x_m_zero, T, S)
+
+ # attention
+ if self.temporal:
+ x_m = rearrange(x_m, "B (T S) C -> (B S) T C", T=T, S=S)
+ x_m = self.attn(x_m)
+ x_m = rearrange(x_m, "(B S) T C -> B (T S) C", T=T, S=S)
+ else:
+ x_m = rearrange(x_m, "B (T S) C -> (B T) S C", T=T, S=S)
+ x_m = self.attn(x_m)
+ x_m = rearrange(x_m, "(B T) S C -> B (T S) C", T=T, S=S)
+
+ # modulate (attention)
+ x_m_s = gate_msa * x_m
+ if x_mask is not None:
+ x_m_s_zero = gate_msa_zero * x_m
+ x_m_s = self.t_mask_select(x_mask, x_m_s, x_m_s_zero, T, S)
+
+ # residual
+ x = x + self.drop_path(x_m_s)
+
+ # cross attention
+ x = x + self.cross_attn(x, y, mask)
+
+ # modulate (MLP)
+ x_m = t2i_modulate(self.norm2(x), shift_mlp, scale_mlp)
+ if x_mask is not None:
+ x_m_zero = t2i_modulate(self.norm2(x), shift_mlp_zero, scale_mlp_zero)
+ x_m = self.t_mask_select(x_mask, x_m, x_m_zero, T, S)
+
+ # MLP
+ x_m = self.mlp(x_m)
+
+ # modulate (MLP)
+ x_m_s = gate_mlp * x_m
+ if x_mask is not None:
+ x_m_s_zero = gate_mlp_zero * x_m
+ x_m_s = self.t_mask_select(x_mask, x_m_s, x_m_s_zero, T, S)
+
+ # residual
+ x = x + self.drop_path(x_m_s)
+
+ return x
+
+
+class STDiT3(nn.Module):
+ def __init__(
+ self,
+ input_size=(None, None, None),
+ input_sq_size=512,
+ in_channels=4,
+ patch_size=(1, 2, 2),
+ hidden_size=1152,
+ depth=28,
+ num_heads=16,
+ mlp_ratio=4.0,
+ class_dropout_prob=0.1,
+ pred_sigma=True,
+ drop_path=0.0,
+ caption_channels=4096,
+ model_max_length=300,
+ dtype=torch.float32,
+ qk_norm=False,
+ enable_flashattn=False,
+ enable_layernorm_kernel=False,
+ enable_sequence_parallelism=False,
+ ):
+ super().__init__()
+ self.pred_sigma = pred_sigma
+ self.in_channels = in_channels
+ self.out_channels = in_channels * 2 if pred_sigma else in_channels
+
+ # model size related
+ self.depth = depth
+ self.mlp_ratio = mlp_ratio
+ self.hidden_size = hidden_size
+ self.num_heads = num_heads
+
+ # computation related
+ self.drop_path = drop_path
+ self.dtype = dtype
+ self.enable_flashattn = enable_flashattn
+ self.enable_layernorm_kernel = enable_layernorm_kernel
+
+ # input size related
+ self.patch_size = patch_size
+ self.input_sq_size = input_sq_size
+ self.pos_embed = PositionEmbedding2D(hidden_size)
+ self.rope = RotaryEmbedding(dim=self.hidden_size // self.num_heads)
+
+ # embedding
+ self.x_embedder = PatchEmbed3D(patch_size, in_channels, hidden_size)
+ self.t_embedder = TimestepEmbedder(hidden_size)
+ self.fps_embedder = SizeEmbedder(self.hidden_size)
+ self.t_block = nn.Sequential(
+ nn.SiLU(),
+ nn.Linear(hidden_size, 6 * hidden_size, bias=True),
+ )
+ self.y_embedder = CaptionEmbedder(
+ in_channels=caption_channels,
+ hidden_size=hidden_size,
+ uncond_prob=class_dropout_prob,
+ act_layer=approx_gelu,
+ token_num=model_max_length,
+ )
+
+ # spatial blocks
+ drop_path = [x.item() for x in torch.linspace(0, self.drop_path, depth)]
+ self.spatial_blocks = nn.ModuleList(
+ [
+ STDiT3Block(
+ hidden_size=hidden_size,
+ num_heads=num_heads,
+ mlp_ratio=mlp_ratio,
+ drop_path=drop_path[i],
+ qk_norm=qk_norm,
+ enable_flashattn=enable_flashattn,
+ enable_layernorm_kernel=enable_layernorm_kernel,
+ enable_sequence_parallelism=enable_sequence_parallelism,
+ )
+ for i in range(depth)
+ ]
+ )
+
+ # temporal blocks
+ drop_path = [x.item() for x in torch.linspace(0, self.drop_path, depth)]
+ self.temporal_blocks = nn.ModuleList(
+ [
+ STDiT3Block(
+ hidden_size=hidden_size,
+ num_heads=num_heads,
+ mlp_ratio=mlp_ratio,
+ drop_path=drop_path[i],
+ qk_norm=qk_norm,
+ enable_flashattn=enable_flashattn,
+ enable_layernorm_kernel=enable_layernorm_kernel,
+ enable_sequence_parallelism=enable_sequence_parallelism,
+ # temporal
+ temporal=True,
+ rope=self.rope.rotate_queries_or_keys,
+ )
+ for i in range(depth)
+ ]
+ )
+
+ # final layer
+ self.final_layer = T2IFinalLayer(hidden_size, np.prod(self.patch_size), self.out_channels)
+
+ def get_dynamic_size(self, x):
+ _, _, T, H, W = x.size()
+ if T % self.patch_size[0] != 0:
+ T += self.patch_size[0] - T % self.patch_size[0]
+ if H % self.patch_size[1] != 0:
+ H += self.patch_size[1] - H % self.patch_size[1]
+ if W % self.patch_size[2] != 0:
+ W += self.patch_size[2] - W % self.patch_size[2]
+ T = T // self.patch_size[0]
+ H = H // self.patch_size[1]
+ W = W // self.patch_size[2]
+ return (T, H, W)
+
+ def forward(self, x, timestep, y, mask=None, x_mask=None, fps=None, height=None, width=None, **kwargs):
+ B = x.shape[0]
+ x = x.to(self.dtype)
+ timestep = timestep.to(self.dtype)
+ y = y.to(self.dtype)
+
+ # === get pos embed ===
+ _, _, Tx, Hx, Wx = x.size()
+ T, H, W = self.get_dynamic_size(x)
+ S = H * W
+ base_size = round(S**0.5)
+ resolution_sq = (height[0].item() * width[0].item()) ** 0.5
+ scale = resolution_sq / self.input_sq_size
+
+ # scale = 4.0 # [debug]
+ # base_size = 2048 // 8 // 2 # [debug]
+
+ pos_emb = self.pos_embed(x, H, W, scale=scale, base_size=base_size)
+
+ # === get timestep embed ===
+ t = self.t_embedder(timestep, dtype=x.dtype) # [B, C]
+ fps = self.fps_embedder(fps.unsqueeze(1), B)
+
+ t = t + fps # [debug]
+
+ t_mlp = self.t_block(t)
+ t0 = t0_mlp = None
+ if x_mask is not None:
+ t0_timestep = torch.zeros_like(timestep)
+ t0 = self.t_embedder(t0_timestep, dtype=x.dtype)
+ t0 = t0 + fps
+ t0_mlp = self.t_block(t0)
+
+ # === get y embed ===
+ y = self.y_embedder(y, self.training) # [B, 1, N_token, C]
+ if mask is not None:
+ if mask.shape[0] != y.shape[0]:
+ mask = mask.repeat(y.shape[0] // mask.shape[0], 1)
+ mask = mask.squeeze(1).squeeze(1)
+ y = y.squeeze(1).masked_select(mask.unsqueeze(-1) != 0).view(1, -1, self.hidden_size)
+ y_lens = mask.sum(dim=1).tolist()
+ else:
+ y_lens = [y.shape[2]] * y.shape[0]
+ y = y.squeeze(1).view(1, -1, self.hidden_size)
+
+ # === get x embed ===
+ x = self.x_embedder(x) # [B, N, C]
+ x = rearrange(x, "B (T S) C -> B T S C", T=T, S=S)
+ x = x + pos_emb
+ x = rearrange(x, "B T S C -> B (T S) C", T=T, S=S)
+
+ # === blocks ===
+ for spatial_block, temporal_block in zip(self.spatial_blocks, self.temporal_blocks):
+ x = auto_grad_checkpoint(spatial_block, x, y, t_mlp, y_lens, x_mask, t0_mlp, T, S)
+ x = auto_grad_checkpoint(temporal_block, x, y, t_mlp, y_lens, x_mask, t0_mlp, T, S)
+
+ # === final layer ===
+ x = self.final_layer(x, t, x_mask, t0, T, S)
+ x = self.unpatchify(x, T, H, W, Tx, Hx, Wx)
+
+ # cast to float32 for better accuracy
+ x = x.to(torch.float32)
+ return x
+
+ def unpatchify(self, x, N_t, N_h, N_w, R_t, R_h, R_w):
+ """
+ Args:
+ x (torch.Tensor): of shape [B, N, C]
+
+ Return:
+ x (torch.Tensor): of shape [B, C_out, T, H, W]
+ """
+
+ # N_t, N_h, N_w = [self.input_size[i] // self.patch_size[i] for i in range(3)]
+ T_p, H_p, W_p = self.patch_size
+ x = rearrange(
+ x,
+ "B (N_t N_h N_w) (T_p H_p W_p C_out) -> B C_out (N_t T_p) (N_h H_p) (N_w W_p)",
+ N_t=N_t,
+ N_h=N_h,
+ N_w=N_w,
+ T_p=T_p,
+ H_p=H_p,
+ W_p=W_p,
+ C_out=self.out_channels,
+ )
+ # unpad
+ x = x[:, :, :R_t, :R_h, :R_w]
+ return x
+
+
[email protected]_module("STDiT3-XL/2")
+def STDiT3_XL_2(from_pretrained=None, **kwargs):
+ model = STDiT3(depth=28, hidden_size=1152, patch_size=(1, 2, 2), num_heads=16, **kwargs)
+ if from_pretrained is not None:
+ load_checkpoint(model, from_pretrained)
+ return model
+
+
[email protected]_module("STDiT3-3B/2")
+def STDiT3_3B_2(from_pretrained=None, **kwargs):
+ model = STDiT3(depth=28, hidden_size=1872, patch_size=(1, 2, 2), num_heads=26, **kwargs)
+ if from_pretrained is not None:
+ load_checkpoint(model, from_pretrained)
+ return model
diff --git a/opensora/utils/ckpt_utils.py b/opensora/utils/ckpt_utils.py
index 157e9cb..8ed0b9c 100644
--- a/opensora/utils/ckpt_utils.py
+++ b/opensora/utils/ckpt_utils.py
@@ -88,6 +88,13 @@ def reparameter(ckpt, name=None, model=None):
f"Shrink y_embedding from {ckpt['y_embedder.y_embedding'].shape[0]} to {model.y_embedder.y_embedding.shape[0]}"
)
ckpt["y_embedder.y_embedding"] = ckpt["y_embedder.y_embedding"][: model.y_embedder.y_embedding.shape[0]]
+ # stdit3 special case
+ if type(model).__name__ == "STDiT3" and "PixArt-Sigma" in name:
+ ckpt_keys = list(ckpt.keys())
+ for key in ckpt_keys:
+ if "blocks." in key:
+ ckpt[key.replace("blocks.", "spatial_blocks.")] = ckpt[key]
+ del ckpt[key]
return ckpt
diff --git a/opensora/utils/train_utils.py b/opensora/utils/train_utils.py
index 5d88c56..9ec7108 100644
--- a/opensora/utils/train_utils.py
+++ b/opensora/utils/train_utils.py
@@ -45,6 +45,8 @@ class MaskGenerator:
"mask_image_head",
"mask_image_tail",
"mask_image_head_tail",
+ "mask_random",
+ "mask_intepolate",
]
assert all(
mask_name in valid_mask_names for mask_name in mask_ratios.keys()
@@ -56,6 +58,8 @@ class MaskGenerator:
mask_ratio <= 1 for mask_ratio in mask_ratios.values()
), f"mask_ratio should be less than or equal to 1, got {mask_ratios.values()}"
# sum of mask_ratios should be 1
+ if "mask_no" not in mask_ratios:
+ mask_ratios["mask_no"] = 1.0 - sum(mask_ratios.values())
assert math.isclose(
sum(mask_ratios.values()), 1.0, abs_tol=1e-6
), f"sum of mask_ratios should be 1, got {sum(mask_ratios.values())}"
@@ -108,6 +112,12 @@ class MaskGenerator:
random_size = 1
mask[:random_size] = 0
mask[-random_size:] = 0
+ elif mask_name == "mask_intepolate":
+ random_start = random.randint(0, 1)
+ mask[random_start::2] = 0
+ elif mask_name == "mask_random":
+ mask_ratio = random.uniform(0.3, 0.7)
+ mask = torch.rand(num_frames, device=x.device) > mask_ratio
return mask
diff --git a/scripts/inference.py b/scripts/inference.py
index 179ef94..7a83ead 100644
--- a/scripts/inference.py
+++ b/scripts/inference.py
@@ -70,7 +70,6 @@ def main():
# 3.2. move to device & eval
vae = vae.to(device, dtype).eval()
model = model.to(device, dtype).eval()
- breakpoint()
# 3.3. build scheduler
scheduler = build_module(cfg.scheduler, SCHEDULERS)