Open-Sora: Democratizing Efficient Video Production for All
Python
complete 3b backbone
zhengzangwf3dcb4d6fb18
diff --git a/assets/texts/t2i_sigma.txt b/assets/texts/t2i_sigma.txtindex 7866969..fa0f0a4 100644--- a/assets/texts/t2i_sigma.txt+++ b/assets/texts/t2i_sigma.txt@@ -1,3 +1,6 @@+Eiffel Tower was Made up of more than 2 million translucent straws to look like a cloud, with the bell tower at the top of the building, Michel installed huge foam-making machines in the forest to blow huge amounts of unpredictable wet clouds in the building's classic architecture.+A gorgeously rendered papercraft world of a coral reef, rife with colorful fish and sea creatures.+Full body shot, a French woman, Photography, French Streets background, backlighting, rim light, Fujifilm.Close-up photos of models, hazy light and shadow, laser metal hair accessories, soft and beautiful, light gold pupils, white eyelashes, low saturation, real skin details, clear pores and fine lines, light reflection and refraction, ultra-clear, cinematography, award-winning works.A litter of golden retriever puppies playing in the snow. Their heads pop out of the snow, covered in.Lego model, future rocket station, intricate details, high resolution, unreal engine, UHD@@ -5,6 +8,3 @@ One giant, sharp, metal square mirror in the center of the frame, four young peoModern luxury contemporary luxury home interiors house, in the style of mimicking ruined materials, ray tracing, haunting houses, and stone, capture the essence of nature, gray and bronze, dynamic outdoor shots.Over the shoulder game perspective, game screen of Diablo 4, Inside the gorgeous palace is the wet ground, The necromancer knelt before the king, and a horde of skeletons he summoned stood at his side, cinematic light.A curvy timber house near a sea, designed by Zaha Hadid, represent the image of a cold, modern architecture, at night, white lighting, highly detailed.-Full body shot, a French woman, Photography, French Streets background, backlighting, rim light, Fujifilm.-Eiffel Tower was Made up of more than 2 million translucent straws to look like a cloud, with the bell tower at the top of the building, Michel installed huge foam-making machines in the forest to blow huge amounts of unpredictable wet clouds in the building's classic architecture.-A gorgeously rendered papercraft world of a coral reef, rife with colorful fish and sea creatures.diff --git a/configs/opensora-v1-2/inference/pixart.py b/configs/opensora-v1-2/inference/pixart.pyindex 32f4f46..1baf45d 100644--- a/configs/opensora-v1-2/inference/pixart.py+++ b/configs/opensora-v1-2/inference/pixart.py@@ -1,11 +1,12 @@num_frames = 1fps = 1-image_size = (2560, 1536)-# image_size = (2048, 2048)+# image_size = (2560, 1536)+image_size = (2048, 2048)model = dict(- type="PixArtMS-1B/2",- # from_pretrained="PixArt-Sigma-XL-2-2K-MS.pth",+ # type="PixArt-1B/2",+ type="PixArt-XL/2",+ from_pretrained="PixArt-Sigma-XL-2-2K-MS.pth",space_scale=4,no_temporal_pos_emb=True,enable_flashattn=True,diff --git a/configs/opensora-v1-2/inference/xl.py b/configs/opensora-v1-2/inference/xl.pynew file mode 100644index 0000000..00bb21d--- /dev/null+++ b/configs/opensora-v1-2/inference/xl.py@@ -0,0 +1,37 @@+num_frames = 2+fps = 1+image_size = (2560, 1536)+# image_size = (2048, 2048)+multi_resolution = "STDiT2"++model = dict(+ type="STDiT3-XL/2",+ from_pretrained="PixArt-Sigma-XL-2-2K-MS.pth",+ enable_flashattn=True,+ enable_layernorm_kernel=True,+)+vae = dict(+ type="VideoAutoencoderKL",+ micro_batch_size=4,+ from_pretrained="PixArt-alpha/pixart_sigma_sdxlvae_T5_diffusers",+ subfolder="vae",+ local_files_only=True,+)+text_encoder = dict(+ type="t5",+ from_pretrained="DeepFloyd/t5-v1_1-xxl",+ model_max_length=300,+ local_files_only=True,+)+scheduler = dict(+ type="dpm-solver",+ num_sampling_steps=14,+ cfg_scale=4.5,+)+dtype = "bf16"++# Others+batch_size = 1+seed = 42+prompt_path = "./assets/texts/t2i_sigma.txt"+save_dir = "./samples/samples/"diff --git a/configs/opensora-v1-2/train/3b.py b/configs/opensora-v1-2/train/3b.pynew file mode 100644index 0000000..d23dfd7--- /dev/null+++ b/configs/opensora-v1-2/train/3b.py@@ -0,0 +1,57 @@+# Define dataset+dataset = dict(+ type="VariableVideoTextDataset",+ frame_interval=3,+ transform_name="resize_crop",+)+bucket_config = {+ "144p": {2: (1.0, 1)},+}++# Define acceleration+num_workers = 4+num_bucket_build_workers = 16+dtype = "bf16"+grad_checkpoint = True+plugin = "zero2"+sp_size = 1++# Define model+model = dict(+ type="STDiT3-3B/2",+ from_pretrained=None,+ enable_flashattn=True,+ enable_layernorm_kernel=True,+)+vae = dict(+ type="VideoAutoencoderKL",+ micro_batch_size=4,+ from_pretrained="PixArt-alpha/pixart_sigma_sdxlvae_T5_diffusers",+ subfolder="vae",+ local_files_only=True,+)+text_encoder = dict(+ type="t5",+ from_pretrained="DeepFloyd/t5-v1_1-xxl",+ model_max_length=300,+ shardformer=True,+ local_files_only=True,+)+scheduler = dict(+ type="iddpm",+ timestep_respacing="",+)++# Others+seed = 42+outputs = "outputs"+wandb = False++epochs = 1000+log_every = 10+ckpt_every = 500+load = None++batch_size = 10 # only for logging+lr = 2e-5+grad_clip = 1.0diff --git a/configs/opensora-v1-2/train/xl.py b/configs/opensora-v1-2/train/xl.pynew file mode 100644index 0000000..adc379b--- /dev/null+++ b/configs/opensora-v1-2/train/xl.py@@ -0,0 +1,69 @@+# Define dataset+dataset = dict(+ type="VariableVideoTextDataset",+ frame_interval=3,+ transform_name="resize_crop",+)+bucket_config = {+ "240p": {16: (1.0, 2)},+}+mask_ratios = { # 20%+ "mask_random": 0.10,+ "mask_quarter_random": 0.01,+ "mask_quarter_head": 0.01,+ "mask_quarter_tail": 0.01,+ "mask_quarter_head_tail": 0.01,+ "mask_image_random": 0.01,+ "mask_image_head": 0.02,+ "mask_image_tail": 0.01,+ "mask_image_head_tail": 0.01,+ "mask_intepolate": 0.01,+}++# Define acceleration+num_workers = 4+num_bucket_build_workers = 16+dtype = "bf16"+grad_checkpoint = True+plugin = "zero2"+sp_size = 1++# Define model+model = dict(+ type="STDiT3-XL/2",+ from_pretrained=None,+ enable_flashattn=True,+ enable_layernorm_kernel=True,+)+vae = dict(+ type="VideoAutoencoderKL",+ micro_batch_size=4,+ from_pretrained="PixArt-alpha/pixart_sigma_sdxlvae_T5_diffusers",+ subfolder="vae",+ local_files_only=True,+)+text_encoder = dict(+ type="t5",+ from_pretrained="DeepFloyd/t5-v1_1-xxl",+ model_max_length=300,+ shardformer=True,+ local_files_only=True,+)+scheduler = dict(+ type="iddpm",+ timestep_respacing="",+)++# Others+seed = 42+outputs = "outputs"+wandb = False++epochs = 1000+log_every = 10+ckpt_every = 500+load = None++batch_size = 10 # only for logging+lr = 2e-5+grad_clip = 1.0diff --git a/opensora/datasets/datasets.py b/opensora/datasets/datasets.pyindex 4c2b316..ca4d70e 100644--- a/opensora/datasets/datasets.py+++ b/opensora/datasets/datasets.py@@ -24,7 +24,7 @@ class VideoTextDataset(torch.utils.data.Dataset):def __init__(self,- data_path,+ data_path=None,num_frames=16,frame_interval=1,image_size=(256, 256),@@ -107,10 +107,10 @@ class VideoTextDataset(torch.utils.data.Dataset):class VariableVideoTextDataset(VideoTextDataset):def __init__(self,- data_path,+ data_path=None,num_frames=None,frame_interval=1,- image_size=None,+ image_size=(None, None),transform_name=None,):super().__init__(data_path, num_frames, frame_interval, image_size, transform_name=None)diff --git a/opensora/models/pixart/pixart.py b/opensora/models/pixart/pixart.pyindex 5fec4d0..b5bd9ca 100644--- a/opensora/models/pixart/pixart.py+++ b/opensora/models/pixart/pixart.py@@ -211,6 +211,7 @@ class PixArt(nn.Module):# embeddingx = self.x_embedder(x) # (B, N, D)x = rearrange(x, "b (t s) d -> b t s d", t=self.num_temporal, s=self.num_spatial)+ breakpoint()x = x + self.pos_embedif not self.no_temporal_pos_emb:x = rearrange(x, "b t s d -> b s t d")@@ -387,17 +388,17 @@ def PixArt_XL_2(from_pretrained=None, **kwargs):return model[email protected]_module("PixArtMS-XL/2")-def PixArtMS_XL_2(from_pretrained=None, **kwargs):- model = PixArtMS(depth=28, hidden_size=1152, patch_size=(1, 2, 2), num_heads=16, **kwargs)[email protected]_module("PixArt-1B/2")+def PixArt_1B_2(from_pretrained=None, **kwargs):+ model = PixArt(depth=28, hidden_size=1872, patch_size=(1, 2, 2), num_heads=26, **kwargs)if from_pretrained is not None:load_checkpoint(model, from_pretrained)return model[email protected]_module("PixArtMS-1B/2")-def PixArtMS_1B_2(from_pretrained=None, **kwargs):- model = PixArtMS(depth=28, hidden_size=1872, patch_size=(1, 2, 2), num_heads=26, **kwargs)[email protected]_module("PixArtMS-XL/2")+def PixArtMS_XL_2(from_pretrained=None, **kwargs):+ model = PixArtMS(depth=28, hidden_size=1152, patch_size=(1, 2, 2), num_heads=16, **kwargs)if from_pretrained is not None:load_checkpoint(model, from_pretrained)return modeldiff --git a/opensora/models/stdit/__init__.py b/opensora/models/stdit/__init__.pyindex 605159e..e2ae7b0 100644--- a/opensora/models/stdit/__init__.py+++ b/opensora/models/stdit/__init__.py@@ -1,2 +1,3 @@from .stdit import STDiTfrom .stdit2 import STDiT2+from .stdit3 import STDiT3diff --git a/opensora/models/stdit/stdit3.py b/opensora/models/stdit/stdit3.pyindex e69de29..85f51e0 100644--- a/opensora/models/stdit/stdit3.py+++ b/opensora/models/stdit/stdit3.py@@ -0,0 +1,368 @@+import numpy as np+import torch+import torch.nn as nn+from einops import rearrange+from rotary_embedding_torch import RotaryEmbedding+from timm.models.layers import DropPath+from timm.models.vision_transformer import Mlp++from opensora.acceleration.checkpoint import auto_grad_checkpoint+from opensora.models.layers.blocks import (+ Attention,+ CaptionEmbedder,+ MultiHeadCrossAttention,+ PatchEmbed3D,+ PositionEmbedding2D,+ SizeEmbedder,+ T2IFinalLayer,+ TimestepEmbedder,+ approx_gelu,+ get_layernorm,+ t2i_modulate,+)+from opensora.registry import MODELS+from opensora.utils.ckpt_utils import load_checkpoint+++class STDiT3Block(nn.Module):+ def __init__(+ self,+ hidden_size,+ num_heads,+ mlp_ratio=4.0,+ drop_path=0.0,+ rope=None,+ qk_norm=False,+ temporal=False,+ enable_flashattn=False,+ enable_layernorm_kernel=False,+ enable_sequence_parallelism=False,+ ):+ super().__init__()+ self.temporal = temporal+ self.hidden_size = hidden_size+ self.enable_flashattn = enable_flashattn+ self._enable_sequence_parallelism = enable_sequence_parallelism+ assert not enable_sequence_parallelism, "Sequence parallelism is not supported in STDiT3Block"++ attn_cls = Attention+ mha_cls = MultiHeadCrossAttention++ self.norm1 = get_layernorm(hidden_size, eps=1e-6, affine=False, use_kernel=enable_layernorm_kernel)+ self.attn = attn_cls(+ hidden_size,+ num_heads=num_heads,+ qkv_bias=True,+ qk_norm=qk_norm,+ rope=rope,+ enable_flashattn=enable_flashattn,+ )+ self.cross_attn = mha_cls(hidden_size, num_heads)+ self.norm2 = get_layernorm(hidden_size, eps=1e-6, affine=False, use_kernel=enable_layernorm_kernel)+ self.mlp = Mlp(+ in_features=hidden_size, hidden_features=int(hidden_size * mlp_ratio), act_layer=approx_gelu, drop=0+ )+ self.drop_path = DropPath(drop_path) if drop_path > 0.0 else nn.Identity()+ self.scale_shift_table = nn.Parameter(torch.randn(6, hidden_size) / hidden_size**0.5)++ def t_mask_select(self, x_mask, x, masked_x, T, S):+ # x: [B, (T, S), C]+ # mased_x: [B, (T, S), C]+ # x_mask: [B, T]+ x = rearrange(x, "B (T S) C -> B T S C", T=T, S=S)+ masked_x = rearrange(masked_x, "B (T S) C -> B T S C", T=T, S=S)+ x = torch.where(x_mask[:, :, None, None], x, masked_x)+ x = rearrange(x, "B T S C -> B (T S) C")+ return x++ def forward(+ self,+ x,+ y,+ t,+ mask=None, # text mask+ x_mask=None, # temporal mask+ t0=None, # t with timestamp=0+ T=None, # number of frames+ S=None, # number of pixel patches+ ):+ # prepare modulate parameters+ B, N, C = x.shape+ shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = (+ self.scale_shift_table[None] + t.reshape(B, 6, -1)+ ).chunk(6, dim=1)+ if x_mask is not None:+ shift_msa_zero, scale_msa_zero, gate_msa_zero, shift_mlp_zero, scale_mlp_zero, gate_mlp_zero = (+ self.scale_shift_table[None] + t0.reshape(B, 6, -1)+ ).chunk(6, dim=1)++ # modulate (attention)+ x_m = t2i_modulate(self.norm1(x), shift_msa, scale_msa)+ if x_mask is not None:+ x_m_zero = t2i_modulate(self.norm1(x), shift_msa_zero, scale_msa_zero)+ x_m = self.t_mask_select(x_mask, x_m, x_m_zero, T, S)++ # attention+ if self.temporal:+ x_m = rearrange(x_m, "B (T S) C -> (B S) T C", T=T, S=S)+ x_m = self.attn(x_m)+ x_m = rearrange(x_m, "(B S) T C -> B (T S) C", T=T, S=S)+ else:+ x_m = rearrange(x_m, "B (T S) C -> (B T) S C", T=T, S=S)+ x_m = self.attn(x_m)+ x_m = rearrange(x_m, "(B T) S C -> B (T S) C", T=T, S=S)++ # modulate (attention)+ x_m_s = gate_msa * x_m+ if x_mask is not None:+ x_m_s_zero = gate_msa_zero * x_m+ x_m_s = self.t_mask_select(x_mask, x_m_s, x_m_s_zero, T, S)++ # residual+ x = x + self.drop_path(x_m_s)++ # cross attention+ x = x + self.cross_attn(x, y, mask)++ # modulate (MLP)+ x_m = t2i_modulate(self.norm2(x), shift_mlp, scale_mlp)+ if x_mask is not None:+ x_m_zero = t2i_modulate(self.norm2(x), shift_mlp_zero, scale_mlp_zero)+ x_m = self.t_mask_select(x_mask, x_m, x_m_zero, T, S)++ # MLP+ x_m = self.mlp(x_m)++ # modulate (MLP)+ x_m_s = gate_mlp * x_m+ if x_mask is not None:+ x_m_s_zero = gate_mlp_zero * x_m+ x_m_s = self.t_mask_select(x_mask, x_m_s, x_m_s_zero, T, S)++ # residual+ x = x + self.drop_path(x_m_s)++ return x+++class STDiT3(nn.Module):+ def __init__(+ self,+ input_size=(None, None, None),+ input_sq_size=512,+ in_channels=4,+ patch_size=(1, 2, 2),+ hidden_size=1152,+ depth=28,+ num_heads=16,+ mlp_ratio=4.0,+ class_dropout_prob=0.1,+ pred_sigma=True,+ drop_path=0.0,+ caption_channels=4096,+ model_max_length=300,+ dtype=torch.float32,+ qk_norm=False,+ enable_flashattn=False,+ enable_layernorm_kernel=False,+ enable_sequence_parallelism=False,+ ):+ super().__init__()+ self.pred_sigma = pred_sigma+ self.in_channels = in_channels+ self.out_channels = in_channels * 2 if pred_sigma else in_channels++ # model size related+ self.depth = depth+ self.mlp_ratio = mlp_ratio+ self.hidden_size = hidden_size+ self.num_heads = num_heads++ # computation related+ self.drop_path = drop_path+ self.dtype = dtype+ self.enable_flashattn = enable_flashattn+ self.enable_layernorm_kernel = enable_layernorm_kernel++ # input size related+ self.patch_size = patch_size+ self.input_sq_size = input_sq_size+ self.pos_embed = PositionEmbedding2D(hidden_size)+ self.rope = RotaryEmbedding(dim=self.hidden_size // self.num_heads)++ # embedding+ self.x_embedder = PatchEmbed3D(patch_size, in_channels, hidden_size)+ self.t_embedder = TimestepEmbedder(hidden_size)+ self.fps_embedder = SizeEmbedder(self.hidden_size)+ self.t_block = nn.Sequential(+ nn.SiLU(),+ nn.Linear(hidden_size, 6 * hidden_size, bias=True),+ )+ self.y_embedder = CaptionEmbedder(+ in_channels=caption_channels,+ hidden_size=hidden_size,+ uncond_prob=class_dropout_prob,+ act_layer=approx_gelu,+ token_num=model_max_length,+ )++ # spatial blocks+ drop_path = [x.item() for x in torch.linspace(0, self.drop_path, depth)]+ self.spatial_blocks = nn.ModuleList(+ [+ STDiT3Block(+ hidden_size=hidden_size,+ num_heads=num_heads,+ mlp_ratio=mlp_ratio,+ drop_path=drop_path[i],+ qk_norm=qk_norm,+ enable_flashattn=enable_flashattn,+ enable_layernorm_kernel=enable_layernorm_kernel,+ enable_sequence_parallelism=enable_sequence_parallelism,+ )+ for i in range(depth)+ ]+ )++ # temporal blocks+ drop_path = [x.item() for x in torch.linspace(0, self.drop_path, depth)]+ self.temporal_blocks = nn.ModuleList(+ [+ STDiT3Block(+ hidden_size=hidden_size,+ num_heads=num_heads,+ mlp_ratio=mlp_ratio,+ drop_path=drop_path[i],+ qk_norm=qk_norm,+ enable_flashattn=enable_flashattn,+ enable_layernorm_kernel=enable_layernorm_kernel,+ enable_sequence_parallelism=enable_sequence_parallelism,+ # temporal+ temporal=True,+ rope=self.rope.rotate_queries_or_keys,+ )+ for i in range(depth)+ ]+ )++ # final layer+ self.final_layer = T2IFinalLayer(hidden_size, np.prod(self.patch_size), self.out_channels)++ def get_dynamic_size(self, x):+ _, _, T, H, W = x.size()+ if T % self.patch_size[0] != 0:+ T += self.patch_size[0] - T % self.patch_size[0]+ if H % self.patch_size[1] != 0:+ H += self.patch_size[1] - H % self.patch_size[1]+ if W % self.patch_size[2] != 0:+ W += self.patch_size[2] - W % self.patch_size[2]+ T = T // self.patch_size[0]+ H = H // self.patch_size[1]+ W = W // self.patch_size[2]+ return (T, H, W)++ def forward(self, x, timestep, y, mask=None, x_mask=None, fps=None, height=None, width=None, **kwargs):+ B = x.shape[0]+ x = x.to(self.dtype)+ timestep = timestep.to(self.dtype)+ y = y.to(self.dtype)++ # === get pos embed ===+ _, _, Tx, Hx, Wx = x.size()+ T, H, W = self.get_dynamic_size(x)+ S = H * W+ base_size = round(S**0.5)+ resolution_sq = (height[0].item() * width[0].item()) ** 0.5+ scale = resolution_sq / self.input_sq_size++ # scale = 4.0 # [debug]+ # base_size = 2048 // 8 // 2 # [debug]++ pos_emb = self.pos_embed(x, H, W, scale=scale, base_size=base_size)++ # === get timestep embed ===+ t = self.t_embedder(timestep, dtype=x.dtype) # [B, C]+ fps = self.fps_embedder(fps.unsqueeze(1), B)++ t = t + fps # [debug]++ t_mlp = self.t_block(t)+ t0 = t0_mlp = None+ if x_mask is not None:+ t0_timestep = torch.zeros_like(timestep)+ t0 = self.t_embedder(t0_timestep, dtype=x.dtype)+ t0 = t0 + fps+ t0_mlp = self.t_block(t0)++ # === get y embed ===+ y = self.y_embedder(y, self.training) # [B, 1, N_token, C]+ if mask is not None:+ if mask.shape[0] != y.shape[0]:+ mask = mask.repeat(y.shape[0] // mask.shape[0], 1)+ mask = mask.squeeze(1).squeeze(1)+ y = y.squeeze(1).masked_select(mask.unsqueeze(-1) != 0).view(1, -1, self.hidden_size)+ y_lens = mask.sum(dim=1).tolist()+ else:+ y_lens = [y.shape[2]] * y.shape[0]+ y = y.squeeze(1).view(1, -1, self.hidden_size)++ # === get x embed ===+ x = self.x_embedder(x) # [B, N, C]+ x = rearrange(x, "B (T S) C -> B T S C", T=T, S=S)+ x = x + pos_emb+ x = rearrange(x, "B T S C -> B (T S) C", T=T, S=S)++ # === blocks ===+ for spatial_block, temporal_block in zip(self.spatial_blocks, self.temporal_blocks):+ x = auto_grad_checkpoint(spatial_block, x, y, t_mlp, y_lens, x_mask, t0_mlp, T, S)+ x = auto_grad_checkpoint(temporal_block, x, y, t_mlp, y_lens, x_mask, t0_mlp, T, S)++ # === final layer ===+ x = self.final_layer(x, t, x_mask, t0, T, S)+ x = self.unpatchify(x, T, H, W, Tx, Hx, Wx)++ # cast to float32 for better accuracy+ x = x.to(torch.float32)+ return x++ def unpatchify(self, x, N_t, N_h, N_w, R_t, R_h, R_w):+ """+ Args:+ x (torch.Tensor): of shape [B, N, C]++ Return:+ x (torch.Tensor): of shape [B, C_out, T, H, W]+ """++ # N_t, N_h, N_w = [self.input_size[i] // self.patch_size[i] for i in range(3)]+ T_p, H_p, W_p = self.patch_size+ x = rearrange(+ x,+ "B (N_t N_h N_w) (T_p H_p W_p C_out) -> B C_out (N_t T_p) (N_h H_p) (N_w W_p)",+ N_t=N_t,+ N_h=N_h,+ N_w=N_w,+ T_p=T_p,+ H_p=H_p,+ W_p=W_p,+ C_out=self.out_channels,+ )+ # unpad+ x = x[:, :, :R_t, :R_h, :R_w]+ return x++[email protected]_module("STDiT3-XL/2")+def STDiT3_XL_2(from_pretrained=None, **kwargs):+ model = STDiT3(depth=28, hidden_size=1152, patch_size=(1, 2, 2), num_heads=16, **kwargs)+ if from_pretrained is not None:+ load_checkpoint(model, from_pretrained)+ return model++[email protected]_module("STDiT3-3B/2")+def STDiT3_3B_2(from_pretrained=None, **kwargs):+ model = STDiT3(depth=28, hidden_size=1872, patch_size=(1, 2, 2), num_heads=26, **kwargs)+ if from_pretrained is not None:+ load_checkpoint(model, from_pretrained)+ return modeldiff --git a/opensora/utils/ckpt_utils.py b/opensora/utils/ckpt_utils.pyindex 157e9cb..8ed0b9c 100644--- a/opensora/utils/ckpt_utils.py+++ b/opensora/utils/ckpt_utils.py@@ -88,6 +88,13 @@ def reparameter(ckpt, name=None, model=None):f"Shrink y_embedding from {ckpt['y_embedder.y_embedding'].shape[0]} to {model.y_embedder.y_embedding.shape[0]}")ckpt["y_embedder.y_embedding"] = ckpt["y_embedder.y_embedding"][: model.y_embedder.y_embedding.shape[0]]+ # stdit3 special case+ if type(model).__name__ == "STDiT3" and "PixArt-Sigma" in name:+ ckpt_keys = list(ckpt.keys())+ for key in ckpt_keys:+ if "blocks." in key:+ ckpt[key.replace("blocks.", "spatial_blocks.")] = ckpt[key]+ del ckpt[key]return ckptdiff --git a/opensora/utils/train_utils.py b/opensora/utils/train_utils.pyindex 5d88c56..9ec7108 100644--- a/opensora/utils/train_utils.py+++ b/opensora/utils/train_utils.py@@ -45,6 +45,8 @@ class MaskGenerator:"mask_image_head","mask_image_tail","mask_image_head_tail",+ "mask_random",+ "mask_intepolate",]assert all(mask_name in valid_mask_names for mask_name in mask_ratios.keys()@@ -56,6 +58,8 @@ class MaskGenerator:mask_ratio <= 1 for mask_ratio in mask_ratios.values()), f"mask_ratio should be less than or equal to 1, got {mask_ratios.values()}"# sum of mask_ratios should be 1+ if "mask_no" not in mask_ratios:+ mask_ratios["mask_no"] = 1.0 - sum(mask_ratios.values())assert math.isclose(sum(mask_ratios.values()), 1.0, abs_tol=1e-6), f"sum of mask_ratios should be 1, got {sum(mask_ratios.values())}"@@ -108,6 +112,12 @@ class MaskGenerator:random_size = 1mask[:random_size] = 0mask[-random_size:] = 0+ elif mask_name == "mask_intepolate":+ random_start = random.randint(0, 1)+ mask[random_start::2] = 0+ elif mask_name == "mask_random":+ mask_ratio = random.uniform(0.3, 0.7)+ mask = torch.rand(num_frames, device=x.device) > mask_ratioreturn maskdiff --git a/scripts/inference.py b/scripts/inference.pyindex 179ef94..7a83ead 100644--- a/scripts/inference.py+++ b/scripts/inference.py@@ -70,7 +70,6 @@ def main():# 3.2. move to device & evalvae = vae.to(device, dtype).eval()model = model.to(device, dtype).eval()- breakpoint()# 3.3. build schedulerscheduler = build_module(cfg.scheduler, SCHEDULERS)