Open-Sora: Democratizing Efficient Video Production for All
Python
[wip] adapt multi-res
Zangwei Zheng00e21e9fedd4
diff --git a/configs/opensora-v1-2/train/adapt.py b/configs/opensora-v1-2/train/adapt.pynew file mode 100644index 0000000..365b714--- /dev/null+++ b/configs/opensora-v1-2/train/adapt.py@@ -0,0 +1,71 @@+# Define dataset+dataset = dict(+ type="VariableVideoTextDataset",+ transform_name="resize_crop",+)+bucket_config = {+ # "144p": {1: (1.0, 128)},+ # ---+ # "256": {1: (0.8, 64)},+ # "240p": {1: (0.8, 64)},+ # ---+ # "360p": {1: (0.5, 28)},+ # "512": {1: (0.5, 28)},+ # ---+ "480p": {1: (0.4, 14)},+ # ---+ # "720p": {1: (0.4, 6)},+ # "1024": {1: (0.3, 6)},+ # ---+ # "1080p": {1: (0.3, 3)},+ # ---+ # "2048": {1: (0.2, 2)},+}++# Define acceleration+num_workers = 4+num_bucket_build_workers = 16+dtype = "bf16"+grad_checkpoint = False+plugin = "zero2"+sp_size = 1++# Define model+model = dict(+ type="STDiT3-XL/2",+ from_pretrained="PixArt-Sigma-XL-2-2K-MS.pth",+ enable_flashattn=True,+ enable_layernorm_kernel=True,+)+vae = dict(+ type="VideoAutoencoderKL",+ micro_batch_size=4,+ from_pretrained="PixArt-alpha/pixart_sigma_sdxlvae_T5_diffusers",+ subfolder="vae",+ local_files_only=True,+)+text_encoder = dict(+ type="t5",+ from_pretrained="DeepFloyd/t5-v1_1-xxl",+ model_max_length=300,+ shardformer=True,+ local_files_only=True,+)+scheduler = dict(+ type="iddpm",+ timestep_respacing="",+)++# Others+seed = 42+outputs = "outputs"+wandb = False++epochs = 1000+log_every = 10+ckpt_every = 500+load = None++batch_size = 10 # only for logging+lr = 2e-5+grad_clip = 1.0diff --git a/opensora/models/stdit/stdit3.py b/opensora/models/stdit/stdit3.pyindex 85f51e0..80387f1 100644--- a/opensora/models/stdit/stdit3.py+++ b/opensora/models/stdit/stdit3.py@@ -2,7 +2,6 @@ import numpy as npimport torchimport torch.nn as nnfrom einops import rearrange-from rotary_embedding_torch import RotaryEmbeddingfrom timm.models.layers import DropPathfrom timm.models.vision_transformer import Mlp@@ -13,7 +12,6 @@ from opensora.models.layers.blocks import (MultiHeadCrossAttention,PatchEmbed3D,PositionEmbedding2D,- SizeEmbedder,T2IFinalLayer,TimestepEmbedder,approx_gelu,@@ -188,12 +186,12 @@ class STDiT3(nn.Module):self.patch_size = patch_sizeself.input_sq_size = input_sq_sizeself.pos_embed = PositionEmbedding2D(hidden_size)- self.rope = RotaryEmbedding(dim=self.hidden_size // self.num_heads)+ # self.rope = RotaryEmbedding(dim=self.hidden_size // self.num_heads)# embeddingself.x_embedder = PatchEmbed3D(patch_size, in_channels, hidden_size)self.t_embedder = TimestepEmbedder(hidden_size)- self.fps_embedder = SizeEmbedder(self.hidden_size)+ # self.fps_embedder = SizeEmbedder(self.hidden_size)self.t_block = nn.Sequential(nn.SiLU(),nn.Linear(hidden_size, 6 * hidden_size, bias=True),@@ -225,25 +223,25 @@ class STDiT3(nn.Module):)# temporal blocks- drop_path = [x.item() for x in torch.linspace(0, self.drop_path, depth)]- self.temporal_blocks = nn.ModuleList(- [- STDiT3Block(- hidden_size=hidden_size,- num_heads=num_heads,- mlp_ratio=mlp_ratio,- drop_path=drop_path[i],- qk_norm=qk_norm,- enable_flashattn=enable_flashattn,- enable_layernorm_kernel=enable_layernorm_kernel,- enable_sequence_parallelism=enable_sequence_parallelism,- # temporal- temporal=True,- rope=self.rope.rotate_queries_or_keys,- )- for i in range(depth)- ]- )+ # drop_path = [x.item() for x in torch.linspace(0, self.drop_path, depth)]+ # self.temporal_blocks = nn.ModuleList(+ # [+ # STDiT3Block(+ # hidden_size=hidden_size,+ # num_heads=num_heads,+ # mlp_ratio=mlp_ratio,+ # drop_path=drop_path[i],+ # qk_norm=qk_norm,+ # enable_flashattn=enable_flashattn,+ # enable_layernorm_kernel=enable_layernorm_kernel,+ # enable_sequence_parallelism=enable_sequence_parallelism,+ # # temporal+ # temporal=True,+ # rope=self.rope.rotate_queries_or_keys,+ # )+ # for i in range(depth)+ # ]+ # )# final layerself.final_layer = T2IFinalLayer(hidden_size, np.prod(self.patch_size), self.out_channels)@@ -262,7 +260,7 @@ class STDiT3(nn.Module):return (T, H, W)def forward(self, x, timestep, y, mask=None, x_mask=None, fps=None, height=None, width=None, **kwargs):- B = x.shape[0]+ x.shape[0]x = x.to(self.dtype)timestep = timestep.to(self.dtype)y = y.to(self.dtype)@@ -282,9 +280,9 @@ class STDiT3(nn.Module):# === get timestep embed ===t = self.t_embedder(timestep, dtype=x.dtype) # [B, C]- fps = self.fps_embedder(fps.unsqueeze(1), B)+ # fps = self.fps_embedder(fps.unsqueeze(1), B)- t = t + fps # [debug]+ # t = t + fps # [debug]t_mlp = self.t_block(t)t0 = t0_mlp = None@@ -313,9 +311,10 @@ class STDiT3(nn.Module):x = rearrange(x, "B T S C -> B (T S) C", T=T, S=S)# === blocks ===- for spatial_block, temporal_block in zip(self.spatial_blocks, self.temporal_blocks):+ # for spatial_block, temporal_block in zip(self.spatial_blocks, self.temporal_blocks):+ for spatial_block in self.spatial_blocks:x = auto_grad_checkpoint(spatial_block, x, y, t_mlp, y_lens, x_mask, t0_mlp, T, S)- x = auto_grad_checkpoint(temporal_block, x, y, t_mlp, y_lens, x_mask, t0_mlp, T, S)+ # x = auto_grad_checkpoint(temporal_block, x, y, t_mlp, y_lens, x_mask, t0_mlp, T, S)# === final layer ===x = self.final_layer(x, t, x_mask, t0, T, S)