Skip to content

Open-Sora: Democratizing Efficient Video Production for All

Python

[wip] adapt multi-res

Zangwei Zheng00e21e9fedd4
diff --git a/configs/opensora-v1-2/train/adapt.py b/configs/opensora-v1-2/train/adapt.py
new file mode 100644
index 0000000..365b714
--- /dev/null
+++ b/configs/opensora-v1-2/train/adapt.py
@@ -0,0 +1,71 @@
+# Define dataset
+dataset = dict(
+ type="VariableVideoTextDataset",
+ transform_name="resize_crop",
+)
+bucket_config = {
+ # "144p": {1: (1.0, 128)},
+ # ---
+ # "256": {1: (0.8, 64)},
+ # "240p": {1: (0.8, 64)},
+ # ---
+ # "360p": {1: (0.5, 28)},
+ # "512": {1: (0.5, 28)},
+ # ---
+ "480p": {1: (0.4, 14)},
+ # ---
+ # "720p": {1: (0.4, 6)},
+ # "1024": {1: (0.3, 6)},
+ # ---
+ # "1080p": {1: (0.3, 3)},
+ # ---
+ # "2048": {1: (0.2, 2)},
+}
+
+# Define acceleration
+num_workers = 4
+num_bucket_build_workers = 16
+dtype = "bf16"
+grad_checkpoint = False
+plugin = "zero2"
+sp_size = 1
+
+# Define model
+model = dict(
+ type="STDiT3-XL/2",
+ from_pretrained="PixArt-Sigma-XL-2-2K-MS.pth",
+ enable_flashattn=True,
+ enable_layernorm_kernel=True,
+)
+vae = dict(
+ type="VideoAutoencoderKL",
+ micro_batch_size=4,
+ from_pretrained="PixArt-alpha/pixart_sigma_sdxlvae_T5_diffusers",
+ subfolder="vae",
+ local_files_only=True,
+)
+text_encoder = dict(
+ type="t5",
+ from_pretrained="DeepFloyd/t5-v1_1-xxl",
+ model_max_length=300,
+ shardformer=True,
+ local_files_only=True,
+)
+scheduler = dict(
+ type="iddpm",
+ timestep_respacing="",
+)
+
+# Others
+seed = 42
+outputs = "outputs"
+wandb = False
+
+epochs = 1000
+log_every = 10
+ckpt_every = 500
+load = None
+
+batch_size = 10 # only for logging
+lr = 2e-5
+grad_clip = 1.0
diff --git a/opensora/models/stdit/stdit3.py b/opensora/models/stdit/stdit3.py
index 85f51e0..80387f1 100644
--- a/opensora/models/stdit/stdit3.py
+++ b/opensora/models/stdit/stdit3.py
@@ -2,7 +2,6 @@ import numpy as np
import torch
import torch.nn as nn
from einops import rearrange
-from rotary_embedding_torch import RotaryEmbedding
from timm.models.layers import DropPath
from timm.models.vision_transformer import Mlp
@@ -13,7 +12,6 @@ from opensora.models.layers.blocks import (
MultiHeadCrossAttention,
PatchEmbed3D,
PositionEmbedding2D,
- SizeEmbedder,
T2IFinalLayer,
TimestepEmbedder,
approx_gelu,
@@ -188,12 +186,12 @@ class STDiT3(nn.Module):
self.patch_size = patch_size
self.input_sq_size = input_sq_size
self.pos_embed = PositionEmbedding2D(hidden_size)
- self.rope = RotaryEmbedding(dim=self.hidden_size // self.num_heads)
+ # self.rope = RotaryEmbedding(dim=self.hidden_size // self.num_heads)
# embedding
self.x_embedder = PatchEmbed3D(patch_size, in_channels, hidden_size)
self.t_embedder = TimestepEmbedder(hidden_size)
- self.fps_embedder = SizeEmbedder(self.hidden_size)
+ # self.fps_embedder = SizeEmbedder(self.hidden_size)
self.t_block = nn.Sequential(
nn.SiLU(),
nn.Linear(hidden_size, 6 * hidden_size, bias=True),
@@ -225,25 +223,25 @@ class STDiT3(nn.Module):
)
# temporal blocks
- drop_path = [x.item() for x in torch.linspace(0, self.drop_path, depth)]
- self.temporal_blocks = nn.ModuleList(
- [
- STDiT3Block(
- hidden_size=hidden_size,
- num_heads=num_heads,
- mlp_ratio=mlp_ratio,
- drop_path=drop_path[i],
- qk_norm=qk_norm,
- enable_flashattn=enable_flashattn,
- enable_layernorm_kernel=enable_layernorm_kernel,
- enable_sequence_parallelism=enable_sequence_parallelism,
- # temporal
- temporal=True,
- rope=self.rope.rotate_queries_or_keys,
- )
- for i in range(depth)
- ]
- )
+ # drop_path = [x.item() for x in torch.linspace(0, self.drop_path, depth)]
+ # self.temporal_blocks = nn.ModuleList(
+ # [
+ # STDiT3Block(
+ # hidden_size=hidden_size,
+ # num_heads=num_heads,
+ # mlp_ratio=mlp_ratio,
+ # drop_path=drop_path[i],
+ # qk_norm=qk_norm,
+ # enable_flashattn=enable_flashattn,
+ # enable_layernorm_kernel=enable_layernorm_kernel,
+ # enable_sequence_parallelism=enable_sequence_parallelism,
+ # # temporal
+ # temporal=True,
+ # rope=self.rope.rotate_queries_or_keys,
+ # )
+ # for i in range(depth)
+ # ]
+ # )
# final layer
self.final_layer = T2IFinalLayer(hidden_size, np.prod(self.patch_size), self.out_channels)
@@ -262,7 +260,7 @@ class STDiT3(nn.Module):
return (T, H, W)
def forward(self, x, timestep, y, mask=None, x_mask=None, fps=None, height=None, width=None, **kwargs):
- B = x.shape[0]
+ x.shape[0]
x = x.to(self.dtype)
timestep = timestep.to(self.dtype)
y = y.to(self.dtype)
@@ -282,9 +280,9 @@ class STDiT3(nn.Module):
# === get timestep embed ===
t = self.t_embedder(timestep, dtype=x.dtype) # [B, C]
- fps = self.fps_embedder(fps.unsqueeze(1), B)
+ # fps = self.fps_embedder(fps.unsqueeze(1), B)
- t = t + fps # [debug]
+ # t = t + fps # [debug]
t_mlp = self.t_block(t)
t0 = t0_mlp = None
@@ -313,9 +311,10 @@ class STDiT3(nn.Module):
x = rearrange(x, "B T S C -> B (T S) C", T=T, S=S)
# === blocks ===
- for spatial_block, temporal_block in zip(self.spatial_blocks, self.temporal_blocks):
+ # for spatial_block, temporal_block in zip(self.spatial_blocks, self.temporal_blocks):
+ for spatial_block in self.spatial_blocks:
x = auto_grad_checkpoint(spatial_block, x, y, t_mlp, y_lens, x_mask, t0_mlp, T, S)
- x = auto_grad_checkpoint(temporal_block, x, y, t_mlp, y_lens, x_mask, t0_mlp, T, S)
+ # x = auto_grad_checkpoint(temporal_block, x, y, t_mlp, y_lens, x_mask, t0_mlp, T, S)
# === final layer ===
x = self.final_layer(x, t, x_mask, t0, T, S)