Skip to content

Open-Sora: Democratizing Efficient Video Production for All

Python

resolved conflicts

FrankLeeeeeec88d2d54ea7
diff --cc .gitignore
index eb382c6,917be8b..258ec66
--- a/.gitignore
+++ b/.gitignore
@@@ -183,5 -182,7 +183,12 @@@ hostfil
gradio_cached_examples/
wandb/
++<<<<<<< HEAD
+# vae weights
+eval/vae/flolpips/weights/
++=======
+ # npm
+ node_modules/
+ package-lock.json
+ package.json
++>>>>>>> upstream/main
diff --cc configs/opensora-v1-1/inference/sample-ref.py
index c214dc8,c896cae..c4c46a2
--- a/configs/opensora-v1-1/inference/sample-ref.py
+++ b/configs/opensora-v1-1/inference/sample-ref.py
@@@ -30,11 -38,10 +38,11 @@@ reference_path =
# Define model
model = dict(
type="STDiT2-XL/2",
- from_pretrained=None,
+ from_pretrained="hpcai-tech/OpenSora-STDiT-v2-stage3",
input_sq_size=512,
qk_norm=True,
+ qk_norm_legacy=True,
- enable_flashattn=True,
+ enable_flash_attn=True,
enable_layernorm_kernel=True,
)
vae = dict(
diff --cc configs/opensora-v1-1/inference/sample.py
index 5b12cd2,29e63ca..c280046
--- a/configs/opensora-v1-1/inference/sample.py
+++ b/configs/opensora-v1-1/inference/sample.py
@@@ -7,11 -7,10 +7,11 @@@ multi_resolution = "STDiT2
# Define model
model = dict(
type="STDiT2-XL/2",
- from_pretrained=None,
+ from_pretrained="hpcai-tech/OpenSora-STDiT-v2-stage3",
input_sq_size=512,
qk_norm=True,
+ qk_norm_legacy=True,
- enable_flashattn=True,
+ enable_flash_attn=True,
enable_layernorm_kernel=True,
)
vae = dict(
diff --cc opensora/models/layers/blocks.py
index 00fcb7c,c4dd5c3..eb0b5b5
--- a/opensora/models/layers/blocks.py
+++ b/opensora/models/layers/blocks.py
@@@ -139,9 -139,8 +139,9 @@@ class Attention(nn.Module)
attn_drop: float = 0.0,
proj_drop: float = 0.0,
norm_layer: nn.Module = LlamaRMSNorm,
- enable_flashattn: bool = False,
+ enable_flash_attn: bool = False,
rope=None,
+ qk_norm_legacy: bool = False,
) -> None:
super().__init__()
assert dim % num_heads == 0, "dim should be divisible by num_heads"
@@@ -173,19 -171,13 +173,19 @@@
qkv = qkv.view(qkv_shape).permute(2, 0, 3, 1, 4)
q, k, v = qkv.unbind(0)
- # WARNING: this may be a bug
- if self.rope:
- q = self.rotary_emb(q)
- k = self.rotary_emb(k)
- q, k = self.q_norm(q), self.k_norm(k)
+ if self.qk_norm_legacy:
+ # WARNING: this may be a bug
+ if self.rope:
+ q = self.rotary_emb(q)
+ k = self.rotary_emb(k)
+ q, k = self.q_norm(q), self.k_norm(k)
+ else:
+ q, k = self.q_norm(q), self.k_norm(k)
+ if self.rope:
+ q = self.rotary_emb(q)
+ k = self.rotary_emb(k)
- if enable_flashattn:
+ if enable_flash_attn:
from flash_attn import flash_attn_func
# (B, #heads, N, #dim) -> (B, N, #heads, #dim)
@@@ -358,9 -220,8 +358,9 @@@ class SeqParallelAttention(Attention)
attn_drop: float = 0.0,
proj_drop: float = 0.0,
norm_layer: nn.Module = LlamaRMSNorm,
- enable_flashattn: bool = False,
+ enable_flash_attn: bool = False,
rope=None,
+ qk_norm_legacy: bool = False,
) -> None:
assert rope is None, "Rope is not supported in SeqParallelAttention"
super().__init__(
diff --cc opensora/models/stdit/stdit2.py
index 7eafec9,5de1769..56fd361
--- a/opensora/models/stdit/stdit2.py
+++ b/opensora/models/stdit/stdit2.py
@@@ -63,9 -51,8 +52,9 @@@ class STDiT2Block(nn.Module)
hidden_size,
num_heads=num_heads,
qkv_bias=True,
- enable_flashattn=enable_flashattn,
+ enable_flash_attn=enable_flash_attn,
qk_norm=qk_norm,
+ qk_norm_legacy=qk_norm_legacy,
)
self.scale_shift_table = nn.Parameter(torch.randn(6, hidden_size) / hidden_size**0.5)
@@@ -85,10 -72,9 +74,10 @@@
hidden_size,
num_heads=num_heads,
qkv_bias=True,
- enable_flashattn=self.enable_flashattn,
+ enable_flash_attn=self.enable_flash_attn,
rope=rope,
qk_norm=qk_norm,
+ qk_norm_legacy=qk_norm_legacy,
)
self.scale_shift_table_temporal = nn.Parameter(torch.randn(3, hidden_size) / hidden_size**0.5) # new
@@@ -195,46 -183,73 +186,75 @@@ class STDiT2Config(PretrainedConfig)
no_temporal_pos_emb=False,
caption_channels=4096,
model_max_length=120,
- dtype=torch.float32,
freeze=None,
qk_norm=False,
+ qk_norm_legacy=False,
- enable_flashattn=False,
+ enable_flash_attn=False,
enable_layernorm_kernel=False,
- enable_sequence_parallelism=False,
+ **kwargs,
):
- super().__init__()
- self.pred_sigma = pred_sigma
+ self.input_size = input_size
+ self.input_sq_size = input_sq_size
self.in_channels = in_channels
- self.out_channels = in_channels * 2 if pred_sigma else in_channels
+ self.patch_size = patch_size
self.hidden_size = hidden_size
- self.num_heads = num_heads
- self.dtype = dtype
- self.no_temporal_pos_emb = no_temporal_pos_emb
self.depth = depth
+ self.num_heads = num_heads
self.mlp_ratio = mlp_ratio
- self.enable_flashattn = enable_flashattn
+ self.class_dropout_prob = class_dropout_prob
+ self.pred_sigma = pred_sigma
+ self.drop_path = drop_path
+ self.no_temporal_pos_emb = no_temporal_pos_emb
+ self.caption_channels = caption_channels
+ self.model_max_length = model_max_length
+ self.freeze = freeze
+ self.qk_norm = qk_norm
++ self.qk_norm_legacy = qk_norm_legacy
+ self.enable_flash_attn = enable_flash_attn
self.enable_layernorm_kernel = enable_layernorm_kernel
+ super().__init__(**kwargs)
- # support dynamic input
- self.patch_size = patch_size
- self.input_size = input_size
- self.input_sq_size = input_sq_size
- self.pos_embed = PositionEmbedding2D(hidden_size)
- self.x_embedder = PatchEmbed3D(patch_size, in_channels, hidden_size)
- self.t_embedder = TimestepEmbedder(hidden_size)
- self.t_block = nn.Sequential(nn.SiLU(), nn.Linear(hidden_size, 6 * hidden_size, bias=True))
- self.t_block_temp = nn.Sequential(nn.SiLU(), nn.Linear(hidden_size, 3 * hidden_size, bias=True)) # new
+ @MODELS.register_module()
+ class STDiT2(PreTrainedModel):
+
+ config_class = STDiT2Config
+
+ def __init__(
+ self,
+ config
+ ):
+ super().__init__(config)
+ self.pred_sigma = config.pred_sigma
+ self.in_channels = config.in_channels
+ self.out_channels = config.in_channels * 2 if config.pred_sigma else config.in_channels
+ self.hidden_size = config.hidden_size
+ self.num_heads = config.num_heads
+ self.no_temporal_pos_emb = config.no_temporal_pos_emb
+ self.depth = config.depth
+ self.mlp_ratio = config.mlp_ratio
+ self.enable_flash_attn = config.enable_flash_attn
+ self.enable_layernorm_kernel = config.enable_layernorm_kernel
+
+ # support dynamic input
+ self.patch_size = config.patch_size
+ self.input_size = config.input_size
+ self.input_sq_size = config.input_sq_size
+ self.pos_embed = PositionEmbedding2D(config.hidden_size)
+
+ self.x_embedder = PatchEmbed3D(config.patch_size, config.in_channels, config.hidden_size)
+ self.t_embedder = TimestepEmbedder(config.hidden_size)
+ self.t_block = nn.Sequential(nn.SiLU(), nn.Linear(config.hidden_size, 6 * config.hidden_size, bias=True))
+ self.t_block_temp = nn.Sequential(nn.SiLU(), nn.Linear(config.hidden_size, 3 * config.hidden_size, bias=True)) # new
self.y_embedder = CaptionEmbedder(
- in_channels=caption_channels,
- hidden_size=hidden_size,
- uncond_prob=class_dropout_prob,
+ in_channels=config.caption_channels,
+ hidden_size=config.hidden_size,
+ uncond_prob=config.class_dropout_prob,
act_layer=approx_gelu,
- token_num=model_max_length,
+ token_num=config.model_max_length,
)
- drop_path = [x.item() for x in torch.linspace(0, drop_path, depth)]
+ drop_path = [x.item() for x in torch.linspace(0, config.drop_path, config.depth)]
self.rope = RotaryEmbedding(dim=self.hidden_size // self.num_heads) # new
self.blocks = nn.ModuleList(
[
@@@ -243,12 -258,10 +263,11 @@@
self.num_heads,
mlp_ratio=self.mlp_ratio,
drop_path=drop_path[i],
- enable_flashattn=self.enable_flashattn,
+ enable_flash_attn=self.enable_flash_attn,
enable_layernorm_kernel=self.enable_layernorm_kernel,
- enable_sequence_parallelism=enable_sequence_parallelism,
rope=self.rope.rotate_queries_or_keys,
- qk_norm=qk_norm,
- qk_norm_legacy=qk_norm_legacy,
+ qk_norm=config.qk_norm,
++ qk_norm_legacy=config.qk_norm_legacy,
)
for i in range(self.depth)
]