Open-Sora: Democratizing Efficient Video Production for All
Python
use qk-norm
zhengzangwda345719c7a1
diff --git a/configs/opensora-v1-1/inference/sample.py b/configs/opensora-v1-1/inference/sample.pyindex 7e5185a..cec8073 100644--- a/configs/opensora-v1-1/inference/sample.py+++ b/configs/opensora-v1-1/inference/sample.py@@ -9,6 +9,7 @@ model = dict(type="STDiT2-XL/2",from_pretrained=None,input_sq_size=512,+ qk_norm=True,enable_flashattn=True,enable_layernorm_kernel=True,)diff --git a/configs/opensora-v1-1/train/stage1-2.py b/configs/opensora-v1-1/train/stage1-2.pyindex a119e37..f637b42 100644--- a/configs/opensora-v1-1/train/stage1-2.py+++ b/configs/opensora-v1-1/train/stage1-2.py@@ -16,15 +16,15 @@ bucket_config = { # 1s/it"1024": {1: (0.3, 4)},}mask_ratios = {- "mask_no": 0.9,- "mask_quarter_random": 0.01,- "mask_quarter_head": 0.01,- "mask_quarter_tail": 0.01,- "mask_quarter_head_tail": 0.02,- "mask_image_random": 0.01,- "mask_image_head": 0.01,- "mask_image_tail": 0.01,- "mask_image_head_tail": 0.02,+ "mask_no": 0.75,+ "mask_quarter_random": 0.025,+ "mask_quarter_head": 0.025,+ "mask_quarter_tail": 0.025,+ "mask_quarter_head_tail": 0.05,+ "mask_image_random": 0.025,+ "mask_image_head": 0.025,+ "mask_image_tail": 0.025,+ "mask_image_head_tail": 0.05,}# Define acceleration@@ -40,6 +40,7 @@ model = dict(type="STDiT2-XL/2",from_pretrained=None,input_sq_size=512, # pretrained model is trained on 512x512+ qk_norm=True,enable_flashattn=True,enable_layernorm_kernel=True,)@@ -57,7 +58,7 @@ text_encoder = dict(local_files_only=True,)scheduler = dict(- type="iddpm-speed",+ type="iddpm",timestep_respacing="",)diff --git a/opensora/models/layers/blocks.py b/opensora/models/layers/blocks.pyindex 8d3763b..616fd62 100644--- a/opensora/models/layers/blocks.py+++ b/opensora/models/layers/blocks.py@@ -29,6 +29,23 @@ from opensora.acceleration.parallel_states import get_sequence_parallel_groupapprox_gelu = lambda: nn.GELU(approximate="tanh")+class LlamaRMSNorm(nn.Module):+ def __init__(self, hidden_size, eps=1e-6):+ """+ LlamaRMSNorm is equivalent to T5LayerNorm+ """+ super().__init__()+ self.weight = nn.Parameter(torch.ones(hidden_size))+ self.variance_epsilon = eps++ def forward(self, hidden_states):+ input_dtype = hidden_states.dtype+ hidden_states = hidden_states.to(torch.float32)+ variance = hidden_states.pow(2).mean(-1, keepdim=True)+ hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon)+ return self.weight * hidden_states.to(input_dtype)++def get_layernorm(hidden_size: torch.Tensor, eps: float, affine: bool, use_kernel: bool):if use_kernel:try:@@ -121,7 +138,7 @@ class Attention(nn.Module):qk_norm: bool = False,attn_drop: float = 0.0,proj_drop: float = 0.0,- norm_layer: nn.Module = nn.LayerNorm,+ norm_layer: nn.Module = LlamaRMSNorm,enable_flashattn: bool = False,rope=None,) -> None:@@ -157,16 +174,15 @@ class Attention(nn.Module):if self.rope:q = self.rotary_emb(q)k = self.rotary_emb(k)+ q, k = self.q_norm(q), self.k_norm(k)+if enable_flashattn:+ from flash_attn import flash_attn_func+# (B, #heads, N, #dim) -> (B, N, #heads, #dim)q = q.permute(0, 2, 1, 3)k = k.permute(0, 2, 1, 3)v = v.permute(0, 2, 1, 3)-- q, k = self.q_norm(q), self.k_norm(k)- if enable_flashattn:- from flash_attn import flash_attn_func-x = flash_attn_func(q,k,@@ -202,7 +218,7 @@ class SeqParallelAttention(Attention):qk_norm: bool = False,attn_drop: float = 0.0,proj_drop: float = 0.0,- norm_layer: nn.Module = nn.LayerNorm,+ norm_layer: nn.Module = LlamaRMSNorm,enable_flashattn: bool = False,rope=None,) -> None:@@ -249,6 +265,7 @@ class SeqParallelAttention(Attention):) # [3, B, NUM_HEAD_PER_DEVICE, N, HEAD_DIM]qkv = qkv.permute(qkv_permute_shape)+ # ERROR: Should qk_norm firstq, k, v = qkv.unbind(0)q, k = self.q_norm(q), self.k_norm(k)if self.enable_flashattn:diff --git a/opensora/models/stdit/stdit2.py b/opensora/models/stdit/stdit2.pyindex 5e53db4..73fe276 100644--- a/opensora/models/stdit/stdit2.py+++ b/opensora/models/stdit/stdit2.py@@ -41,6 +41,7 @@ class STDiT2Block(nn.Module):enable_layernorm_kernel=False,enable_sequence_parallelism=False,rope=None,+ qk_norm=False,):super().__init__()self.hidden_size = hidden_size@@ -62,6 +63,7 @@ class STDiT2Block(nn.Module):num_heads=num_heads,qkv_bias=True,enable_flashattn=enable_flashattn,+ qk_norm=qk_norm,)self.scale_shift_table = nn.Parameter(torch.randn(6, hidden_size) / hidden_size**0.5)@@ -83,6 +85,7 @@ class STDiT2Block(nn.Module):qkv_bias=True,enable_flashattn=self.enable_flashattn,rope=rope,+ qk_norm=qk_norm,)self.scale_shift_table_temporal = nn.Parameter(torch.randn(3, hidden_size) / hidden_size**0.5) # new@@ -191,6 +194,7 @@ class STDiT2(nn.Module):model_max_length=120,dtype=torch.float32,freeze=None,+ qk_norm=False,enable_flashattn=False,enable_layernorm_kernel=False,enable_sequence_parallelism=False,@@ -239,6 +243,7 @@ class STDiT2(nn.Module):enable_layernorm_kernel=self.enable_layernorm_kernel,enable_sequence_parallelism=enable_sequence_parallelism,rope=self.rope.rotate_queries_or_keys,+ qk_norm=qk_norm,)for i in range(self.depth)]