Skip to content

Open-Sora: Democratizing Efficient Video Production for All

Python

use qk-norm

zhengzangwda345719c7a1
diff --git a/configs/opensora-v1-1/inference/sample.py b/configs/opensora-v1-1/inference/sample.py
index 7e5185a..cec8073 100644
--- a/configs/opensora-v1-1/inference/sample.py
+++ b/configs/opensora-v1-1/inference/sample.py
@@ -9,6 +9,7 @@ model = dict(
type="STDiT2-XL/2",
from_pretrained=None,
input_sq_size=512,
+ qk_norm=True,
enable_flashattn=True,
enable_layernorm_kernel=True,
)
diff --git a/configs/opensora-v1-1/train/stage1-2.py b/configs/opensora-v1-1/train/stage1-2.py
index a119e37..f637b42 100644
--- a/configs/opensora-v1-1/train/stage1-2.py
+++ b/configs/opensora-v1-1/train/stage1-2.py
@@ -16,15 +16,15 @@ bucket_config = { # 1s/it
"1024": {1: (0.3, 4)},
}
mask_ratios = {
- "mask_no": 0.9,
- "mask_quarter_random": 0.01,
- "mask_quarter_head": 0.01,
- "mask_quarter_tail": 0.01,
- "mask_quarter_head_tail": 0.02,
- "mask_image_random": 0.01,
- "mask_image_head": 0.01,
- "mask_image_tail": 0.01,
- "mask_image_head_tail": 0.02,
+ "mask_no": 0.75,
+ "mask_quarter_random": 0.025,
+ "mask_quarter_head": 0.025,
+ "mask_quarter_tail": 0.025,
+ "mask_quarter_head_tail": 0.05,
+ "mask_image_random": 0.025,
+ "mask_image_head": 0.025,
+ "mask_image_tail": 0.025,
+ "mask_image_head_tail": 0.05,
}
# Define acceleration
@@ -40,6 +40,7 @@ model = dict(
type="STDiT2-XL/2",
from_pretrained=None,
input_sq_size=512, # pretrained model is trained on 512x512
+ qk_norm=True,
enable_flashattn=True,
enable_layernorm_kernel=True,
)
@@ -57,7 +58,7 @@ text_encoder = dict(
local_files_only=True,
)
scheduler = dict(
- type="iddpm-speed",
+ type="iddpm",
timestep_respacing="",
)
diff --git a/opensora/models/layers/blocks.py b/opensora/models/layers/blocks.py
index 8d3763b..616fd62 100644
--- a/opensora/models/layers/blocks.py
+++ b/opensora/models/layers/blocks.py
@@ -29,6 +29,23 @@ from opensora.acceleration.parallel_states import get_sequence_parallel_group
approx_gelu = lambda: nn.GELU(approximate="tanh")
+class LlamaRMSNorm(nn.Module):
+ def __init__(self, hidden_size, eps=1e-6):
+ """
+ LlamaRMSNorm is equivalent to T5LayerNorm
+ """
+ super().__init__()
+ self.weight = nn.Parameter(torch.ones(hidden_size))
+ self.variance_epsilon = eps
+
+ def forward(self, hidden_states):
+ input_dtype = hidden_states.dtype
+ hidden_states = hidden_states.to(torch.float32)
+ variance = hidden_states.pow(2).mean(-1, keepdim=True)
+ hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon)
+ return self.weight * hidden_states.to(input_dtype)
+
+
def get_layernorm(hidden_size: torch.Tensor, eps: float, affine: bool, use_kernel: bool):
if use_kernel:
try:
@@ -121,7 +138,7 @@ class Attention(nn.Module):
qk_norm: bool = False,
attn_drop: float = 0.0,
proj_drop: float = 0.0,
- norm_layer: nn.Module = nn.LayerNorm,
+ norm_layer: nn.Module = LlamaRMSNorm,
enable_flashattn: bool = False,
rope=None,
) -> None:
@@ -157,16 +174,15 @@ class Attention(nn.Module):
if self.rope:
q = self.rotary_emb(q)
k = self.rotary_emb(k)
+ q, k = self.q_norm(q), self.k_norm(k)
+
if enable_flashattn:
+ from flash_attn import flash_attn_func
+
# (B, #heads, N, #dim) -> (B, N, #heads, #dim)
q = q.permute(0, 2, 1, 3)
k = k.permute(0, 2, 1, 3)
v = v.permute(0, 2, 1, 3)
-
- q, k = self.q_norm(q), self.k_norm(k)
- if enable_flashattn:
- from flash_attn import flash_attn_func
-
x = flash_attn_func(
q,
k,
@@ -202,7 +218,7 @@ class SeqParallelAttention(Attention):
qk_norm: bool = False,
attn_drop: float = 0.0,
proj_drop: float = 0.0,
- norm_layer: nn.Module = nn.LayerNorm,
+ norm_layer: nn.Module = LlamaRMSNorm,
enable_flashattn: bool = False,
rope=None,
) -> None:
@@ -249,6 +265,7 @@ class SeqParallelAttention(Attention):
) # [3, B, NUM_HEAD_PER_DEVICE, N, HEAD_DIM]
qkv = qkv.permute(qkv_permute_shape)
+ # ERROR: Should qk_norm first
q, k, v = qkv.unbind(0)
q, k = self.q_norm(q), self.k_norm(k)
if self.enable_flashattn:
diff --git a/opensora/models/stdit/stdit2.py b/opensora/models/stdit/stdit2.py
index 5e53db4..73fe276 100644
--- a/opensora/models/stdit/stdit2.py
+++ b/opensora/models/stdit/stdit2.py
@@ -41,6 +41,7 @@ class STDiT2Block(nn.Module):
enable_layernorm_kernel=False,
enable_sequence_parallelism=False,
rope=None,
+ qk_norm=False,
):
super().__init__()
self.hidden_size = hidden_size
@@ -62,6 +63,7 @@ class STDiT2Block(nn.Module):
num_heads=num_heads,
qkv_bias=True,
enable_flashattn=enable_flashattn,
+ qk_norm=qk_norm,
)
self.scale_shift_table = nn.Parameter(torch.randn(6, hidden_size) / hidden_size**0.5)
@@ -83,6 +85,7 @@ class STDiT2Block(nn.Module):
qkv_bias=True,
enable_flashattn=self.enable_flashattn,
rope=rope,
+ qk_norm=qk_norm,
)
self.scale_shift_table_temporal = nn.Parameter(torch.randn(3, hidden_size) / hidden_size**0.5) # new
@@ -191,6 +194,7 @@ class STDiT2(nn.Module):
model_max_length=120,
dtype=torch.float32,
freeze=None,
+ qk_norm=False,
enable_flashattn=False,
enable_layernorm_kernel=False,
enable_sequence_parallelism=False,
@@ -239,6 +243,7 @@ class STDiT2(nn.Module):
enable_layernorm_kernel=self.enable_layernorm_kernel,
enable_sequence_parallelism=enable_sequence_parallelism,
rope=self.rope.rotate_queries_or_keys,
+ qk_norm=qk_norm,
)
for i in range(self.depth)
]