Skip to content

Open-Sora: Democratizing Efficient Video Production for All

Python

[wip]

Zangwei Zheng917643d5bdcc
diff --git a/README.md b/README.md
index 6116f8b..633cb8a 100644
--- a/README.md
+++ b/README.md
@@ -161,7 +161,7 @@ conda activate opensora
pip install torch torchvision
# install flash attention (optional)
-# set enable_flashattn=False in config to avoid using flash attention
+# set enable_flash_attn=False in config to avoid using flash attention
pip install packaging ninja
pip install flash-attn --no-build-isolation
diff --git a/configs/dit/train/16x256x256.py b/configs/dit/train/16x256x256.py
index 42845de..c36e06b 100644
--- a/configs/dit/train/16x256x256.py
+++ b/configs/dit/train/16x256x256.py
@@ -18,7 +18,7 @@ sp_size = 1
model = dict(
type="DiT-XL/2",
from_pretrained="DiT-XL-2-256x256.pt",
- enable_flashattn=True,
+ enable_flash_attn=True,
enable_layernorm_kernel=True,
)
vae = dict(
diff --git a/configs/dit/train/1x256x256.py b/configs/dit/train/1x256x256.py
index c423b24..aa5d478 100644
--- a/configs/dit/train/1x256x256.py
+++ b/configs/dit/train/1x256x256.py
@@ -19,7 +19,7 @@ sp_size = 1
model = dict(
type="DiT-XL/2",
no_temporal_pos_emb=True,
- enable_flashattn=True,
+ enable_flash_attn=True,
enable_layernorm_kernel=True,
)
vae = dict(
diff --git a/configs/latte/train/16x256x256.py b/configs/latte/train/16x256x256.py
index 41573a0..e087f8a 100644
--- a/configs/latte/train/16x256x256.py
+++ b/configs/latte/train/16x256x256.py
@@ -17,7 +17,7 @@ sp_size = 1
# Define model
model = dict(
type="Latte-XL/2",
- enable_flashattn=True,
+ enable_flash_attn=True,
enable_layernorm_kernel=True,
)
vae = dict(
diff --git a/configs/opensora-v1-1/train/benchmark.py b/configs/opensora-v1-1/train/benchmark.py
index 0a6a01b..da6cc06 100644
--- a/configs/opensora-v1-1/train/benchmark.py
+++ b/configs/opensora-v1-1/train/benchmark.py
@@ -66,7 +66,7 @@ model = dict(
input_sq_size=512, # pretrained model is trained on 512x512
qk_norm=True,
qk_norm_legacy=True,
- enable_flashattn=True,
+ enable_flash_attn=True,
enable_layernorm_kernel=True,
)
vae = dict(
diff --git a/configs/opensora-v1-1/train/image.py b/configs/opensora-v1-1/train/image.py
index de104d9..ee43c2e 100644
--- a/configs/opensora-v1-1/train/image.py
+++ b/configs/opensora-v1-1/train/image.py
@@ -30,7 +30,7 @@ model = dict(
input_sq_size=512, # pretrained model is trained on 512x512
qk_norm=True,
qk_norm_legacy=True,
- enable_flashattn=True,
+ enable_flash_attn=True,
enable_layernorm_kernel=True,
)
vae = dict(
diff --git a/configs/opensora-v1-1/train/image_rflow.py b/configs/opensora-v1-1/train/image_rflow.py
index 66c7b56..08d52ef 100644
--- a/configs/opensora-v1-1/train/image_rflow.py
+++ b/configs/opensora-v1-1/train/image_rflow.py
@@ -35,7 +35,7 @@ sp_size = 1
# type="DiT-XL/2",
# from_pretrained="/home/zhaowangbo/wangbo/PixArt-alpha/pretrained_models/PixArt-XL-2-512x512.pth",
# # input_sq_size=512, # pretrained model is trained on 512x512
-# enable_flashattn=True,
+# enable_flash_attn=True,
# enable_layernorm_kernel=True,
# )
model = dict(
@@ -44,7 +44,7 @@ model = dict(
time_scale=1.0,
no_temporal_pos_emb=True,
from_pretrained="PixArt-XL-2-512x512.pth",
- enable_flashattn=True,
+ enable_flash_attn=True,
enable_layernorm_kernel=True,
)
# model = dict(
@@ -54,7 +54,7 @@ model = dict(
# no_temporal_pos_emb=True,
# # from_pretrained="PixArt-XL-2-512x512.pth",
# from_pretrained="/home/zhaowangbo/wangbo/PixArt-alpha/pretrained_models/PixArt-XL-2-512x512.pth",
-# enable_flashattn=True,
+# enable_flash_attn=True,
# enable_layernorm_kernel=True,
# )
vae = dict(
diff --git a/configs/opensora-v1-1/train/stage1.py b/configs/opensora-v1-1/train/stage1.py
index 8bd9b35..03905fe 100644
--- a/configs/opensora-v1-1/train/stage1.py
+++ b/configs/opensora-v1-1/train/stage1.py
@@ -42,7 +42,7 @@ model = dict(
input_sq_size=512, # pretrained model is trained on 512x512
qk_norm=True,
qk_norm_legacy=True,
- enable_flashattn=True,
+ enable_flash_attn=True,
enable_layernorm_kernel=True,
)
vae = dict(
diff --git a/configs/opensora-v1-1/train/stage2.py b/configs/opensora-v1-1/train/stage2.py
index 66572d5..ad66ed1 100644
--- a/configs/opensora-v1-1/train/stage2.py
+++ b/configs/opensora-v1-1/train/stage2.py
@@ -44,7 +44,7 @@ model = dict(
input_sq_size=512, # pretrained model is trained on 512x512
qk_norm=True,
qk_norm_legacy=True,
- enable_flashattn=True,
+ enable_flash_attn=True,
enable_layernorm_kernel=True,
)
vae = dict(
diff --git a/configs/opensora-v1-1/train/stage3.py b/configs/opensora-v1-1/train/stage3.py
index db4d42b..a774af3 100644
--- a/configs/opensora-v1-1/train/stage3.py
+++ b/configs/opensora-v1-1/train/stage3.py
@@ -44,7 +44,7 @@ model = dict(
input_sq_size=512, # pretrained model is trained on 512x512
qk_norm=True,
qk_norm_legacy=True,
- enable_flashattn=True,
+ enable_flash_attn=True,
enable_layernorm_kernel=True,
)
vae = dict(
diff --git a/configs/opensora-v1-1/train/video.py b/configs/opensora-v1-1/train/video.py
index 2912a40..0a068a5 100644
--- a/configs/opensora-v1-1/train/video.py
+++ b/configs/opensora-v1-1/train/video.py
@@ -32,7 +32,7 @@ model = dict(
input_sq_size=512, # pretrained model is trained on 512x512
qk_norm=True,
qk_norm_legacy=True,
- enable_flashattn=True,
+ enable_flash_attn=True,
enable_layernorm_kernel=True,
)
vae = dict(
diff --git a/configs/opensora-v1-2/inference/sample-ref.py b/configs/opensora-v1-2/inference/sample-ref.py
index e57e2ec..4d9dd4d 100644
--- a/configs/opensora-v1-2/inference/sample-ref.py
+++ b/configs/opensora-v1-2/inference/sample-ref.py
@@ -22,7 +22,7 @@ model = dict(
type="STDiT3-XL/2",
from_pretrained=None,
qk_norm=True,
- enable_flashattn=True,
+ enable_flash_attn=True,
enable_layernorm_kernel=True,
)
vae = dict(
diff --git a/configs/opensora-v1-2/inference/sample.py b/configs/opensora-v1-2/inference/sample.py
index ae0b7ca..58306ce 100644
--- a/configs/opensora-v1-2/inference/sample.py
+++ b/configs/opensora-v1-2/inference/sample.py
@@ -14,7 +14,7 @@ model = dict(
type="STDiT3-XL/2",
from_pretrained=None,
qk_norm=True,
- enable_flashattn=True,
+ enable_flash_attn=True,
enable_layernorm_kernel=True,
)
vae = dict(
diff --git a/configs/opensora-v1-2/train/stage1-gc.py b/configs/opensora-v1-2/train/stage1-gc.py
index 3b98f78..7e58a36 100644
--- a/configs/opensora-v1-2/train/stage1-gc.py
+++ b/configs/opensora-v1-2/train/stage1-gc.py
@@ -36,7 +36,7 @@ model = dict(
type="STDiT3-XL/2",
from_pretrained=None,
qk_norm=True,
- enable_flashattn=True,
+ enable_flash_attn=True,
enable_layernorm_kernel=True,
)
vae = dict(
diff --git a/configs/opensora-v1-2/train/stage1.py b/configs/opensora-v1-2/train/stage1.py
index 2a05262..3bb43b9 100644
--- a/configs/opensora-v1-2/train/stage1.py
+++ b/configs/opensora-v1-2/train/stage1.py
@@ -34,12 +34,12 @@ model = dict(
type="STDiT3-XL/2",
from_pretrained=None,
qk_norm=True,
- enable_flashattn=True,
+ enable_flash_attn=True,
enable_layernorm_kernel=True,
)
vae = dict(
type="VideoAutoencoderPipeline",
- from_pretrained="pretrained_models/vae-v3",
+ from_pretrained="pretrained_models/vae-v2",
micro_frame_size=17,
vae_2d=dict(
type="VideoAutoencoderKL",
diff --git a/configs/opensora/inference/16x256x256.py b/configs/opensora/inference/16x256x256.py
index 50ead83..4053e12 100644
--- a/configs/opensora/inference/16x256x256.py
+++ b/configs/opensora/inference/16x256x256.py
@@ -7,7 +7,7 @@ model = dict(
type="STDiT-XL/2",
space_scale=0.5,
time_scale=1.0,
- enable_flashattn=True,
+ enable_flash_attn=True,
enable_layernorm_kernel=True,
from_pretrained="PRETRAINED_MODEL",
)
diff --git a/configs/opensora/inference/16x512x512-rflow.py b/configs/opensora/inference/16x512x512-rflow.py
index 35804d5..cf23810 100644
--- a/configs/opensora/inference/16x512x512-rflow.py
+++ b/configs/opensora/inference/16x512x512-rflow.py
@@ -7,7 +7,7 @@ model = dict(
type="STDiT-XL/2",
space_scale=1.0,
time_scale=1.0,
- enable_flashattn=True,
+ enable_flash_attn=True,
enable_layernorm_kernel=True,
from_pretrained="PRETRAINED_MODEL",
)
diff --git a/configs/opensora/inference/16x512x512.py b/configs/opensora/inference/16x512x512.py
index 58d8243..478cb5b 100644
--- a/configs/opensora/inference/16x512x512.py
+++ b/configs/opensora/inference/16x512x512.py
@@ -7,7 +7,7 @@ model = dict(
type="STDiT-XL/2",
space_scale=1.0,
time_scale=1.0,
- enable_flashattn=True,
+ enable_flash_attn=True,
enable_layernorm_kernel=True,
from_pretrained="PRETRAINED_MODEL",
)
diff --git a/configs/opensora/inference/64x512x512.py b/configs/opensora/inference/64x512x512.py
index dbbe240..03cce23 100644
--- a/configs/opensora/inference/64x512x512.py
+++ b/configs/opensora/inference/64x512x512.py
@@ -7,7 +7,7 @@ model = dict(
type="STDiT-XL/2",
space_scale=1.0,
time_scale=2 / 3,
- enable_flashattn=True,
+ enable_flash_attn=True,
enable_layernorm_kernel=True,
from_pretrained="PRETRAINED_MODEL",
)
diff --git a/configs/opensora/train/16x256x256-mask.py b/configs/opensora/train/16x256x256-mask.py
index 0e478e5..12b4b65 100644
--- a/configs/opensora/train/16x256x256-mask.py
+++ b/configs/opensora/train/16x256x256-mask.py
@@ -20,7 +20,7 @@ model = dict(
space_scale=0.5,
time_scale=1.0,
from_pretrained="PixArt-XL-2-512x512.pth",
- enable_flashattn=True,
+ enable_flash_attn=True,
enable_layernorm_kernel=True,
)
mask_ratios = {
diff --git a/configs/opensora/train/16x256x256-spee-rflow.py b/configs/opensora/train/16x256x256-spee-rflow.py
index cbb929c..809803b 100644
--- a/configs/opensora/train/16x256x256-spee-rflow.py
+++ b/configs/opensora/train/16x256x256-spee-rflow.py
@@ -22,8 +22,8 @@ model = dict(
# from_pretrained="PixArt-XL-2-512x512.pth",
# from_pretrained = "/home/zhaowangbo/wangbo/PixArt-alpha/pretrained_models/OpenSora-v1-HQ-16x512x512.pth",
# from_pretrained = "OpenSora-v1-HQ-16x512x512.pth",
- from_pretrained = "PRETRAINED_MODEL",
- enable_flashattn=True,
+ from_pretrained="PRETRAINED_MODEL",
+ enable_flash_attn=True,
enable_layernorm_kernel=True,
)
# mask_ratios = [0.5, 0.29, 0.07, 0.07, 0.07]
diff --git a/configs/opensora/train/16x256x256-spee.py b/configs/opensora/train/16x256x256-spee.py
index b46c749..3af468b 100644
--- a/configs/opensora/train/16x256x256-spee.py
+++ b/configs/opensora/train/16x256x256-spee.py
@@ -20,7 +20,7 @@ model = dict(
space_scale=0.5,
time_scale=1.0,
from_pretrained="PixArt-XL-2-512x512.pth",
- enable_flashattn=True,
+ enable_flash_attn=True,
enable_layernorm_kernel=True,
)
mask_ratios = {
diff --git a/configs/opensora/train/16x256x256.py b/configs/opensora/train/16x256x256.py
index 144adee..21ebb26 100644
--- a/configs/opensora/train/16x256x256.py
+++ b/configs/opensora/train/16x256x256.py
@@ -20,7 +20,7 @@ model = dict(
space_scale=0.5,
time_scale=1.0,
from_pretrained="PixArt-XL-2-512x512.pth",
- enable_flashattn=True,
+ enable_flash_attn=True,
enable_layernorm_kernel=True,
)
vae = dict(
diff --git a/configs/opensora/train/16x512x512.py b/configs/opensora/train/16x512x512.py
index a2a87b1..c566fd1 100644
--- a/configs/opensora/train/16x512x512.py
+++ b/configs/opensora/train/16x512x512.py
@@ -20,7 +20,7 @@ model = dict(
space_scale=1.0,
time_scale=1.0,
from_pretrained=None,
- enable_flashattn=True,
+ enable_flash_attn=True,
enable_layernorm_kernel=True,
)
vae = dict(
diff --git a/configs/opensora/train/360x512x512.py b/configs/opensora/train/360x512x512.py
index 7cb4f56..62bfd14 100644
--- a/configs/opensora/train/360x512x512.py
+++ b/configs/opensora/train/360x512x512.py
@@ -26,7 +26,7 @@ model = dict(
space_scale=1.0,
time_scale=2 / 3,
from_pretrained=None,
- enable_flashattn=True,
+ enable_flash_attn=True,
enable_layernorm_kernel=True,
enable_sequence_parallelism=True, # enable sq here
)
diff --git a/configs/opensora/train/64x512x512-sp.py b/configs/opensora/train/64x512x512-sp.py
index 880adb2..fd34a2a 100644
--- a/configs/opensora/train/64x512x512-sp.py
+++ b/configs/opensora/train/64x512x512-sp.py
@@ -20,7 +20,7 @@ model = dict(
space_scale=1.0,
time_scale=2 / 3,
from_pretrained=None,
- enable_flashattn=True,
+ enable_flash_attn=True,
enable_layernorm_kernel=True,
enable_sequence_parallelism=True, # enable sq here
)
diff --git a/configs/opensora/train/64x512x512.py b/configs/opensora/train/64x512x512.py
index b65a419..e07f8c1 100644
--- a/configs/opensora/train/64x512x512.py
+++ b/configs/opensora/train/64x512x512.py
@@ -20,7 +20,7 @@ model = dict(
space_scale=1.0,
time_scale=2 / 3,
from_pretrained=None,
- enable_flashattn=True,
+ enable_flash_attn=True,
enable_layernorm_kernel=True,
)
vae = dict(
diff --git a/configs/pixart/inference/1x20481B.py b/configs/pixart/inference/1x20481B.py
index f3bfe8c..3d5eba9 100644
--- a/configs/pixart/inference/1x20481B.py
+++ b/configs/pixart/inference/1x20481B.py
@@ -8,7 +8,7 @@ model = dict(
from_pretrained="PixArt-1B-2.pth",
space_scale=4,
no_temporal_pos_emb=True,
- enable_flashattn=True,
+ enable_flash_attn=True,
enable_layernorm_kernel=True,
base_size=2048 // 8,
)
diff --git a/configs/pixart/inference/1x2048MS.py b/configs/pixart/inference/1x2048MS.py
index 42a7fa3..a0daca4 100644
--- a/configs/pixart/inference/1x2048MS.py
+++ b/configs/pixart/inference/1x2048MS.py
@@ -8,7 +8,7 @@ model = dict(
from_pretrained="PixArt-Sigma-XL-2-2K-MS.pth",
space_scale=4,
no_temporal_pos_emb=True,
- enable_flashattn=True,
+ enable_flash_attn=True,
enable_layernorm_kernel=True,
base_size=2048 // 8,
)
diff --git a/configs/pixart/train/16x256x256.py b/configs/pixart/train/16x256x256.py
index 701b9da..66285bf 100644
--- a/configs/pixart/train/16x256x256.py
+++ b/configs/pixart/train/16x256x256.py
@@ -20,7 +20,7 @@ model = dict(
space_scale=0.5,
time_scale=1.0,
from_pretrained="PixArt-XL-2-512x512.pth",
- enable_flashattn=True,
+ enable_flash_attn=True,
enable_layernorm_kernel=True,
)
vae = dict(
diff --git a/configs/pixart/train/1x2048x2048.py b/configs/pixart/train/1x2048x2048.py
index fdd4f1d..31f1784 100644
--- a/configs/pixart/train/1x2048x2048.py
+++ b/configs/pixart/train/1x2048x2048.py
@@ -20,7 +20,7 @@ model = dict(
space_scale=4.0,
no_temporal_pos_emb=True,
from_pretrained="PixArt-1B-2.pth",
- enable_flashattn=True,
+ enable_flash_attn=True,
enable_layernorm_kernel=True,
)
diff --git a/configs/pixart/train/1x512x512-rflow.py b/configs/pixart/train/1x512x512-rflow.py
index 8925cdb..9551ce8 100644
--- a/configs/pixart/train/1x512x512-rflow.py
+++ b/configs/pixart/train/1x512x512-rflow.py
@@ -21,8 +21,8 @@ model = dict(
time_scale=1.0,
no_temporal_pos_emb=True,
# from_pretrained="PixArt-XL-2-512x512.pth",
- from_pretrained = "PRETRAINED_MODEL",
- enable_flashattn=True,
+ from_pretrained="PRETRAINED_MODEL",
+ enable_flash_attn=True,
enable_layernorm_kernel=True,
)
vae = dict(
diff --git a/configs/pixart/train/1x512x512.py b/configs/pixart/train/1x512x512.py
index 8b2b5d4..0e7a1f7 100644
--- a/configs/pixart/train/1x512x512.py
+++ b/configs/pixart/train/1x512x512.py
@@ -21,7 +21,7 @@ model = dict(
time_scale=1.0,
no_temporal_pos_emb=True,
from_pretrained="PixArt-XL-2-512x512.pth",
- enable_flashattn=True,
+ enable_flash_attn=True,
enable_layernorm_kernel=True,
)
vae = dict(
diff --git a/configs/pixart/train/64x512x512.py b/configs/pixart/train/64x512x512.py
index 3448ce7..dd59d11 100644
--- a/configs/pixart/train/64x512x512.py
+++ b/configs/pixart/train/64x512x512.py
@@ -21,7 +21,7 @@ model = dict(
space_scale=1.0,
time_scale=2 / 3,
from_pretrained=None,
- enable_flashattn=True,
+ enable_flash_attn=True,
enable_layernorm_kernel=True,
)
vae = dict(
diff --git a/docs/config.md b/docs/config.md
index 5cecc73..3ecf726 100644
--- a/docs/config.md
+++ b/docs/config.md
@@ -26,8 +26,8 @@ model = dict(
from_pretrained="PRETRAINED_MODEL", # (Optional) Load from pretrained model
input_sq_size=512, # Base spatial position embedding size
qk_norm=True, # Normalize query and key in attention
- enable_flashattn=True, # (Optional) Speed up training and inference with flash attention
- # Turn enable_flashattn to False if you skip flashattn installation
+ enable_flash_attn=True, # (Optional) Speed up training and inference with flash attention
+ # Turn enable_flash_attn to False if you skip flashattn installation
enable_layernorm_kernel=True, # (Optional) Speed up training and inference with fused kernel
# Turn enable_layernorm_kernel to False if you skip apex installation
)
@@ -177,7 +177,7 @@ model = dict(
from_pretrained=None, # Load from pretrained model
input_sq_size=512, # Base spatial position embedding size
qk_norm=True, # Normalize query and key in attention
- enable_flashattn=True, # (Optional) Speed up training and inference with flash attention
+ enable_flash_attn=True, # (Optional) Speed up training and inference with flash attention
enable_layernorm_kernel=True, # (Optional) Speed up training and inference with fused kernel
)
vae = dict(
diff --git a/docs/zh_CN/structure.md b/docs/zh_CN/structure.md
index 6e25d84..1e3460e 100644
--- a/docs/zh_CN/structure.md
+++ b/docs/zh_CN/structure.md
@@ -89,7 +89,7 @@ model = dict(
type="STDiT-XL/2", # Select model type (STDiT-XL/2, DiT-XL/2, etc.)
space_scale=1.0, # (Optional) Space positional encoding scale (new height / old height)
time_scale=2 / 3, # (Optional) Time positional encoding scale (new frame_interval / old frame_interval)
- enable_flashattn=True, # (Optional) Speed up training and inference with flash attention
+ enable_flash_attn=True, # (Optional) Speed up training and inference with flash attention
enable_layernorm_kernel=True, # (Optional) Speed up training and inference with fused kernel
from_pretrained="PRETRAINED_MODEL", # (Optional) Load from pretrained model
no_temporal_pos_emb=True, # (Optional) Disable temporal positional encoding (for image)
@@ -144,7 +144,7 @@ model = dict(
space_scale=1.0,
time_scale=2 / 3,
from_pretrained="YOUR_PRETRAINED_MODEL",
- enable_flashattn=True, # Enable flash attention
+ enable_flash_attn=True, # Enable flash attention
enable_layernorm_kernel=True, # Enable layernorm kernel
)
vae = dict(
diff --git a/opensora/models/dit/dit.py b/opensora/models/dit/dit.py
index f264f8e..68a3eb7 100644
--- a/opensora/models/dit/dit.py
+++ b/opensora/models/dit/dit.py
@@ -44,13 +44,13 @@ class DiTBlock(nn.Module):
hidden_size,
num_heads,
mlp_ratio=4.0,
- enable_flashattn=False,
+ enable_flash_attn=False,
enable_layernorm_kernel=False,
):
super().__init__()
self.hidden_size = hidden_size
self.num_heads = num_heads
- self.enable_flashattn = enable_flashattn
+ self.enable_flash_attn = enable_flash_attn
mlp_hidden_dim = int(hidden_size * mlp_ratio)
self.norm1 = get_layernorm(hidden_size, eps=1e-6, affine=False, use_kernel=enable_layernorm_kernel)
@@ -58,7 +58,7 @@ class DiTBlock(nn.Module):
hidden_size,
num_heads=num_heads,
qkv_bias=True,
- enable_flashattn=enable_flashattn,
+ enable_flash_attn=enable_flash_attn,
)
self.norm2 = get_layernorm(hidden_size, eps=1e-6, affine=False, use_kernel=enable_layernorm_kernel)
self.mlp = Mlp(in_features=hidden_size, hidden_features=mlp_hidden_dim, act_layer=approx_gelu, drop=0)
@@ -93,7 +93,7 @@ class DiT(nn.Module):
caption_channels=512,
model_max_length=77,
dtype=torch.float32,
- enable_flashattn=False,
+ enable_flash_attn=False,
enable_layernorm_kernel=False,
enable_sequence_parallelism=False,
):
@@ -111,7 +111,7 @@ class DiT(nn.Module):
self.num_heads = num_heads
self.dtype = dtype
self.use_text_encoder = not condition.startswith("label")
- if enable_flashattn:
+ if enable_flash_attn:
assert dtype in [
torch.float16,
torch.bfloat16,
@@ -143,7 +143,7 @@ class DiT(nn.Module):
hidden_size,
num_heads,
mlp_ratio=mlp_ratio,
- enable_flashattn=enable_flashattn,
+ enable_flash_attn=enable_flash_attn,
enable_layernorm_kernel=enable_layernorm_kernel,
)
for _ in range(depth)
@@ -152,7 +152,7 @@ class DiT(nn.Module):
self.final_layer = FinalLayer(hidden_size, np.prod(self.patch_size), self.out_channels)
self.initialize_weights()
- self.enable_flashattn = enable_flashattn
+ self.enable_flash_attn = enable_flash_attn
self.enable_layernorm_kernel = enable_layernorm_kernel
def get_spatial_pos_embed(self):
diff --git a/opensora/models/layers/blocks.py b/opensora/models/layers/blocks.py
index eb0b5b5..61c10f7 100644
--- a/opensora/models/layers/blocks.py
+++ b/opensora/models/layers/blocks.py
@@ -228,7 +228,7 @@ class KVCompressAttention(nn.Module):
attn_drop: float = 0.0,
proj_drop: float = 0.0,
norm_layer: nn.Module = LlamaRMSNorm,
- enable_flashattn: bool = False,
+ enable_flash_attn: bool = False,
sampling="conv",
sr_ratio=1,
mem_eff_attention=False,
@@ -240,7 +240,7 @@ class KVCompressAttention(nn.Module):
self.num_heads = num_heads
self.head_dim = dim // num_heads
self.scale = self.head_dim**-0.5
- self.enable_flashattn = enable_flashattn
+ self.enable_flash_attn = enable_flash_attn
self.qkv = nn.Linear(dim, dim * 3, bias=qkv_bias)
@@ -291,7 +291,7 @@ class KVCompressAttention(nn.Module):
new_N = N
H, W = HW
# flash attn is not memory efficient for small sequences, this is empirical
- enable_flashattn = self.enable_flashattn and (N > B)
+ enable_flash_attn = self.enable_flash_attn and (N > B)
qkv = self.qkv(x).reshape(B, N, 3, C)
q, k, v = qkv.unbind(2)
@@ -307,7 +307,7 @@ class KVCompressAttention(nn.Module):
q, k = self.q_norm(q), self.k_norm(k)
- if enable_flashattn:
+ if enable_flash_attn:
from flash_attn import flash_attn_func
x = flash_attn_func(
@@ -340,7 +340,7 @@ class KVCompressAttention(nn.Module):
x = attn @ v
x_output_shape = (B, N, C)
- if not enable_flashattn:
+ if not enable_flash_attn:
x = x.transpose(1, 2)
x = x.reshape(x_output_shape)
x = self.proj(x)
diff --git a/opensora/models/pixart/pixart.py b/opensora/models/pixart/pixart.py
index 2ae9643..02f8b67 100644
--- a/opensora/models/pixart/pixart.py
+++ b/opensora/models/pixart/pixart.py
@@ -61,13 +61,13 @@ class PixArtBlock(nn.Module):
num_heads,
mlp_ratio=4.0,
drop_path=0.0,
- enable_flashattn=False,
+ enable_flash_attn=False,
enable_layernorm_kernel=False,
enable_sequence_parallelism=False,
):
super().__init__()
self.hidden_size = hidden_size
- self.enable_flashattn = enable_flashattn
+ self.enable_flash_attn = enable_flash_attn
self._enable_sequence_parallelism = enable_sequence_parallelism
if enable_sequence_parallelism:
@@ -82,7 +82,7 @@ class PixArtBlock(nn.Module):
hidden_size,
num_heads=num_heads,
qkv_bias=True,
- enable_flashattn=enable_flashattn,
+ enable_flash_attn=enable_flash_attn,
)
self.cross_attn = self.mha_cls(hidden_size, num_heads)
self.norm2 = get_layernorm(hidden_size, eps=1e-6, affine=False, use_kernel=enable_layernorm_kernel)
@@ -130,7 +130,7 @@ class PixArt(nn.Module):
freeze=None,
space_scale=1.0,
time_scale=1.0,
- enable_flashattn=False,
+ enable_flash_attn=False,
enable_layernorm_kernel=False,
enable_sequence_parallelism=False,
base_size=None,
@@ -156,7 +156,7 @@ class PixArt(nn.Module):
self.no_temporal_pos_emb = no_temporal_pos_emb
self.depth = depth
self.mlp_ratio = mlp_ratio
- self.enable_flashattn = enable_flashattn
+ self.enable_flash_attn = enable_flash_attn
self.enable_layernorm_kernel = enable_layernorm_kernel
self.space_scale = space_scale
self.time_scale = time_scale
@@ -183,7 +183,7 @@ class PixArt(nn.Module):
num_heads,
mlp_ratio=mlp_ratio,
drop_path=drop_path[i],
- enable_flashattn=enable_flashattn,
+ enable_flash_attn=enable_flash_attn,
enable_layernorm_kernel=enable_layernorm_kernel,
)
for i in range(depth)
diff --git a/opensora/models/pixart/pixart_sigma.py b/opensora/models/pixart/pixart_sigma.py
index e815aa9..835831c 100644
--- a/opensora/models/pixart/pixart_sigma.py
+++ b/opensora/models/pixart/pixart_sigma.py
@@ -58,7 +58,7 @@ class PixArtBlock(nn.Module):
num_heads,
mlp_ratio=4.0,
drop_path=0.0,
- enable_flashattn=False,
+ enable_flash_attn=False,
enable_layernorm_kernel=False,
enable_sequence_parallelism=False,
qk_norm=False,
@@ -67,7 +67,7 @@ class PixArtBlock(nn.Module):
):
super().__init__()
self.hidden_size = hidden_size
- self.enable_flashattn = enable_flashattn
+ self.enable_flash_attn = enable_flash_attn
self._enable_sequence_parallelism = enable_sequence_parallelism
assert not enable_sequence_parallelism, "Sequence parallelism is not supported in this version."
@@ -79,7 +79,7 @@ class PixArtBlock(nn.Module):
hidden_size,
num_heads=num_heads,
qkv_bias=True,
- enable_flashattn=enable_flashattn,
+ enable_flash_attn=enable_flash_attn,
qk_norm=qk_norm,
sr_ratio=sr_ratio,
sampling=sampling,
@@ -136,7 +136,7 @@ class PixArt_Sigma(nn.Module):
qk_norm=False,
space_scale=1.0,
time_scale=1.0,
- enable_flashattn=False,
+ enable_flash_attn=False,
enable_layernorm_kernel=False,
enable_sequence_parallelism=False,
kv_compress_config=None,
@@ -159,7 +159,7 @@ class PixArt_Sigma(nn.Module):
self.no_temporal_pos_emb = no_temporal_pos_emb
self.depth = depth
self.mlp_ratio = mlp_ratio
- self.enable_flashattn = enable_flashattn
+ self.enable_flash_attn = enable_flash_attn
self.enable_layernorm_kernel = enable_layernorm_kernel
self.space_scale = space_scale
self.time_scale = time_scale
@@ -195,7 +195,7 @@ class PixArt_Sigma(nn.Module):
num_heads,
mlp_ratio=mlp_ratio,
drop_path=drop_path[i],
- enable_flashattn=enable_flashattn,
+ enable_flash_attn=enable_flash_attn,
enable_layernorm_kernel=enable_layernorm_kernel,
qk_norm=qk_norm,
sr_ratio=(
diff --git a/opensora/models/stdit/stdit.py b/opensora/models/stdit/stdit.py
index 6e16058..6882572 100644
--- a/opensora/models/stdit/stdit.py
+++ b/opensora/models/stdit/stdit.py
@@ -37,13 +37,13 @@ class STDiTBlock(nn.Module):
d_t=None,
mlp_ratio=4.0,
drop_path=0.0,
- enable_flashattn=False,
+ enable_flash_attn=False,
enable_layernorm_kernel=False,
enable_sequence_parallelism=False,
):
super().__init__()
self.hidden_size = hidden_size
- self.enable_flashattn = enable_flashattn
+ self.enable_flash_attn = enable_flash_attn
self._enable_sequence_parallelism = enable_sequence_parallelism
if enable_sequence_parallelism:
@@ -58,7 +58,7 @@ class STDiTBlock(nn.Module):
hidden_size,
num_heads=num_heads,
qkv_bias=True,
- enable_flashattn=enable_flashattn,
+ enable_flash_attn=enable_flash_attn,
)
self.cross_attn = self.mha_cls(hidden_size, num_heads)
self.norm2 = get_layernorm(hidden_size, eps=1e-6, affine=False, use_kernel=enable_layernorm_kernel)
@@ -82,7 +82,7 @@ class STDiTBlock(nn.Module):
hidden_size,
num_heads=num_heads,
qkv_bias=True,
- enable_flashattn=self.enable_flashattn,
+ enable_flash_attn=self.enable_flash_attn,
)
def t_mask_select(self, x, masked_x, x_mask):
@@ -174,7 +174,7 @@ class STDiT(nn.Module):
space_scale=1.0,
time_scale=1.0,
freeze=None,
- enable_flashattn=False,
+ enable_flash_attn=False,
enable_layernorm_kernel=False,
enable_sequence_parallelism=False,
):
@@ -194,7 +194,7 @@ class STDiT(nn.Module):
self.no_temporal_pos_emb = no_temporal_pos_emb
self.depth = depth
self.mlp_ratio = mlp_ratio
- self.enable_flashattn = enable_flashattn
+ self.enable_flash_attn = enable_flash_attn
self.enable_layernorm_kernel = enable_layernorm_kernel
self.space_scale = space_scale
self.time_scale = time_scale
@@ -221,7 +221,7 @@ class STDiT(nn.Module):
self.num_heads,
mlp_ratio=self.mlp_ratio,
drop_path=drop_path[i],
- enable_flashattn=self.enable_flashattn,
+ enable_flash_attn=self.enable_flash_attn,
enable_layernorm_kernel=self.enable_layernorm_kernel,
enable_sequence_parallelism=enable_sequence_parallelism,
d_t=self.num_temporal,
diff --git a/opensora/models/stdit/stdit3.py b/opensora/models/stdit/stdit3.py
index efafc37..f7fb2a4 100644
--- a/opensora/models/stdit/stdit3.py
+++ b/opensora/models/stdit/stdit3.py
@@ -34,14 +34,14 @@ class STDiT3Block(nn.Module):
rope=None,
qk_norm=False,
temporal=False,
- enable_flashattn=False,
+ enable_flash_attn=False,
enable_layernorm_kernel=False,
enable_sequence_parallelism=False,
):
super().__init__()
self.temporal = temporal
self.hidden_size = hidden_size
- self.enable_flashattn = enable_flashattn
+ self.enable_flash_attn = enable_flash_attn
self._enable_sequence_parallelism = enable_sequence_parallelism
assert not enable_sequence_parallelism, "Sequence parallelism is not supported in STDiT3Block"
@@ -55,7 +55,7 @@ class STDiT3Block(nn.Module):
qkv_bias=True,
qk_norm=qk_norm,
rope=rope,
- enable_flashattn=enable_flashattn,
+ enable_flash_attn=enable_flash_attn,
)
self.cross_attn = mha_cls(hidden_size, num_heads)
self.norm2 = get_layernorm(hidden_size, eps=1e-6, affine=False, use_kernel=enable_layernorm_kernel)
@@ -163,7 +163,7 @@ class STDiT3(nn.Module):
model_max_length=300,
dtype=torch.float32,
qk_norm=False,
- enable_flashattn=False,
+ enable_flash_attn=False,
enable_layernorm_kernel=False,
enable_sequence_parallelism=False,
only_train_temporal=False,
@@ -182,7 +182,7 @@ class STDiT3(nn.Module):
# computation related
self.drop_path = drop_path
self.dtype = dtype
- self.enable_flashattn = enable_flashattn
+ self.enable_flash_attn = enable_flash_attn
self.enable_layernorm_kernel = enable_layernorm_kernel
# input size related
@@ -217,7 +217,7 @@ class STDiT3(nn.Module):
mlp_ratio=mlp_ratio,
drop_path=drop_path[i],
qk_norm=qk_norm,
- enable_flashattn=enable_flashattn,
+ enable_flash_attn=enable_flash_attn,
enable_layernorm_kernel=enable_layernorm_kernel,
enable_sequence_parallelism=enable_sequence_parallelism,
)
@@ -235,7 +235,7 @@ class STDiT3(nn.Module):
mlp_ratio=mlp_ratio,
drop_path=drop_path[i],
qk_norm=qk_norm,
- enable_flashattn=enable_flashattn,
+ enable_flash_attn=enable_flash_attn,
enable_layernorm_kernel=enable_layernorm_kernel,
enable_sequence_parallelism=enable_sequence_parallelism,
# temporal
@@ -297,6 +297,7 @@ class STDiT3(nn.Module):
x = x.to(self.dtype)
timestep = timestep.to(self.dtype)
y = y.to(self.dtype)
+ breakpoint()
# === get pos embed ===
_, _, Tx, Hx, Wx = x.size()
diff --git a/scripts/train.py b/scripts/train.py
index 5adce31..184008c 100644
--- a/scripts/train.py
+++ b/scripts/train.py
@@ -133,7 +133,7 @@ def main():
input_size=latent_size,
in_channels=vae.out_channels,
caption_channels=text_encoder.output_dim,
- model_max_length=text_encoder.model_max_length
+ model_max_length=text_encoder.model_max_length,
)
model_numel, model_numel_trainable = get_model_numel(model)
logger.info(
@@ -238,6 +238,7 @@ def main():
x = vae.encode(x) # [B, C, T, H/P, W/P]
# Prepare text inputs
model_args = text_encoder.encode(y)
+ breakpoint()
# Mask
if cfg.mask_ratios is not None:
diff --git a/tests/test_attn.py b/tests/test_attn.py
index 48f8c43..4cf5844 100644
--- a/tests/test_attn.py
+++ b/tests/test_attn.py
@@ -11,7 +11,7 @@ B, S, H = 128, 32, 1152
N, D = 16, 72
-def run_attn(enable_flashattn: bool):
+def run_attn(enable_flash_attn: bool):
get_accelerator().reset_peak_memory_stats()
rope = RotaryEmbedding(D).to(device=get_current_device(), dtype=torch.bfloat16)
attn = Attention(
@@ -19,7 +19,7 @@ def run_attn(enable_flashattn: bool):
N,
qkv_bias=True,
rope=rope.rotate_queries_or_keys,
- enable_flashattn=enable_flashattn,
+ enable_flash_attn=enable_flash_attn,
).to(device=get_current_device(), dtype=torch.bfloat16)
x = torch.randn(B, S, H, device=get_current_device(), dtype=torch.bfloat16).requires_grad_()
y = attn(x)
diff --git a/tests/test_seq_parallel_attention.py b/tests/test_seq_parallel_attention.py
index 9cce2b2..841115a 100644
--- a/tests/test_seq_parallel_attention.py
+++ b/tests/test_seq_parallel_attention.py
@@ -18,14 +18,14 @@ def run_attention(rank, world_size):
torch.manual_seed(1024)
set_sequence_parallel_group(dist.group.WORLD)
- seq_parallel_attention = SeqParallelAttention(dim=256, num_heads=4, qkv_bias=True, enable_flashattn=False).cuda()
+ seq_parallel_attention = SeqParallelAttention(dim=256, num_heads=4, qkv_bias=True, enable_flash_attn=False).cuda()
torch.manual_seed(1024)
attention = Attention(
dim=256,
num_heads=4,
qkv_bias=True,
- enable_flashattn=False,
+ enable_flash_attn=False,
).cuda()
# create inputs
diff --git a/tools/architecture/net2net.py b/tools/architecture/net2net.py
index 80d346a..d5d7eb3 100644
--- a/tools/architecture/net2net.py
+++ b/tools/architecture/net2net.py
@@ -43,7 +43,7 @@ if __name__ == "__main__":
from opensora.models.pixart import PixArt_1B_2
- model = PixArt_1B_2(no_temporal_pos_emb=True, space_scale=4, enable_flashattn=True, enable_layernorm_kernel=True)
+ model = PixArt_1B_2(no_temporal_pos_emb=True, space_scale=4, enable_flash_attn=True, enable_layernorm_kernel=True)
print("load model done")
ckpt = torch.load("/home/zhouyukun/projs/opensora/pretrained_models/PixArt-Sigma-XL-2-2K-MS.pth")